4 * DEBUG: section 33 Client-side Routines
5 * AUTHOR: Duane Wessels
7 * SQUID Web Proxy Cache http://www.squid-cache.org/
8 * ----------------------------------------------------------
10 * Squid is the result of efforts by numerous individuals from
11 * the Internet community; see the CONTRIBUTORS file for full
12 * details. Many organizations have provided support for Squid's
13 * development; see the SPONSORS file for full details. Squid is
14 * Copyrighted (C) 2001 by the Regents of the University of
15 * California; see the COPYRIGHT file for full details. Squid
16 * incorporates software developed and/or copyrighted by other
17 * sources; see the CREDITS file for full details.
19 * This program is free software; you can redistribute it and/or modify
20 * it under the terms of the GNU General Public License as published by
21 * the Free Software Foundation; either version 2 of the License, or
22 * (at your option) any later version.
24 * This program is distributed in the hope that it will be useful,
25 * but WITHOUT ANY WARRANTY; without even the implied warranty of
26 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 * GNU General Public License for more details.
29 * You should have received a copy of the GNU General Public License
30 * along with this program; if not, write to the Free Software
31 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
36 \defgroup ClientSide Client-Side Logics
38 \section cserrors Errors and client side
40 \par Problem the first:
41 * the store entry is no longer authoritative on the
42 * reply status. EBITTEST (E_ABORT) is no longer a valid test outside
43 * of client_side_reply.c.
44 * Problem the second: resources are wasted if we delay in cleaning up.
45 * Problem the third we can't depend on a connection close to clean up.
47 \par Nice thing the first:
48 * Any step in the stream can callback with data
49 * representing an error.
50 * Nice thing the second: once you stop requesting reads from upstream,
51 * upstream can be stopped too.
54 * Error has a callback mechanism to hand over a membuf
55 * with the error content. The failing node pushes that back as the
56 * reply. Can this be generalised to reduce duplicate efforts?
57 * A: Possibly. For now, only one location uses this.
58 * How to deal with pre-stream errors?
59 * Tell client_side_reply that we *want* an error page before any
60 * stream calls occur. Then we simply read as normal.
63 \section pconn_logic Persistent connection logic:
66 * requests (httpClientRequest structs) get added to the connection
67 * list, with the current one being chr
70 * The request is *immediately* kicked off, and data flows through
71 * to clientSocketRecipient.
74 * If the data that arrives at clientSocketRecipient is not for the current
75 * request, clientSocketRecipient simply returns, without requesting more
76 * data, or sending it.
79 * ClientKeepAliveNextRequest will then detect the presence of data in
80 * the next ClientHttpRequest, and will send it, restablishing the
86 #include "acl/FilledChecklist.h"
87 #include "auth/UserRequest.h"
88 #include "base/Subscription.h"
89 #include "base/TextException.h"
90 #include "ChunkedCodingParser.h"
91 #include "client_side.h"
92 #include "client_side_reply.h"
93 #include "client_side_request.h"
94 #include "ClientRequestContext.h"
95 #include "clientStream.h"
97 #include "comm/Connection.h"
98 #include "comm/ConnAcceptor.h"
99 #include "eui/Config.h"
101 #include "HttpHdrContRange.h"
102 #include "HttpReply.h"
103 #include "HttpRequest.h"
104 #include "ident/Config.h"
105 #include "ident/Ident.h"
106 #include "ip/Intercept.h"
107 #include "ipc/StartListening.h"
109 #include "MemObject.h"
110 #include "ProtoPort.h"
112 #include "SquidTime.h"
116 #include "ClientInfo.h"
120 #define comm_close comm_lingering_close
123 /// dials clientListenerConnectionOpened call
124 class ListeningStartedDialer
: public CallDialer
, public Ipc::StartListeningCb
127 typedef void (*Handler
)(int errNo
, http_port_list
*portCfg
, bool uses_ssl
);
128 ListeningStartedDialer(Handler aHandler
, http_port_list
*aPortCfg
, bool aSslFlag
):
129 handler(aHandler
), portCfg(aPortCfg
), uses_ssl(aSslFlag
) {}
131 virtual void print(std::ostream
&os
) const {
133 ", " << (uses_ssl
? "SSL " :"") << "port=" << (void*)portCfg
<< ')';
136 virtual bool canDial(AsyncCall
&) const { return true; }
137 virtual void dial(AsyncCall
&) { (handler
)(errNo
, portCfg
, uses_ssl
); }
143 http_port_list
*portCfg
; ///< from Config.Sockaddr.http
148 static void clientListenerConnectionOpened(int errNo
, http_port_list
*s
, bool uses_ssl
);
150 /* our socket-related context */
153 CBDATA_CLASS_INIT(ClientSocketContext
);
156 ClientSocketContext::operator new (size_t byteCount
)
158 /* derived classes with different sizes must implement their own new */
159 assert (byteCount
== sizeof (ClientSocketContext
));
160 CBDATA_INIT_TYPE(ClientSocketContext
);
161 return cbdataAlloc(ClientSocketContext
);
165 ClientSocketContext::operator delete (void *address
)
167 cbdataFree (address
);
170 /* Local functions */
171 /* ClientSocketContext */
172 static ClientSocketContext
*ClientSocketContextNew(const Comm::ConnectionPointer
&clientConn
, ClientHttpRequest
*);
174 static IOCB clientWriteComplete
;
175 static IOCB clientWriteBodyComplete
;
176 static IOACB httpAccept
;
177 static IOACB httpsAccept
;
178 static bool clientParseRequest(ConnStateData
* conn
, bool &do_next_read
);
179 static PF clientLifetimeTimeout
;
180 static ClientSocketContext
*parseHttpRequestAbort(ConnStateData
* conn
, const char *uri
);
181 static ClientSocketContext
*parseHttpRequest(ConnStateData
*, HttpParser
*, HttpRequestMethod
*, HttpVersion
*);
183 static IDCB clientIdentDone
;
185 static CSCB clientSocketRecipient
;
186 static CSD clientSocketDetach
;
187 static void clientSetKeepaliveFlag(ClientHttpRequest
*);
188 static int clientIsContentLengthValid(HttpRequest
* r
);
189 static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength
);
191 static void clientUpdateStatHistCounters(log_type logType
, int svc_time
);
192 static void clientUpdateStatCounters(log_type logType
);
193 static void clientUpdateHierCounters(HierarchyLogEntry
*);
194 static bool clientPingHasFinished(ping_data
const *aPing
);
195 void prepareLogWithRequestDetails(HttpRequest
*, AccessLogEntry
*);
197 static bool connIsUsable(ConnStateData
* conn
);
199 static int responseFinishedOrFailed(HttpReply
* rep
, StoreIOBuffer
const &receivedData
);
200 static void ClientSocketContextPushDeferredIfNeeded(ClientSocketContext::Pointer deferredRequest
, ConnStateData
* conn
);
201 static void clientUpdateSocketStats(log_type logType
, size_t size
);
203 char *skipLeadingSpace(char *aString
);
204 static void connNoteUseOfBuffer(ConnStateData
* conn
, size_t byteCount
);
206 static ConnStateData
*connStateCreate(const Comm::ConnectionPointer
&client
, http_port_list
*port
);
209 const Comm::ConnectionPointer
&
210 ClientSocketContext::clientConn() const
212 assert (clientConnection
!= NULL
);
213 return clientConnection
;
217 ClientSocketContext::getTail() const
219 if (http
->client_stream
.tail
)
220 return (clientStreamNode
*)http
->client_stream
.tail
->data
;
226 ClientSocketContext::getClientReplyContext() const
228 return (clientStreamNode
*)http
->client_stream
.tail
->prev
->data
;
232 * This routine should be called to grow the inbuf and then
236 ConnStateData::readSomeData()
241 debugs(33, 4, "clientReadSomeData: FD " << clientConn
->fd
<< ": reading request...");
243 makeSpaceAvailable();
245 typedef CommCbMemFunT
<ConnStateData
, CommIoCbParams
> Dialer
;
246 reader
= JobCallback(33, 5, Dialer
, this, ConnStateData::clientReadRequest
);
247 comm_read(clientConn
, in
.addressToReadInto(), getAvailableBufferLength(), reader
);
252 ClientSocketContext::removeFromConnectionList(ConnStateData
* conn
)
254 ClientSocketContext::Pointer
*tempContextPointer
;
255 assert(conn
!= NULL
&& cbdataReferenceValid(conn
));
256 assert(conn
->getCurrentContext() != NULL
);
257 /* Unlink us from the connection request list */
258 tempContextPointer
= & conn
->currentobject
;
260 while (tempContextPointer
->getRaw()) {
261 if (*tempContextPointer
== this)
264 tempContextPointer
= &(*tempContextPointer
)->next
;
267 assert(tempContextPointer
->getRaw() != NULL
);
268 *tempContextPointer
= next
;
272 ClientSocketContext::~ClientSocketContext()
274 clientStreamNode
*node
= getTail();
277 ClientSocketContext
*streamContext
= dynamic_cast<ClientSocketContext
*> (node
->data
.getRaw());
280 /* We are *always* the tail - prevent recursive free */
281 assert(this == streamContext
);
287 deRegisterWithConn();
289 httpRequestFree(http
);
291 /* clean up connection links to us */
292 assert(this != next
.getRaw());
296 ClientSocketContext::registerWithConn()
298 assert (!connRegistered_
);
300 assert (http
->getConn() != NULL
);
301 connRegistered_
= true;
302 http
->getConn()->addContextToQueue(this);
306 ClientSocketContext::deRegisterWithConn()
308 assert (connRegistered_
);
309 removeFromConnectionList(http
->getConn());
310 connRegistered_
= false;
314 ClientSocketContext::connIsFinished()
317 assert (http
->getConn() != NULL
);
318 deRegisterWithConn();
319 /* we can't handle any more stream data - detach */
320 clientStreamDetach(getTail(), http
);
323 ClientSocketContext::ClientSocketContext() : http(NULL
), reply(NULL
), next(NULL
),
325 mayUseConnection_ (false),
326 connRegistered_ (false)
328 memset (reqbuf
, '\0', sizeof (reqbuf
));
331 deferredparams
.node
= NULL
;
332 deferredparams
.rep
= NULL
;
335 ClientSocketContext
*
336 ClientSocketContextNew(const Comm::ConnectionPointer
&client
, ClientHttpRequest
* http
)
338 ClientSocketContext
*newContext
;
339 assert(http
!= NULL
);
340 newContext
= new ClientSocketContext
;
341 newContext
->http
= http
;
342 newContext
->clientConnection
= client
;
347 ClientSocketContext::writeControlMsg(HttpControlMsg
&msg
)
349 HttpReply
*rep
= msg
.reply
;
352 // apply selected clientReplyContext::buildReplyHeader() mods
353 // it is not clear what headers are required for control messages
354 rep
->header
.removeHopByHopEntries();
355 rep
->header
.putStr(HDR_CONNECTION
, "keep-alive");
356 httpHdrMangleList(&rep
->header
, http
->request
, ROR_REPLY
);
358 // remember the callback
359 cbControlMsgSent
= msg
.cbSuccess
;
361 MemBuf
*mb
= rep
->pack();
363 AsyncCall::Pointer call
= commCbCall(33, 5, "ClientSocketContext::wroteControlMsg",
364 CommIoCbPtrFun(&WroteControlMsg
, this));
365 comm_write_mbuf(clientConn(), mb
, call
);
370 /// called when we wrote the 1xx response
372 ClientSocketContext::wroteControlMsg(const Comm::ConnectionPointer
&conn
, char *, size_t, comm_err_t errflag
, int xerrno
)
374 if (errflag
== COMM_ERR_CLOSING
)
377 if (errflag
== COMM_OK
) {
378 ScheduleCallHere(cbControlMsgSent
);
382 debugs(33, 3, HERE
<< "1xx writing failed: " << xstrerr(xerrno
));
383 // no error notification: see HttpControlMsg.h for rationale and
384 // note that some errors are detected elsewhere (e.g., close handler)
386 // close on 1xx errors to be conservative and to simplify the code
387 // (if we do not close, we must notify the source of a failure!)
391 /// wroteControlMsg() wrapper: ClientSocketContext is not an AsyncJob
393 ClientSocketContext::WroteControlMsg(const Comm::ConnectionPointer
&conn
, char *bufnotused
, size_t size
, comm_err_t errflag
, int xerrno
, void *data
)
395 ClientSocketContext
*context
= static_cast<ClientSocketContext
*>(data
);
396 context
->wroteControlMsg(conn
, bufnotused
, size
, errflag
, xerrno
);
401 clientIdentDone(const char *ident
, void *data
)
403 ConnStateData
*conn
= (ConnStateData
*)data
;
404 xstrncpy(conn
->rfc931
, ident
? ident
: dash_str
, USER_IDENT_SZ
);
409 clientUpdateStatCounters(log_type logType
)
411 statCounter
.client_http
.requests
++;
413 if (logTypeIsATcpHit(logType
))
414 statCounter
.client_http
.hits
++;
416 if (logType
== LOG_TCP_HIT
)
417 statCounter
.client_http
.disk_hits
++;
418 else if (logType
== LOG_TCP_MEM_HIT
)
419 statCounter
.client_http
.mem_hits
++;
423 clientUpdateStatHistCounters(log_type logType
, int svc_time
)
425 statHistCount(&statCounter
.client_http
.all_svc_time
, svc_time
);
427 * The idea here is not to be complete, but to get service times
428 * for only well-defined types. For example, we don't include
429 * LOG_TCP_REFRESH_FAIL because its not really a cache hit
430 * (we *tried* to validate it, but failed).
435 case LOG_TCP_REFRESH_UNMODIFIED
:
436 statHistCount(&statCounter
.client_http
.nh_svc_time
, svc_time
);
439 case LOG_TCP_IMS_HIT
:
440 statHistCount(&statCounter
.client_http
.nm_svc_time
, svc_time
);
445 case LOG_TCP_MEM_HIT
:
447 case LOG_TCP_OFFLINE_HIT
:
448 statHistCount(&statCounter
.client_http
.hit_svc_time
, svc_time
);
453 case LOG_TCP_CLIENT_REFRESH_MISS
:
454 statHistCount(&statCounter
.client_http
.miss_svc_time
, svc_time
);
458 /* make compiler warnings go away */
464 clientPingHasFinished(ping_data
const *aPing
)
466 if (0 != aPing
->stop
.tv_sec
&& 0 != aPing
->start
.tv_sec
)
473 clientUpdateHierCounters(HierarchyLogEntry
* someEntry
)
477 switch (someEntry
->code
) {
478 #if USE_CACHE_DIGESTS
483 statCounter
.cd
.times_used
++;
491 case FIRST_PARENT_MISS
:
493 case CLOSEST_PARENT_MISS
:
494 statCounter
.icp
.times_used
++;
495 i
= &someEntry
->ping
;
497 if (clientPingHasFinished(i
))
498 statHistCount(&statCounter
.icp
.query_svc_time
,
499 tvSubUsec(i
->start
, i
->stop
));
502 statCounter
.icp
.query_timeouts
++;
509 statCounter
.netdb
.times_used
++;
519 ClientHttpRequest::updateCounters()
521 clientUpdateStatCounters(logType
);
523 if (request
->errType
!= ERR_NONE
)
524 statCounter
.client_http
.errors
++;
526 clientUpdateStatHistCounters(logType
,
527 tvSubMsec(start_time
, current_time
));
529 clientUpdateHierCounters(&request
->hier
);
533 prepareLogWithRequestDetails(HttpRequest
* request
, AccessLogEntry
* aLogEntry
)
539 Adaptation::Icap::History::Pointer ih
= request
->icapHistory();
541 if (Config
.onoff
.log_mime_hdrs
) {
545 packerToMemInit(&p
, &mb
);
546 request
->header
.packInto(&p
);
547 //This is the request after adaptation or redirection
548 aLogEntry
->headers
.adapted_request
= xstrdup(mb
.buf
);
550 // the virgin request is saved to aLogEntry->request
551 if (aLogEntry
->request
) {
554 packerToMemInit(&p
, &mb
);
555 aLogEntry
->request
->header
.packInto(&p
);
556 aLogEntry
->headers
.request
= xstrdup(mb
.buf
);
562 packerToMemInit(&p
, &mb
);
565 ih
->lastIcapHeader
.packInto(&p
);
566 aLogEntry
->headers
.icap
= xstrdup(mb
.buf
);
575 aLogEntry
->icap
.processingTime
= ih
->processingTime();
578 aLogEntry
->http
.method
= request
->method
;
579 aLogEntry
->http
.version
= request
->http_ver
;
580 aLogEntry
->hier
= request
->hier
;
581 if (request
->content_length
> 0) // negative when no body or unknown length
582 aLogEntry
->cache
.requestSize
+= request
->content_length
;
583 aLogEntry
->cache
.extuser
= request
->extacl_user
.termedBuf();
585 if (request
->auth_user_request
!= NULL
) {
587 if (request
->auth_user_request
->username())
588 aLogEntry
->cache
.authuser
= xstrdup(request
->auth_user_request
->username());
590 // WTF?? request->auth_user_request = NULL;
593 if (aLogEntry
->request
) {
594 aLogEntry
->request
->errType
= request
->errType
;
595 aLogEntry
->request
->errDetail
= request
->errDetail
;
600 ClientHttpRequest::logRequest()
602 if (out
.size
|| logType
) {
603 al
.icp
.opcode
= ICP_INVALID
;
605 debugs(33, 9, "clientLogRequest: al.url='" << al
.url
<< "'");
608 al
.http
.code
= al
.reply
->sline
.status
;
609 al
.http
.content_type
= al
.reply
->content_type
.termedBuf();
610 } else if (loggingEntry() && loggingEntry()->mem_obj
) {
611 al
.http
.code
= loggingEntry()->mem_obj
->getReply()->sline
.status
;
612 al
.http
.content_type
= loggingEntry()->mem_obj
->getReply()->content_type
.termedBuf();
615 debugs(33, 9, "clientLogRequest: http.code='" << al
.http
.code
<< "'");
617 if (loggingEntry() && loggingEntry()->mem_obj
)
618 al
.cache
.objectSize
= loggingEntry()->contentLen();
620 al
.cache
.caddr
.SetNoAddr();
622 if (getConn() != NULL
) al
.cache
.caddr
= getConn()->log_addr
;
624 al
.cache
.requestSize
= req_sz
;
625 al
.cache
.requestHeadersSize
= req_sz
;
627 al
.cache
.replySize
= out
.size
;
628 al
.cache
.replyHeadersSize
= out
.headers_sz
;
630 al
.cache
.highOffset
= out
.offset
;
632 al
.cache
.code
= logType
;
634 al
.cache
.msec
= tvSubMsec(start_time
, current_time
);
637 prepareLogWithRequestDetails(request
, &al
);
639 if (getConn() != NULL
&& getConn()->rfc931
[0])
640 al
.cache
.rfc931
= getConn()->rfc931
;
644 /* This is broken. Fails if the connection has been closed. Needs
645 * to snarf the ssl details some place earlier..
647 if (getConn() != NULL
)
648 al
.cache
.ssluser
= sslGetUserEmail(fd_table
[getConn()->fd
].ssl
);
652 ACLFilledChecklist
*checklist
= clientAclChecklistCreate(Config
.accessList
.log
, this);
655 checklist
->reply
= HTTPMSGLOCK(al
.reply
);
657 if (!Config
.accessList
.log
|| checklist
->fastCheck()) {
659 al
.adapted_request
= HTTPMSGLOCK(request
);
660 accessLogLog(&al
, checklist
);
663 if (getConn() != NULL
&& getConn()->clientConn
!= NULL
)
664 clientdbUpdate(getConn()->clientConn
->remote
, logType
, PROTO_HTTP
, out
.size
);
670 accessLogFreeMemory(&al
);
674 ClientHttpRequest::freeResources()
678 safe_free(redirect
.location
);
679 range_iter
.boundary
.clean();
680 HTTPMSGUNLOCK(request
);
682 if (client_stream
.tail
)
683 clientStreamAbort((clientStreamNode
*)client_stream
.tail
->data
, this);
687 httpRequestFree(void *data
)
689 ClientHttpRequest
*http
= (ClientHttpRequest
*)data
;
690 assert(http
!= NULL
);
695 ConnStateData::areAllContextsForThisConnection() const
697 assert(this != NULL
);
698 ClientSocketContext::Pointer context
= getCurrentContext();
700 while (context
.getRaw()) {
701 if (context
->http
->getConn() != this)
704 context
= context
->next
;
711 ConnStateData::freeAllContexts()
713 ClientSocketContext::Pointer context
;
715 while ((context
= getCurrentContext()).getRaw() != NULL
) {
716 assert(getCurrentContext() !=
717 getCurrentContext()->next
);
718 context
->connIsFinished();
719 assert (context
!= currentobject
);
723 /// propagates abort event to all contexts
725 ConnStateData::notifyAllContexts(int xerrno
)
727 typedef ClientSocketContext::Pointer CSCP
;
728 for (CSCP c
= getCurrentContext(); c
.getRaw(); c
= c
->next
)
729 c
->noteIoError(xerrno
);
732 /* This is a handler normally called by comm_close() */
733 void ConnStateData::connStateClosed(const CommCloseCbParams
&io
)
735 deleteThis("ConnStateData::connStateClosed");
738 // cleans up before destructor is called
740 ConnStateData::swanSong()
742 debugs(33, 2, HERE
<< clientConn
);
744 flags
.readMoreRequests
= false;
745 clientdbEstablished(clientConn
->remote
, -1); /* decrement */
746 assert(areAllContextsForThisConnection());
749 if (auth_user_request
!= NULL
) {
750 debugs(33, 4, "ConnStateData::swanSong: freeing auth_user_request '" << auth_user_request
<< "' (this is '" << this << "')");
751 auth_user_request
->onConnectionClose(this);
755 comm_close(pinning
.fd
);
757 BodyProducer::swanSong();
758 flags
.swanSang
= true;
762 ConnStateData::isOpen() const
764 return cbdataReferenceValid(this) && // XXX: checking "this" in a method
765 Comm::IsConnOpen(clientConn
) &&
766 !fd_table
[clientConn
->fd
].closing();
769 ConnStateData::~ConnStateData()
771 assert(this != NULL
);
772 debugs(33, 3, HERE
<< clientConn
);
775 debugs(33, 1, "BUG: ConnStateData did not close " << clientConn
);
778 debugs(33, 1, "BUG: ConnStateData was not destroyed properly; " << clientConn
);
780 cbdataReferenceDone(port
);
782 if (bodyPipe
!= NULL
)
783 stopProducingFor(bodyPipe
, false);
787 * clientSetKeepaliveFlag() sets request->flags.proxy_keepalive.
788 * This is the client-side persistent connection flag. We need
789 * to set this relatively early in the request processing
790 * to handle hacks for broken servers and clients.
793 clientSetKeepaliveFlag(ClientHttpRequest
* http
)
795 HttpRequest
*request
= http
->request
;
797 debugs(33, 3, "clientSetKeepaliveFlag: http_ver = " <<
798 request
->http_ver
.major
<< "." << request
->http_ver
.minor
);
799 debugs(33, 3, "clientSetKeepaliveFlag: method = " <<
800 RequestMethodStr(request
->method
));
802 // TODO: move to HttpRequest::hdrCacheInit, just like HttpReply.
803 request
->flags
.proxy_keepalive
= request
->persistent() ? 1 : 0;
807 clientIsContentLengthValid(HttpRequest
* r
)
809 switch (r
->method
.id()) {
814 /* PUT/POST requires a request entity */
815 return (r
->content_length
>= 0);
820 /* We do not want to see a request entity on GET/HEAD requests */
821 return (r
->content_length
<= 0 || Config
.onoff
.request_entities
);
824 /* For other types of requests we don't care */
832 clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength
)
834 if (Config
.maxRequestBodySize
&&
835 bodyLength
> Config
.maxRequestBodySize
)
836 return 1; /* too large */
843 connIsUsable(ConnStateData
* conn
)
845 if (conn
== NULL
|| !cbdataReferenceValid(conn
) || !Comm::IsConnOpen(conn
->clientConn
))
853 // careful: the "current" context may be gone if we wrote an early response
854 ClientSocketContext::Pointer
855 ConnStateData::getCurrentContext() const
858 return currentobject
;
862 ClientSocketContext::deferRecipientForLater(clientStreamNode
* node
, HttpReply
* rep
, StoreIOBuffer receivedData
)
864 debugs(33, 2, "clientSocketRecipient: Deferring request " << http
->uri
);
865 assert(flags
.deferred
== 0);
867 deferredparams
.node
= node
;
868 deferredparams
.rep
= rep
;
869 deferredparams
.queuedBuffer
= receivedData
;
874 responseFinishedOrFailed(HttpReply
* rep
, StoreIOBuffer
const & receivedData
)
876 if (rep
== NULL
&& receivedData
.data
== NULL
&& receivedData
.length
== 0)
883 ClientSocketContext::startOfOutput() const
885 return http
->out
.size
== 0;
889 ClientSocketContext::lengthToSend(Range
<int64_t> const &available
)
891 /*the size of available range can always fit in a size_t type*/
892 size_t maximum
= (size_t)available
.size();
894 if (!http
->request
->range
)
897 assert (canPackMoreRanges());
899 if (http
->range_iter
.debt() == -1)
902 assert (http
->range_iter
.debt() > 0);
904 /* TODO this + the last line could be a range intersection calculation */
905 if (available
.start
< http
->range_iter
.currentSpec()->offset
)
908 return min(http
->range_iter
.debt(), (int64_t)maximum
);
912 ClientSocketContext::noteSentBodyBytes(size_t bytes
)
914 http
->out
.offset
+= bytes
;
916 if (!http
->request
->range
)
919 if (http
->range_iter
.debt() != -1) {
920 http
->range_iter
.debt(http
->range_iter
.debt() - bytes
);
921 assert (http
->range_iter
.debt() >= 0);
924 /* debt() always stops at -1, below that is a bug */
925 assert (http
->range_iter
.debt() >= -1);
929 ClientHttpRequest::multipartRangeRequest() const
931 return request
->multipartRangeRequest();
935 ClientSocketContext::multipartRangeRequest() const
937 return http
->multipartRangeRequest();
941 ClientSocketContext::sendBody(HttpReply
* rep
, StoreIOBuffer bodyData
)
945 if (!multipartRangeRequest() && !http
->request
->flags
.chunked_reply
) {
946 size_t length
= lengthToSend(bodyData
.range());
947 noteSentBodyBytes (length
);
948 AsyncCall::Pointer call
= commCbCall(33, 5, "clientWriteBodyComplete",
949 CommIoCbPtrFun(clientWriteBodyComplete
, this));
950 comm_write(clientConn(), bodyData
.data
, length
, call
);
956 if (multipartRangeRequest())
957 packRange(bodyData
, &mb
);
959 packChunk(bodyData
, mb
);
961 if (mb
.contentSize()) {
963 AsyncCall::Pointer call
= commCbCall(33, 5, "clientWriteComplete",
964 CommIoCbPtrFun(clientWriteComplete
, this));
965 comm_write_mbuf(clientConn(), &mb
, call
);
967 writeComplete(clientConn(), NULL
, 0, COMM_OK
);
971 * Packs bodyData into mb using chunked encoding. Packs the last-chunk
972 * if bodyData is empty.
975 ClientSocketContext::packChunk(const StoreIOBuffer
&bodyData
, MemBuf
&mb
)
977 const uint64_t length
=
978 static_cast<uint64_t>(lengthToSend(bodyData
.range()));
979 noteSentBodyBytes(length
);
981 mb
.Printf("%"PRIX64
"\r\n", length
);
982 mb
.append(bodyData
.data
, length
);
986 /** put terminating boundary for multiparts */
988 clientPackTermBound(String boundary
, MemBuf
* mb
)
990 mb
->Printf("\r\n--" SQUIDSTRINGPH
"--\r\n", SQUIDSTRINGPRINT(boundary
));
991 debugs(33, 6, "clientPackTermBound: buf offset: " << mb
->size
);
994 /** appends a "part" HTTP header (as in a multi-part/range reply) to the buffer */
996 clientPackRangeHdr(const HttpReply
* rep
, const HttpHdrRangeSpec
* spec
, String boundary
, MemBuf
* mb
)
998 HttpHeader
hdr(hoReply
);
1004 debugs(33, 5, "clientPackRangeHdr: appending boundary: " << boundary
);
1005 /* rfc2046 requires to _prepend_ boundary with <crlf>! */
1006 mb
->Printf("\r\n--" SQUIDSTRINGPH
"\r\n", SQUIDSTRINGPRINT(boundary
));
1008 /* stuff the header with required entries and pack it */
1010 if (rep
->header
.has(HDR_CONTENT_TYPE
))
1011 hdr
.putStr(HDR_CONTENT_TYPE
, rep
->header
.getStr(HDR_CONTENT_TYPE
));
1013 httpHeaderAddContRange(&hdr
, *spec
, rep
->content_length
);
1015 packerToMemInit(&p
, mb
);
1023 /* append <crlf> (we packed a header, not a reply) */
1028 * extracts a "range" from *buf and appends them to mb, updating
1029 * all offsets and such.
1032 ClientSocketContext::packRange(StoreIOBuffer
const &source
, MemBuf
* mb
)
1034 HttpHdrRangeIter
* i
= &http
->range_iter
;
1035 Range
<int64_t> available (source
.range());
1036 char const *buf
= source
.data
;
1038 while (i
->currentSpec() && available
.size()) {
1039 const size_t copy_sz
= lengthToSend(available
);
1043 * intersection of "have" and "need" ranges must not be empty
1045 assert(http
->out
.offset
< i
->currentSpec()->offset
+ i
->currentSpec()->length
);
1046 assert(http
->out
.offset
+ available
.size() > i
->currentSpec()->offset
);
1049 * put boundary and headers at the beginning of a range in a
1053 if (http
->multipartRangeRequest() && i
->debt() == i
->currentSpec()->length
) {
1054 assert(http
->memObject());
1056 http
->memObject()->getReply(), /* original reply */
1057 i
->currentSpec(), /* current range */
1058 i
->boundary
, /* boundary, the same for all */
1065 debugs(33, 3, "clientPackRange: appending " << copy_sz
<< " bytes");
1067 noteSentBodyBytes (copy_sz
);
1069 mb
->append(buf
, copy_sz
);
1074 available
.start
+= copy_sz
;
1083 assert((available
.size() >= 0 && i
->debt() >= 0) || i
->debt() == -1);
1085 if (!canPackMoreRanges()) {
1086 debugs(33, 3, "clientPackRange: Returning because !canPackMoreRanges.");
1089 /* put terminating boundary for multiparts */
1090 clientPackTermBound(i
->boundary
, mb
);
1095 int64_t nextOffset
= getNextRangeOffset();
1097 assert (nextOffset
>= http
->out
.offset
);
1099 int64_t skip
= nextOffset
- http
->out
.offset
;
1101 /* adjust for not to be transmitted bytes */
1102 http
->out
.offset
= nextOffset
;
1104 if (available
.size() <= skip
)
1107 available
.start
+= skip
;
1116 /** returns expected content length for multi-range replies
1117 * note: assumes that httpHdrRangeCanonize has already been called
1118 * warning: assumes that HTTP headers for individual ranges at the
1119 * time of the actuall assembly will be exactly the same as
1120 * the headers when clientMRangeCLen() is called */
1122 ClientHttpRequest::mRangeCLen()
1127 assert(memObject());
1130 HttpHdrRange::iterator pos
= request
->range
->begin();
1132 while (pos
!= request
->range
->end()) {
1133 /* account for headers for this range */
1135 clientPackRangeHdr(memObject()->getReply(),
1136 *pos
, range_iter
.boundary
, &mb
);
1139 /* account for range content */
1140 clen
+= (*pos
)->length
;
1142 debugs(33, 6, "clientMRangeCLen: (clen += " << mb
.size
<< " + " << (*pos
)->length
<< ") == " << clen
);
1146 /* account for the terminating boundary */
1149 clientPackTermBound(range_iter
.boundary
, &mb
);
1159 * returns true if If-Range specs match reply, false otherwise
1162 clientIfRangeMatch(ClientHttpRequest
* http
, HttpReply
* rep
)
1164 const TimeOrTag spec
= http
->request
->header
.getTimeOrTag(HDR_IF_RANGE
);
1165 /* check for parsing falure */
1172 ETag rep_tag
= rep
->header
.getETag(HDR_ETAG
);
1173 debugs(33, 3, "clientIfRangeMatch: ETags: " << spec
.tag
.str
<< " and " <<
1174 (rep_tag
.str
? rep_tag
.str
: "<none>"));
1177 return 0; /* entity has no etag to compare with! */
1179 if (spec
.tag
.weak
|| rep_tag
.weak
) {
1180 debugs(33, 1, "clientIfRangeMatch: Weak ETags are not allowed in If-Range: " << spec
.tag
.str
<< " ? " << rep_tag
.str
);
1181 return 0; /* must use strong validator for sub-range requests */
1184 return etagIsStrongEqual(rep_tag
, spec
.tag
);
1187 /* got modification time? */
1188 if (spec
.time
>= 0) {
1189 return http
->storeEntry()->lastmod
<= spec
.time
;
1192 assert(0); /* should not happen */
1197 * generates a "unique" boundary string for multipart responses
1198 * the caller is responsible for cleaning the string */
1200 ClientHttpRequest::rangeBoundaryStr() const
1204 String
b(APP_FULLNAME
);
1206 key
= storeEntry()->getMD5Text();
1207 b
.append(key
, strlen(key
));
1211 /** adds appropriate Range headers if needed */
1213 ClientSocketContext::buildRangeHeader(HttpReply
* rep
)
1215 HttpHeader
*hdr
= rep
? &rep
->header
: 0;
1216 const char *range_err
= NULL
;
1217 HttpRequest
*request
= http
->request
;
1218 assert(request
->range
);
1219 /* check if we still want to do ranges */
1221 int64_t roffLimit
= request
->getRangeOffsetLimit();
1224 range_err
= "no [parse-able] reply";
1225 else if ((rep
->sline
.status
!= HTTP_OK
) && (rep
->sline
.status
!= HTTP_PARTIAL_CONTENT
))
1226 range_err
= "wrong status code";
1227 else if (hdr
->has(HDR_CONTENT_RANGE
))
1228 range_err
= "origin server does ranges";
1229 else if (rep
->content_length
< 0)
1230 range_err
= "unknown length";
1231 else if (rep
->content_length
!= http
->memObject()->getReply()->content_length
)
1232 range_err
= "INCONSISTENT length"; /* a bug? */
1234 /* hits only - upstream peer determines correct behaviour on misses, and client_side_reply determines
1237 else if (logTypeIsATcpHit(http
->logType
) && http
->request
->header
.has(HDR_IF_RANGE
) && !clientIfRangeMatch(http
, rep
))
1238 range_err
= "If-Range match failed";
1239 else if (!http
->request
->range
->canonize(rep
))
1240 range_err
= "canonization failed";
1241 else if (http
->request
->range
->isComplex())
1242 range_err
= "too complex range header";
1243 else if (!logTypeIsATcpHit(http
->logType
) && http
->request
->range
->offsetLimitExceeded(roffLimit
))
1244 range_err
= "range outside range_offset_limit";
1246 /* get rid of our range specs on error */
1248 /* XXX We do this here because we need canonisation etc. However, this current
1249 * code will lead to incorrect store offset requests - the store will have the
1250 * offset data, but we won't be requesting it.
1251 * So, we can either re-request, or generate an error
1253 debugs(33, 3, "clientBuildRangeHeader: will not do ranges: " << range_err
<< ".");
1254 delete http
->request
->range
;
1255 http
->request
->range
= NULL
;
1257 /* XXX: TODO: Review, this unconditional set may be wrong. - TODO: review. */
1258 httpStatusLineSet(&rep
->sline
, rep
->sline
.version
,
1259 HTTP_PARTIAL_CONTENT
, NULL
);
1260 // web server responded with a valid, but unexpected range.
1261 // will (try-to) forward as-is.
1262 //TODO: we should cope with multirange request/responses
1263 bool replyMatchRequest
= rep
->content_range
!= NULL
?
1264 request
->range
->contains(rep
->content_range
->spec
) :
1266 const int spec_count
= http
->request
->range
->specs
.count
;
1267 int64_t actual_clen
= -1;
1269 debugs(33, 3, "clientBuildRangeHeader: range spec count: " <<
1270 spec_count
<< " virgin clen: " << rep
->content_length
);
1271 assert(spec_count
> 0);
1272 /* append appropriate header(s) */
1274 if (spec_count
== 1) {
1275 if (!replyMatchRequest
) {
1276 hdr
->delById(HDR_CONTENT_RANGE
);
1277 hdr
->putContRange(rep
->content_range
);
1278 actual_clen
= rep
->content_length
;
1279 //http->range_iter.pos = rep->content_range->spec.begin();
1280 (*http
->range_iter
.pos
)->offset
= rep
->content_range
->spec
.offset
;
1281 (*http
->range_iter
.pos
)->length
= rep
->content_range
->spec
.length
;
1284 HttpHdrRange::iterator pos
= http
->request
->range
->begin();
1286 /* append Content-Range */
1288 if (!hdr
->has(HDR_CONTENT_RANGE
)) {
1289 /* No content range, so this was a full object we are
1292 httpHeaderAddContRange(hdr
, **pos
, rep
->content_length
);
1295 /* set new Content-Length to the actual number of bytes
1296 * transmitted in the message-body */
1297 actual_clen
= (*pos
)->length
;
1301 /* generate boundary string */
1302 http
->range_iter
.boundary
= http
->rangeBoundaryStr();
1303 /* delete old Content-Type, add ours */
1304 hdr
->delById(HDR_CONTENT_TYPE
);
1305 httpHeaderPutStrf(hdr
, HDR_CONTENT_TYPE
,
1306 "multipart/byteranges; boundary=\"" SQUIDSTRINGPH
"\"",
1307 SQUIDSTRINGPRINT(http
->range_iter
.boundary
));
1308 /* Content-Length is not required in multipart responses
1309 * but it is always nice to have one */
1310 actual_clen
= http
->mRangeCLen();
1311 /* http->out needs to start where we want data at */
1312 http
->out
.offset
= http
->range_iter
.currentSpec()->offset
;
1315 /* replace Content-Length header */
1316 assert(actual_clen
>= 0);
1318 hdr
->delById(HDR_CONTENT_LENGTH
);
1320 hdr
->putInt64(HDR_CONTENT_LENGTH
, actual_clen
);
1322 debugs(33, 3, "clientBuildRangeHeader: actual content length: " << actual_clen
);
1324 /* And start the range iter off */
1325 http
->range_iter
.updateSpec();
1330 ClientSocketContext::prepareReply(HttpReply
* rep
)
1334 if (http
->request
->range
)
1335 buildRangeHeader(rep
);
1339 ClientSocketContext::sendStartOfMessage(HttpReply
* rep
, StoreIOBuffer bodyData
)
1343 MemBuf
*mb
= rep
->pack();
1344 /* Save length of headers for persistent conn checks */
1345 http
->out
.headers_sz
= mb
->contentSize();
1348 headersLog(0, 0, http
->request
->method
, rep
);
1351 if (bodyData
.data
&& bodyData
.length
) {
1352 if (multipartRangeRequest())
1353 packRange(bodyData
, mb
);
1354 else if (http
->request
->flags
.chunked_reply
) {
1355 packChunk(bodyData
, *mb
);
1357 size_t length
= lengthToSend(bodyData
.range());
1358 noteSentBodyBytes (length
);
1360 mb
->append(bodyData
.data
, length
);
1365 debugs(33,7, HERE
<< "sendStartOfMessage schedules clientWriteComplete");
1366 AsyncCall::Pointer call
= commCbCall(33, 5, "clientWriteComplete",
1367 CommIoCbPtrFun(clientWriteComplete
, this));
1368 comm_write_mbuf(clientConn(), mb
, call
);
1374 * Write a chunk of data to a client socket. If the reply is present,
1375 * send the reply headers down the wire too, and clean them up when
1378 * The request is one backed by a connection, not an internal request.
1379 * data context is not NULL
1380 * There are no more entries in the stream chain.
1383 clientSocketRecipient(clientStreamNode
* node
, ClientHttpRequest
* http
,
1384 HttpReply
* rep
, StoreIOBuffer receivedData
)
1386 /* Test preconditions */
1387 assert(node
!= NULL
);
1388 PROF_start(clientSocketRecipient
);
1389 /* TODO: handle this rather than asserting
1390 * - it should only ever happen if we cause an abort and
1391 * the callback chain loops back to here, so we can simply return.
1392 * However, that itself shouldn't happen, so it stays as an assert for now.
1394 assert(cbdataReferenceValid(node
));
1395 assert(node
->node
.next
== NULL
);
1396 ClientSocketContext::Pointer context
= dynamic_cast<ClientSocketContext
*>(node
->data
.getRaw());
1397 assert(context
!= NULL
);
1398 assert(connIsUsable(http
->getConn()));
1400 /* TODO: check offset is what we asked for */
1402 if (context
!= http
->getConn()->getCurrentContext()) {
1403 context
->deferRecipientForLater(node
, rep
, receivedData
);
1404 PROF_stop(clientSocketRecipient
);
1408 // After sending Transfer-Encoding: chunked (at least), always send
1409 // the last-chunk if there was no error, ignoring responseFinishedOrFailed.
1410 const bool mustSendLastChunk
= http
->request
->flags
.chunked_reply
&&
1411 !http
->request
->flags
.stream_error
&& !context
->startOfOutput();
1412 if (responseFinishedOrFailed(rep
, receivedData
) && !mustSendLastChunk
) {
1413 context
->writeComplete(http
->getConn()->clientConn
, NULL
, 0, COMM_OK
);
1414 PROF_stop(clientSocketRecipient
);
1418 if (!context
->startOfOutput())
1419 context
->sendBody(rep
, receivedData
);
1422 http
->al
.reply
= HTTPMSGLOCK(rep
);
1423 context
->sendStartOfMessage(rep
, receivedData
);
1426 PROF_stop(clientSocketRecipient
);
1430 * Called when a downstream node is no longer interested in
1431 * our data. As we are a terminal node, this means on aborts
1435 clientSocketDetach(clientStreamNode
* node
, ClientHttpRequest
* http
)
1437 /* Test preconditions */
1438 assert(node
!= NULL
);
1439 /* TODO: handle this rather than asserting
1440 * - it should only ever happen if we cause an abort and
1441 * the callback chain loops back to here, so we can simply return.
1442 * However, that itself shouldn't happen, so it stays as an assert for now.
1444 assert(cbdataReferenceValid(node
));
1445 /* Set null by ContextFree */
1446 assert(node
->node
.next
== NULL
);
1447 /* this is the assert discussed above */
1448 assert(NULL
== dynamic_cast<ClientSocketContext
*>(node
->data
.getRaw()));
1449 /* We are only called when the client socket shutsdown.
1450 * Tell the prev pipeline member we're finished
1452 clientStreamDetach(node
, http
);
1456 clientWriteBodyComplete(const Comm::ConnectionPointer
&conn
, char *buf
, size_t size
, comm_err_t errflag
, int xerrno
, void *data
)
1458 debugs(33,7, HERE
<< "clientWriteBodyComplete schedules clientWriteComplete");
1459 clientWriteComplete(conn
, NULL
, size
, errflag
, xerrno
, data
);
1463 ConnStateData::readNextRequest()
1465 debugs(33, 5, HERE
<< clientConn
<< " reading next req");
1467 fd_note(clientConn
->fd
, "Waiting for next request");
1469 * Set the timeout BEFORE calling clientReadRequest().
1471 typedef CommCbMemFunT
<ConnStateData
, CommTimeoutCbParams
> TimeoutDialer
;
1472 AsyncCall::Pointer timeoutCall
= JobCallback(33, 5,
1473 TimeoutDialer
, this, ConnStateData::requestTimeout
);
1474 commSetTimeout(clientConn
->fd
, Config
.Timeout
.persistent_request
, timeoutCall
);
1477 /** Please don't do anything with the FD past here! */
1481 ClientSocketContextPushDeferredIfNeeded(ClientSocketContext::Pointer deferredRequest
, ConnStateData
* conn
)
1483 debugs(33, 2, HERE
<< conn
->clientConn
<< " Sending next");
1485 /** If the client stream is waiting on a socket write to occur, then */
1487 if (deferredRequest
->flags
.deferred
) {
1488 /** NO data is allowed to have been sent. */
1489 assert(deferredRequest
->http
->out
.size
== 0);
1491 clientSocketRecipient(deferredRequest
->deferredparams
.node
,
1492 deferredRequest
->http
,
1493 deferredRequest
->deferredparams
.rep
,
1494 deferredRequest
->deferredparams
.queuedBuffer
);
1497 /** otherwise, the request is still active in a callbacksomewhere,
1503 ClientSocketContext::keepaliveNextRequest()
1505 ConnStateData
* conn
= http
->getConn();
1506 bool do_next_read
= false;
1508 debugs(33, 3, HERE
<< conn
->clientConn
);
1511 if (conn
->pinning
.pinned
&& conn
->pinning
.fd
== -1) {
1512 debugs(33, 2, HERE
<< conn
->clientConn
<< " Connection was pinned but server side gone. Terminating client connection");
1513 conn
->clientConn
->close();
1518 * Attempt to parse a request from the request buffer.
1519 * If we've been fed a pipelined request it may already
1520 * be in our read buffer.
1523 * This needs to fall through - if we're unlucky and parse the _last_ request
1524 * from our read buffer we may never re-register for another client read.
1527 if (clientParseRequest(conn
, do_next_read
)) {
1528 debugs(33, 3, HERE
<< conn
->clientConn
<< ": parsed next request from buffer");
1532 * Either we need to kick-start another read or, if we have
1533 * a half-closed connection, kill it after the last request.
1534 * This saves waiting for half-closed connections to finished being
1535 * half-closed _AND_ then, sometimes, spending "Timeout" time in
1536 * the keepalive "Waiting for next request" state.
1538 if (commIsHalfClosed(conn
->clientConn
->fd
) && (conn
->getConcurrentRequestCount() == 0)) {
1539 debugs(33, 3, "ClientSocketContext::keepaliveNextRequest: half-closed client with no pending requests, closing");
1540 conn
->clientConn
->close();
1544 ClientSocketContext::Pointer deferredRequest
;
1547 * At this point we either have a parsed request (which we've
1548 * kicked off the processing for) or not. If we have a deferred
1549 * request (parsed but deferred for pipeling processing reasons)
1550 * then look at processing it. If not, simply kickstart
1554 if ((deferredRequest
= conn
->getCurrentContext()).getRaw()) {
1555 debugs(33, 3, HERE
<< conn
->clientConn
<< ": calling PushDeferredIfNeeded");
1556 ClientSocketContextPushDeferredIfNeeded(deferredRequest
, conn
);
1558 debugs(33, 3, HERE
<< conn
->clientConn
<< ": calling conn->readNextRequest()");
1559 conn
->readNextRequest();
1564 clientUpdateSocketStats(log_type logType
, size_t size
)
1569 kb_incr(&statCounter
.client_http
.kbytes_out
, size
);
1571 if (logTypeIsATcpHit(logType
))
1572 kb_incr(&statCounter
.client_http
.hit_kbytes_out
, size
);
1576 * increments iterator "i"
1577 * used by clientPackMoreRanges
1579 \retval true there is still data available to pack more ranges
1583 ClientSocketContext::canPackMoreRanges() const
1585 /** first update iterator "i" if needed */
1587 if (!http
->range_iter
.debt()) {
1588 debugs(33, 5, "ClientSocketContext::canPackMoreRanges: At end of current range spec for FD " << clientConn());
1590 if (http
->range_iter
.pos
.incrementable())
1591 ++http
->range_iter
.pos
;
1593 http
->range_iter
.updateSpec();
1596 assert(!http
->range_iter
.debt() == !http
->range_iter
.currentSpec());
1598 /* paranoid sync condition */
1599 /* continue condition: need_more_data */
1600 debugs(33, 5, "ClientSocketContext::canPackMoreRanges: returning " << (http
->range_iter
.currentSpec() ? true : false));
1601 return http
->range_iter
.currentSpec() ? true : false;
1605 ClientSocketContext::getNextRangeOffset() const
1607 if (http
->request
->range
) {
1608 /* offset in range specs does not count the prefix of an http msg */
1609 debugs (33, 5, "ClientSocketContext::getNextRangeOffset: http offset " << http
->out
.offset
);
1610 /* check: reply was parsed and range iterator was initialized */
1611 assert(http
->range_iter
.valid
);
1612 /* filter out data according to range specs */
1613 assert (canPackMoreRanges());
1615 int64_t start
; /* offset of still missing data */
1616 assert(http
->range_iter
.currentSpec());
1617 start
= http
->range_iter
.currentSpec()->offset
+ http
->range_iter
.currentSpec()->length
- http
->range_iter
.debt();
1618 debugs(33, 3, "clientPackMoreRanges: in: offset: " << http
->out
.offset
);
1619 debugs(33, 3, "clientPackMoreRanges: out:"
1620 " start: " << start
<<
1621 " spec[" << http
->range_iter
.pos
- http
->request
->range
->begin() << "]:" <<
1622 " [" << http
->range_iter
.currentSpec()->offset
<<
1623 ", " << http
->range_iter
.currentSpec()->offset
+ http
->range_iter
.currentSpec()->length
<< "),"
1624 " len: " << http
->range_iter
.currentSpec()->length
<<
1625 " debt: " << http
->range_iter
.debt());
1626 if (http
->range_iter
.currentSpec()->length
!= -1)
1627 assert(http
->out
.offset
<= start
); /* we did not miss it */
1632 } else if (reply
&& reply
->content_range
) {
1633 /* request does not have ranges, but reply does */
1634 /** \todo FIXME: should use range_iter_pos on reply, as soon as reply->content_range
1635 * becomes HttpHdrRange rather than HttpHdrRangeSpec.
1637 return http
->out
.offset
+ reply
->content_range
->spec
.offset
;
1640 return http
->out
.offset
;
1644 ClientSocketContext::pullData()
1646 debugs(33, 5, "ClientSocketContext::pullData: FD " << clientConn() <<
1647 " attempting to pull upstream data");
1649 /* More data will be coming from the stream. */
1650 StoreIOBuffer readBuffer
;
1651 /* XXX: Next requested byte in the range sequence */
1652 /* XXX: length = getmaximumrangelenfgth */
1653 readBuffer
.offset
= getNextRangeOffset();
1654 readBuffer
.length
= HTTP_REQBUF_SZ
;
1655 readBuffer
.data
= reqbuf
;
1656 /* we may note we have reached the end of the wanted ranges */
1657 clientStreamRead(getTail(), http
, readBuffer
);
1660 clientStream_status_t
1661 ClientSocketContext::socketState()
1663 switch (clientStreamStatus(getTail(), http
)) {
1666 /* check for range support ending */
1668 if (http
->request
->range
) {
1669 /* check: reply was parsed and range iterator was initialized */
1670 assert(http
->range_iter
.valid
);
1671 /* filter out data according to range specs */
1673 if (!canPackMoreRanges()) {
1674 debugs(33, 5, HERE
<< "Range request at end of returnable " <<
1675 "range sequence on " << clientConn());
1677 if (http
->request
->flags
.proxy_keepalive
)
1678 return STREAM_COMPLETE
;
1680 return STREAM_UNPLANNED_COMPLETE
;
1682 } else if (reply
&& reply
->content_range
) {
1683 /* reply has content-range, but Squid is not managing ranges */
1684 const int64_t &bytesSent
= http
->out
.offset
;
1685 const int64_t &bytesExpected
= reply
->content_range
->spec
.length
;
1687 debugs(33, 7, HERE
<< "body bytes sent vs. expected: " <<
1688 bytesSent
<< " ? " << bytesExpected
<< " (+" <<
1689 reply
->content_range
->spec
.offset
<< ")");
1691 // did we get at least what we expected, based on range specs?
1693 if (bytesSent
== bytesExpected
) { // got everything
1694 if (http
->request
->flags
.proxy_keepalive
)
1695 return STREAM_COMPLETE
;
1697 return STREAM_UNPLANNED_COMPLETE
;
1700 // The logic below is not clear: If we got more than we
1701 // expected why would persistency matter? Should not this
1702 // always be an error?
1703 if (bytesSent
> bytesExpected
) { // got extra
1704 if (http
->request
->flags
.proxy_keepalive
)
1705 return STREAM_COMPLETE
;
1707 return STREAM_UNPLANNED_COMPLETE
;
1710 // did not get enough yet, expecting more
1715 case STREAM_COMPLETE
:
1716 return STREAM_COMPLETE
;
1718 case STREAM_UNPLANNED_COMPLETE
:
1719 return STREAM_UNPLANNED_COMPLETE
;
1722 return STREAM_FAILED
;
1725 fatal ("unreachable code\n");
1730 * A write has just completed to the client, or we have just realised there is
1731 * no more data to send.
1734 clientWriteComplete(const Comm::ConnectionPointer
&conn
, char *bufnotused
, size_t size
, comm_err_t errflag
, int xerrno
, void *data
)
1736 ClientSocketContext
*context
= (ClientSocketContext
*)data
;
1737 context
->writeComplete(conn
, bufnotused
, size
, errflag
);
1740 /// remembers the abnormal connection termination for logging purposes
1742 ClientSocketContext::noteIoError(const int xerrno
)
1745 if (xerrno
== ETIMEDOUT
)
1746 http
->al
.http
.timedout
= true;
1747 else // even if xerrno is zero (which means read abort/eof)
1748 http
->al
.http
.aborted
= true;
1754 ClientSocketContext::doClose()
1756 http
->getConn()->clientConn
->close();
1759 /** Called to initiate (and possibly complete) closing of the context.
1760 * The underlying socket may be already closed */
1762 ClientSocketContext::initiateClose(const char *reason
)
1764 debugs(33, 5, HERE
<< "initiateClose: closing for " << reason
);
1767 ConnStateData
* conn
= http
->getConn();
1770 if (const int64_t expecting
= conn
->mayNeedToReadMoreBody()) {
1771 debugs(33, 5, HERE
<< "ClientSocketContext::initiateClose: " <<
1772 "closing, but first " << conn
<< " needs to read " <<
1773 expecting
<< " request body bytes with " <<
1774 conn
->in
.notYetUsed
<< " notYetUsed");
1776 if (conn
->closing()) {
1777 debugs(33, 2, HERE
<< "avoiding double-closing " << conn
);
1782 * XXX We assume the reply fits in the TCP transmit
1783 * window. If not the connection may stall while sending
1784 * the reply (before reaching here) if the client does not
1785 * try to read the response while sending the request body.
1786 * As of yet we have not received any complaints indicating
1787 * this may be an issue.
1789 conn
->startClosing(reason
);
1800 ClientSocketContext::writeComplete(const Comm::ConnectionPointer
&conn
, char *bufnotused
, size_t size
, comm_err_t errflag
)
1802 StoreEntry
*entry
= http
->storeEntry();
1803 http
->out
.size
+= size
;
1804 assert(Comm::IsConnOpen(conn
));
1805 debugs(33, 5, HERE
<< conn
<< ", sz " << size
<<
1806 ", err " << errflag
<< ", off " << http
->out
.size
<< ", len " <<
1807 entry
? entry
->objectLen() : 0);
1808 clientUpdateSocketStats(http
->logType
, size
);
1810 /* Bail out quickly on COMM_ERR_CLOSING - close handlers will tidy up */
1812 if (errflag
== COMM_ERR_CLOSING
)
1815 assert(Comm::IsConnOpen(clientConn()) && clientConn()->fd
== conn
->fd
);
1817 if (errflag
|| clientHttpRequestStatus(conn
->fd
, http
)) {
1818 initiateClose("failure or true request status");
1819 /* Do we leak here ? */
1823 switch (socketState()) {
1829 case STREAM_COMPLETE
:
1830 debugs(33, 5, HERE
<< conn
<< " Keeping Alive");
1831 keepaliveNextRequest();
1834 case STREAM_UNPLANNED_COMPLETE
:
1835 initiateClose("STREAM_UNPLANNED_COMPLETE");
1839 initiateClose("STREAM_FAILED");
1843 fatal("Hit unreachable code in clientWriteComplete\n");
1847 extern "C" CSR clientGetMoreData
;
1848 extern "C" CSS clientReplyStatus
;
1849 extern "C" CSD clientReplyDetach
;
1851 static ClientSocketContext
*
1852 parseHttpRequestAbort(ConnStateData
* csd
, const char *uri
)
1854 ClientHttpRequest
*http
;
1855 ClientSocketContext
*context
;
1856 StoreIOBuffer tempBuffer
;
1857 http
= new ClientHttpRequest(csd
);
1858 http
->req_sz
= csd
->in
.notYetUsed
;
1859 http
->uri
= xstrdup(uri
);
1860 setLogUri (http
, uri
);
1861 context
= ClientSocketContextNew(csd
->clientConn
, http
);
1862 tempBuffer
.data
= context
->reqbuf
;
1863 tempBuffer
.length
= HTTP_REQBUF_SZ
;
1864 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
1865 clientReplyStatus
, new clientReplyContext(http
), clientSocketRecipient
,
1866 clientSocketDetach
, context
, tempBuffer
);
1871 skipLeadingSpace(char *aString
)
1873 char *result
= aString
;
1875 while (xisspace(*aString
))
1882 * 'end' defaults to NULL for backwards compatibility
1883 * remove default value if we ever get rid of NULL-terminated
1887 findTrailingHTTPVersion(const char *uriAndHTTPVersion
, const char *end
)
1890 end
= uriAndHTTPVersion
+ strcspn(uriAndHTTPVersion
, "\r\n");
1894 for (; end
> uriAndHTTPVersion
; end
--) {
1895 if (*end
== '\n' || *end
== '\r')
1898 if (xisspace(*end
)) {
1899 if (strncasecmp(end
+ 1, "HTTP/", 5) == 0)
1910 setLogUri(ClientHttpRequest
* http
, char const *uri
)
1912 safe_free(http
->log_uri
);
1914 if (!stringHasCntl(uri
))
1915 http
->log_uri
= xstrndup(uri
, MAX_URL
);
1917 http
->log_uri
= xstrndup(rfc1738_escape_unescaped(uri
), MAX_URL
);
1921 prepareAcceleratedURL(ConnStateData
* conn
, ClientHttpRequest
*http
, char *url
, const char *req_hdr
)
1923 int vhost
= conn
->port
->vhost
;
1924 int vport
= conn
->port
->vport
;
1926 char ipbuf
[MAX_IPSTRLEN
];
1928 http
->flags
.accel
= 1;
1930 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
1932 if (strncasecmp(url
, "cache_object://", 15) == 0)
1933 return; /* already in good shape */
1936 if (conn
->port
->vhost
)
1937 return; /* already in good shape */
1939 /* else we need to ignore the host name */
1940 url
= strstr(url
, "//");
1942 #if SHOULD_REJECT_UNKNOWN_URLS
1945 hp
->request_parse_status
= HTTP_BAD_REQUEST
;
1946 return parseHttpRequestAbort(conn
, "error:invalid-request");
1951 url
= strchr(url
+ 2, '/');
1957 if (internalCheck(url
)) {
1958 /* prepend our name & port */
1959 http
->uri
= xstrdup(internalLocalUri(NULL
, url
));
1963 const bool switchedToHttps
= conn
->switchedToHttps();
1964 const bool tryHostHeader
= vhost
|| switchedToHttps
;
1965 if (tryHostHeader
&& (host
= mime_get_header(req_hdr
, "Host")) != NULL
) {
1966 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
+
1968 http
->uri
= (char *)xcalloc(url_sz
, 1);
1969 const char *protocol
= switchedToHttps
?
1970 "https" : conn
->port
->protocol
;
1971 snprintf(http
->uri
, url_sz
, "%s://%s%s", protocol
, host
, url
);
1972 debugs(33, 5, "ACCEL VHOST REWRITE: '" << http
->uri
<< "'");
1973 } else if (conn
->port
->defaultsite
) {
1974 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
+
1975 strlen(conn
->port
->defaultsite
);
1976 http
->uri
= (char *)xcalloc(url_sz
, 1);
1977 snprintf(http
->uri
, url_sz
, "%s://%s%s",
1978 conn
->port
->protocol
, conn
->port
->defaultsite
, url
);
1979 debugs(33, 5, "ACCEL DEFAULTSITE REWRITE: '" << http
->uri
<<"'");
1980 } else if (vport
== -1) {
1981 /* Put the local socket IP address as the hostname. */
1982 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
;
1983 http
->uri
= (char *)xcalloc(url_sz
, 1);
1984 http
->getConn()->clientConn
->local
.ToHostname(ipbuf
,MAX_IPSTRLEN
);
1985 snprintf(http
->uri
, url_sz
, "%s://%s:%d%s",
1986 http
->getConn()->port
->protocol
,
1987 ipbuf
, http
->getConn()->clientConn
->local
.GetPort(), url
);
1988 debugs(33, 5, "ACCEL VPORT REWRITE: '" << http
->uri
<< "'");
1989 } else if (vport
> 0) {
1990 /* Put the local socket IP address as the hostname, but static port */
1991 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
;
1992 http
->uri
= (char *)xcalloc(url_sz
, 1);
1993 http
->getConn()->clientConn
->local
.ToHostname(ipbuf
,MAX_IPSTRLEN
);
1994 snprintf(http
->uri
, url_sz
, "%s://%s:%d%s",
1995 http
->getConn()->port
->protocol
,
1997 debugs(33, 5, "ACCEL VPORT REWRITE: '" << http
->uri
<< "'");
2002 prepareTransparentURL(ConnStateData
* conn
, ClientHttpRequest
*http
, char *url
, const char *req_hdr
)
2005 char ipbuf
[MAX_IPSTRLEN
];
2008 return; /* already in good shape */
2010 /* BUG: Squid cannot deal with '*' URLs (RFC2616 5.1.2) */
2012 if ((host
= mime_get_header(req_hdr
, "Host")) != NULL
) {
2013 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
+
2015 http
->uri
= (char *)xcalloc(url_sz
, 1);
2016 snprintf(http
->uri
, url_sz
, "%s://%s%s",
2017 conn
->port
->protocol
, host
, url
);
2018 debugs(33, 5, "TRANSPARENT HOST REWRITE: '" << http
->uri
<<"'");
2020 /* Put the local socket IP address as the hostname. */
2021 int url_sz
= strlen(url
) + 32 + Config
.appendDomainLen
;
2022 http
->uri
= (char *)xcalloc(url_sz
, 1);
2023 http
->getConn()->clientConn
->local
.ToHostname(ipbuf
,MAX_IPSTRLEN
),
2024 snprintf(http
->uri
, url_sz
, "%s://%s:%d%s",
2025 http
->getConn()->port
->protocol
,
2026 ipbuf
, http
->getConn()->clientConn
->local
.GetPort(), url
);
2027 debugs(33, 5, "TRANSPARENT REWRITE: '" << http
->uri
<< "'");
2032 * parseHttpRequest()
2035 * NULL on incomplete requests
2036 * a ClientSocketContext structure on success or failure.
2037 * Sets result->flags.parsed_ok to 0 if failed to parse the request.
2038 * Sets result->flags.parsed_ok to 1 if we have a good request.
2040 static ClientSocketContext
*
2041 parseHttpRequest(ConnStateData
*csd
, HttpParser
*hp
, HttpRequestMethod
* method_p
, HttpVersion
*http_ver
)
2043 char *req_hdr
= NULL
;
2046 ClientHttpRequest
*http
;
2047 ClientSocketContext
*result
;
2048 StoreIOBuffer tempBuffer
;
2051 /* pre-set these values to make aborting simpler */
2052 *method_p
= METHOD_NONE
;
2054 /* NP: don't be tempted to move this down or remove again.
2055 * It's the only DDoS protection old-String has against long URL */
2056 if ( hp
->bufsiz
<= 0) {
2057 debugs(33, 5, "Incomplete request, waiting for end of request line");
2059 } else if ( (size_t)hp
->bufsiz
>= Config
.maxRequestHeaderSize
&& headersEnd(hp
->buf
, Config
.maxRequestHeaderSize
) == 0) {
2060 debugs(33, 5, "parseHttpRequest: Too large request");
2061 hp
->request_parse_status
= HTTP_HEADER_TOO_LARGE
;
2062 return parseHttpRequestAbort(csd
, "error:request-too-large");
2065 /* Attempt to parse the first line; this'll define the method, url, version and header begin */
2066 r
= HttpParserParseReqLine(hp
);
2069 debugs(33, 5, "Incomplete request, waiting for end of request line");
2074 return parseHttpRequestAbort(csd
, "error:invalid-request");
2077 /* Request line is valid here .. */
2078 *http_ver
= HttpVersion(hp
->v_maj
, hp
->v_min
);
2080 /* This call scans the entire request, not just the headers */
2081 if (hp
->v_maj
> 0) {
2082 if ((req_sz
= headersEnd(hp
->buf
, hp
->bufsiz
)) == 0) {
2083 debugs(33, 5, "Incomplete request, waiting for end of headers");
2087 debugs(33, 3, "parseHttpRequest: Missing HTTP identifier");
2088 req_sz
= HttpParserReqSz(hp
);
2091 /* We know the whole request is in hp->buf now */
2093 assert(req_sz
<= (size_t) hp
->bufsiz
);
2095 /* Will the following be true with HTTP/0.9 requests? probably not .. */
2096 /* So the rest of the code will need to deal with '0'-byte headers (ie, none, so don't try parsing em) */
2099 hp
->hdr_end
= req_sz
- 1;
2101 hp
->hdr_start
= hp
->req_end
+ 1;
2103 /* Enforce max_request_size */
2104 if (req_sz
>= Config
.maxRequestHeaderSize
) {
2105 debugs(33, 5, "parseHttpRequest: Too large request");
2106 hp
->request_parse_status
= HTTP_HEADER_TOO_LARGE
;
2107 return parseHttpRequestAbort(csd
, "error:request-too-large");
2111 *method_p
= HttpRequestMethod(&hp
->buf
[hp
->m_start
], &hp
->buf
[hp
->m_end
]+1);
2113 /* deny CONNECT via accelerated ports */
2114 if (*method_p
== METHOD_CONNECT
&& csd
&& csd
->port
&& csd
->port
->accel
) {
2115 debugs(33, DBG_IMPORTANT
, "WARNING: CONNECT method received on " << csd
->port
->protocol
<< " Accelerator port " << csd
->port
->s
.GetPort() );
2116 /* XXX need a way to say "this many character length string" */
2117 debugs(33, DBG_IMPORTANT
, "WARNING: for request: " << hp
->buf
);
2118 hp
->request_parse_status
= HTTP_METHOD_NOT_ALLOWED
;
2119 return parseHttpRequestAbort(csd
, "error:method-not-allowed");
2122 if (*method_p
== METHOD_NONE
) {
2123 /* XXX need a way to say "this many character length string" */
2124 debugs(33, 1, "clientParseRequestMethod: Unsupported method in request '" << hp
->buf
<< "'");
2125 hp
->request_parse_status
= HTTP_METHOD_NOT_ALLOWED
;
2126 return parseHttpRequestAbort(csd
, "error:unsupported-request-method");
2130 * Process headers after request line
2131 * TODO: Use httpRequestParse here.
2133 /* XXX this code should be modified to take a const char * later! */
2134 req_hdr
= (char *) hp
->buf
+ hp
->req_end
+ 1;
2136 debugs(33, 3, "parseHttpRequest: req_hdr = {" << req_hdr
<< "}");
2138 end
= (char *) hp
->buf
+ hp
->hdr_end
;
2140 debugs(33, 3, "parseHttpRequest: end = {" << end
<< "}");
2142 debugs(33, 3, "parseHttpRequest: prefix_sz = " <<
2143 (int) HttpParserRequestLen(hp
) << ", req_line_sz = " <<
2144 HttpParserReqSz(hp
));
2146 /* Ok, all headers are received */
2147 http
= new ClientHttpRequest(csd
);
2149 http
->req_sz
= HttpParserRequestLen(hp
);
2150 result
= ClientSocketContextNew(csd
->clientConn
, http
);
2151 tempBuffer
.data
= result
->reqbuf
;
2152 tempBuffer
.length
= HTTP_REQBUF_SZ
;
2154 ClientStreamData newServer
= new clientReplyContext(http
);
2155 ClientStreamData newClient
= result
;
2156 clientStreamInit(&http
->client_stream
, clientGetMoreData
, clientReplyDetach
,
2157 clientReplyStatus
, newServer
, clientSocketRecipient
,
2158 clientSocketDetach
, newClient
, tempBuffer
);
2160 debugs(33, 5, "parseHttpRequest: Request Header is\n" <<(hp
->buf
) + hp
->hdr_start
);
2164 * XXX this should eventually not use a malloc'ed buffer; the transformation code
2165 * below needs to be modified to not expect a mutable nul-terminated string.
2167 char *url
= (char *)xmalloc(hp
->u_end
- hp
->u_start
+ 16);
2169 memcpy(url
, hp
->buf
+ hp
->u_start
, hp
->u_end
- hp
->u_start
+ 1);
2171 url
[hp
->u_end
- hp
->u_start
+ 1] = '\0';
2173 #if THIS_VIOLATES_HTTP_SPECS_ON_URL_TRANSFORMATION
2175 if ((t
= strchr(url
, '#'))) /* remove HTML anchors */
2180 /* Rewrite the URL in transparent or accelerator mode */
2181 /* NP: there are several cases to traverse here:
2182 * - standard mode (forward proxy)
2183 * - transparent mode (TPROXY)
2184 * - transparent mode with failures
2185 * - intercept mode (NAT)
2186 * - intercept mode with failures
2187 * - accelerator mode (reverse proxy)
2189 * - mixed combos of the above with internal URL
2191 if (csd
->transparent()) {
2192 /* intercept or transparent mode, properly working with no failures */
2193 http
->flags
.intercepted
= csd
->port
->intercepted
;
2194 http
->flags
.spoof_client_ip
= csd
->port
->spoof_client_ip
;
2195 prepareTransparentURL(csd
, http
, url
, req_hdr
);
2197 } else if (csd
->port
->intercepted
|| csd
->port
->spoof_client_ip
) {
2198 /* transparent or intercept mode with failures */
2199 prepareTransparentURL(csd
, http
, url
, req_hdr
);
2201 } else if (csd
->port
->accel
|| csd
->switchedToHttps()) {
2202 /* accelerator mode */
2203 prepareAcceleratedURL(csd
, http
, url
, req_hdr
);
2205 } else if (internalCheck(url
)) {
2206 /* internal URL mode */
2207 /* prepend our name & port */
2208 http
->uri
= xstrdup(internalLocalUri(NULL
, url
));
2209 http
->flags
.accel
= 1;
2213 /* No special rewrites have been applied above, use the
2214 * requested url. may be rewritten later, so make extra room */
2215 int url_sz
= strlen(url
) + Config
.appendDomainLen
+ 5;
2216 http
->uri
= (char *)xcalloc(url_sz
, 1);
2217 strcpy(http
->uri
, url
);
2220 setLogUri(http
, http
->uri
);
2221 debugs(33, 5, "parseHttpRequest: Complete request received");
2222 result
->flags
.parsed_ok
= 1;
2228 ConnStateData::getAvailableBufferLength() const
2230 int result
= in
.allocatedSize
- in
.notYetUsed
- 1;
2231 assert (result
>= 0);
2236 ConnStateData::makeSpaceAvailable()
2238 if (getAvailableBufferLength() < 2) {
2239 in
.buf
= (char *)memReallocBuf(in
.buf
, in
.allocatedSize
* 2, &in
.allocatedSize
);
2240 debugs(33, 2, "growing request buffer: notYetUsed=" << in
.notYetUsed
<< " size=" << in
.allocatedSize
);
2245 ConnStateData::addContextToQueue(ClientSocketContext
* context
)
2247 ClientSocketContext::Pointer
*S
;
2249 for (S
= (ClientSocketContext::Pointer
*) & currentobject
; S
->getRaw();
2257 ConnStateData::getConcurrentRequestCount() const
2260 ClientSocketContext::Pointer
*T
;
2262 for (T
= (ClientSocketContext::Pointer
*) ¤tobject
;
2263 T
->getRaw(); T
= &(*T
)->next
, ++result
);
2268 ConnStateData::connReadWasError(comm_err_t flag
, int size
, int xerrno
)
2270 if (flag
!= COMM_OK
) {
2271 debugs(33, 2, "connReadWasError: FD " << (clientConn
!=NULL
?clientConn
->fd
:-1) << ": got flag " << flag
);
2276 if (!ignoreErrno(xerrno
)) {
2277 debugs(33, 2, "connReadWasError: FD " << clientConn
->fd
<< ": " << xstrerr(xerrno
));
2279 } else if (in
.notYetUsed
== 0) {
2280 debugs(33, 2, "connReadWasError: FD " << clientConn
->fd
<< ": no data to process (" << xstrerr(xerrno
) << ")");
2288 ConnStateData::connFinishedWithConn(int size
)
2291 if (getConcurrentRequestCount() == 0 && in
.notYetUsed
== 0) {
2292 /* no current or pending requests */
2293 debugs(33, 4, "connFinishedWithConn: FD " << clientConn
->fd
<< " closed");
2295 } else if (!Config
.onoff
.half_closed_clients
) {
2296 /* admin doesn't want to support half-closed client sockets */
2297 debugs(33, 3, "connFinishedWithConn: FD " << clientConn
->fd
<< " aborted (half_closed_clients disabled)");
2298 notifyAllContexts(0); // no specific error implies abort
2307 connNoteUseOfBuffer(ConnStateData
* conn
, size_t byteCount
)
2309 assert(byteCount
> 0 && byteCount
<= conn
->in
.notYetUsed
);
2310 conn
->in
.notYetUsed
-= byteCount
;
2311 debugs(33, 5, HERE
<< "conn->in.notYetUsed = " << conn
->in
.notYetUsed
);
2313 * If there is still data that will be used,
2314 * move it to the beginning.
2317 if (conn
->in
.notYetUsed
> 0)
2318 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ byteCount
,
2319 conn
->in
.notYetUsed
);
2322 /// respond with ERR_TOO_BIG if request header exceeds request_header_max_size
2324 ConnStateData::checkHeaderLimits()
2326 if (in
.notYetUsed
< Config
.maxRequestHeaderSize
)
2327 return; // can accumulte more header data
2329 debugs(33, 3, "Request header is too large (" << in
.notYetUsed
<< " > " <<
2330 Config
.maxRequestHeaderSize
<< " bytes)");
2332 ClientSocketContext
*context
= parseHttpRequestAbort(this, "error:request-too-large");
2333 clientStreamNode
*node
= context
->getClientReplyContext();
2334 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2335 assert (repContext
);
2336 repContext
->setReplyToError(ERR_TOO_BIG
,
2337 HTTP_BAD_REQUEST
, METHOD_NONE
, NULL
,
2338 clientConn
->remote
, NULL
, NULL
, NULL
);
2339 context
->registerWithConn();
2340 context
->pullData();
2344 ConnStateData::clientMaybeReadData(int do_next_read
)
2347 flags
.readMoreRequests
= true;
2353 ConnStateData::clientAfterReadingRequests(int do_next_read
)
2355 // Were we expecting to read more request body from half-closed connection?
2356 if (mayNeedToReadMoreBody() && commIsHalfClosed(clientConn
->fd
)) {
2357 debugs(33, 3, HERE
<< "truncated body: closing half-closed " << clientConn
);
2358 clientConn
->close();
2362 clientMaybeReadData (do_next_read
);
2366 clientProcessRequest(ConnStateData
*conn
, HttpParser
*hp
, ClientSocketContext
*context
, const HttpRequestMethod
& method
, HttpVersion http_ver
)
2368 ClientHttpRequest
*http
= context
->http
;
2369 HttpRequest
*request
= NULL
;
2370 bool notedUseOfBuffer
= false;
2371 bool chunked
= false;
2372 bool mustReplyToOptions
= false;
2373 bool unsupportedTe
= false;
2374 bool expectBody
= false;
2376 /* We have an initial client stream in place should it be needed */
2377 /* setup our private context */
2378 context
->registerWithConn();
2380 if (context
->flags
.parsed_ok
== 0) {
2381 clientStreamNode
*node
= context
->getClientReplyContext();
2382 debugs(33, 1, "clientProcessRequest: Invalid Request");
2383 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2384 assert (repContext
);
2385 switch (hp
->request_parse_status
) {
2386 case HTTP_HEADER_TOO_LARGE
:
2387 repContext
->setReplyToError(ERR_TOO_BIG
, HTTP_BAD_REQUEST
, method
, http
->uri
, conn
->clientConn
->remote
, NULL
, conn
->in
.buf
, NULL
);
2389 case HTTP_METHOD_NOT_ALLOWED
:
2390 repContext
->setReplyToError(ERR_UNSUP_REQ
, HTTP_METHOD_NOT_ALLOWED
, method
, http
->uri
,
2391 conn
->clientConn
->remote
, NULL
, conn
->in
.buf
, NULL
);
2394 repContext
->setReplyToError(ERR_INVALID_REQ
, HTTP_BAD_REQUEST
, method
, http
->uri
,
2395 conn
->clientConn
->remote
, NULL
, conn
->in
.buf
, NULL
);
2397 assert(context
->http
->out
.offset
== 0);
2398 context
->pullData();
2399 conn
->flags
.readMoreRequests
= false;
2403 if ((request
= HttpRequest::CreateFromUrlAndMethod(http
->uri
, method
)) == NULL
) {
2404 clientStreamNode
*node
= context
->getClientReplyContext();
2405 debugs(33, 5, "Invalid URL: " << http
->uri
);
2406 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2407 assert (repContext
);
2408 repContext
->setReplyToError(ERR_INVALID_URL
, HTTP_BAD_REQUEST
, method
, http
->uri
, conn
->clientConn
->remote
, NULL
, NULL
, NULL
);
2409 assert(context
->http
->out
.offset
== 0);
2410 context
->pullData();
2411 conn
->flags
.readMoreRequests
= false;
2415 /* RFC 2616 section 10.5.6 : handle unsupported HTTP versions cleanly. */
2416 /* We currently only accept 0.9, 1.0, 1.1 */
2417 if ( (http_ver
.major
== 0 && http_ver
.minor
!= 9) ||
2418 (http_ver
.major
== 1 && http_ver
.minor
> 1 ) ||
2419 (http_ver
.major
> 1) ) {
2421 clientStreamNode
*node
= context
->getClientReplyContext();
2422 debugs(33, 5, "Unsupported HTTP version discovered. :\n" << HttpParserHdrBuf(hp
));
2423 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2424 assert (repContext
);
2425 repContext
->setReplyToError(ERR_UNSUP_HTTPVERSION
, HTTP_HTTP_VERSION_NOT_SUPPORTED
, method
, http
->uri
,
2426 conn
->clientConn
->remote
, NULL
, HttpParserHdrBuf(hp
), NULL
);
2427 assert(context
->http
->out
.offset
== 0);
2428 context
->pullData();
2429 conn
->flags
.readMoreRequests
= false;
2433 /* compile headers */
2434 /* we should skip request line! */
2435 /* XXX should actually know the damned buffer size here */
2436 if (http_ver
.major
>= 1 && !request
->parseHeader(HttpParserHdrBuf(hp
), HttpParserHdrSz(hp
))) {
2437 clientStreamNode
*node
= context
->getClientReplyContext();
2438 debugs(33, 5, "Failed to parse request headers:\n" << HttpParserHdrBuf(hp
));
2439 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2440 assert (repContext
);
2441 repContext
->setReplyToError(ERR_INVALID_REQ
, HTTP_BAD_REQUEST
, method
, http
->uri
, conn
->clientConn
->remote
, NULL
, NULL
, NULL
);
2442 assert(context
->http
->out
.offset
== 0);
2443 context
->pullData();
2444 conn
->flags
.readMoreRequests
= false;
2448 request
->flags
.accelerated
= http
->flags
.accel
;
2449 request
->flags
.ignore_cc
= conn
->port
->ignore_cc
;
2450 request
->flags
.no_direct
= request
->flags
.accelerated
? !conn
->port
->allow_direct
: 0;
2453 * If transparent or interception mode is working clone the transparent and interception flags
2454 * from the port settings to the request.
2456 if (Ip::Interceptor
.InterceptActive()) {
2457 request
->flags
.intercepted
= http
->flags
.intercepted
;
2459 if (Ip::Interceptor
.TransparentActive()) {
2460 request
->flags
.spoof_client_ip
= conn
->port
->spoof_client_ip
;
2463 if (internalCheck(request
->urlpath
.termedBuf())) {
2464 if (internalHostnameIs(request
->GetHost()) &&
2465 request
->port
== getMyPort()) {
2466 http
->flags
.internal
= 1;
2467 } else if (Config
.onoff
.global_internal_static
&& internalStaticCheck(request
->urlpath
.termedBuf())) {
2468 request
->SetHost(internalHostname());
2469 request
->port
= getMyPort();
2470 http
->flags
.internal
= 1;
2474 if (http
->flags
.internal
) {
2475 request
->protocol
= PROTO_HTTP
;
2476 request
->login
[0] = '\0';
2479 request
->flags
.internal
= http
->flags
.internal
;
2480 setLogUri (http
, urlCanonicalClean(request
));
2481 request
->client_addr
= conn
->clientConn
->remote
; // XXX: remove reuest->client_addr member.
2483 request
->client_eui48
= conn
->peer_eui48
;
2484 request
->client_eui64
= conn
->peer_eui64
;
2486 #if FOLLOW_X_FORWARDED_FOR
2487 request
->indirect_client_addr
= conn
->clientConn
->remote
;
2488 #endif /* FOLLOW_X_FORWARDED_FOR */
2489 request
->my_addr
= conn
->clientConn
->local
;
2490 request
->myportname
= conn
->port
->name
;
2491 request
->http_ver
= http_ver
;
2493 if (request
->header
.chunked()) {
2495 } else if (request
->header
.has(HDR_TRANSFER_ENCODING
)) {
2496 const String te
= request
->header
.getList(HDR_TRANSFER_ENCODING
);
2497 // HTTP/1.1 requires chunking to be the last encoding if there is one
2498 unsupportedTe
= te
.size() && te
!= "identity";
2499 } // else implied identity coding
2501 if (method
== METHOD_TRACE
|| method
== METHOD_OPTIONS
)
2502 request
->max_forwards
= request
->header
.getInt64(HDR_MAX_FORWARDS
);
2504 mustReplyToOptions
= (method
== METHOD_OPTIONS
) && (request
->max_forwards
== 0);
2505 if (!urlCheckRequest(request
) || mustReplyToOptions
|| unsupportedTe
) {
2506 clientStreamNode
*node
= context
->getClientReplyContext();
2507 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2508 assert (repContext
);
2509 repContext
->setReplyToError(ERR_UNSUP_REQ
, HTTP_NOT_IMPLEMENTED
, request
->method
, NULL
,
2510 conn
->clientConn
->remote
, request
, NULL
, NULL
);
2511 assert(context
->http
->out
.offset
== 0);
2512 context
->pullData();
2513 conn
->flags
.readMoreRequests
= false;
2518 if (!chunked
&& !clientIsContentLengthValid(request
)) {
2519 clientStreamNode
*node
= context
->getClientReplyContext();
2520 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2521 assert (repContext
);
2522 repContext
->setReplyToError(ERR_INVALID_REQ
,
2523 HTTP_LENGTH_REQUIRED
, request
->method
, NULL
,
2524 conn
->clientConn
->remote
, request
, NULL
, NULL
);
2525 assert(context
->http
->out
.offset
== 0);
2526 context
->pullData();
2527 conn
->flags
.readMoreRequests
= false;
2531 if (request
->header
.has(HDR_EXPECT
)) {
2532 const String expect
= request
->header
.getList(HDR_EXPECT
);
2533 const bool supportedExpect
= (expect
.caseCmp("100-continue") == 0);
2534 if (!supportedExpect
) {
2535 clientStreamNode
*node
= context
->getClientReplyContext();
2536 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2537 assert (repContext
);
2538 repContext
->setReplyToError(ERR_INVALID_REQ
, HTTP_EXPECTATION_FAILED
, request
->method
, http
->uri
,
2539 conn
->clientConn
->remote
, request
, NULL
, NULL
);
2540 assert(context
->http
->out
.offset
== 0);
2541 context
->pullData();
2546 http
->request
= HTTPMSGLOCK(request
);
2547 clientSetKeepaliveFlag(http
);
2549 /* If this is a CONNECT, don't schedule a read - ssl.c will handle it */
2550 if (http
->request
->method
== METHOD_CONNECT
)
2551 context
->mayUseConnection(true);
2553 /* Do we expect a request-body? */
2554 expectBody
= chunked
|| request
->content_length
> 0;
2555 if (!context
->mayUseConnection() && expectBody
) {
2556 request
->body_pipe
= conn
->expectRequestBody(
2557 chunked
? -1 : request
->content_length
);
2559 // consume header early so that body pipe gets just the body
2560 connNoteUseOfBuffer(conn
, http
->req_sz
);
2561 notedUseOfBuffer
= true;
2563 /* Is it too large? */
2564 if (!chunked
&& // if chunked, we will check as we accumulate
2565 clientIsRequestBodyTooLargeForPolicy(request
->content_length
)) {
2566 clientStreamNode
*node
= context
->getClientReplyContext();
2567 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2568 assert (repContext
);
2569 repContext
->setReplyToError(ERR_TOO_BIG
,
2570 HTTP_REQUEST_ENTITY_TOO_LARGE
, METHOD_NONE
, NULL
,
2571 conn
->clientConn
->remote
, http
->request
, NULL
, NULL
);
2572 assert(context
->http
->out
.offset
== 0);
2573 context
->pullData();
2577 // We may stop producing, comm_close, and/or call setReplyToError()
2578 // below, so quit on errors to avoid http->doCallouts()
2579 if (!conn
->handleRequestBodyData())
2582 if (!request
->body_pipe
->productionEnded())
2583 conn
->readSomeData();
2585 context
->mayUseConnection(!request
->body_pipe
->productionEnded());
2588 http
->calloutContext
= new ClientRequestContext(http
);
2593 if (!notedUseOfBuffer
)
2594 connNoteUseOfBuffer(conn
, http
->req_sz
);
2598 * Moved the TCP_RESET feature from clientReplyContext::sendMoreData
2599 * to here because calling comm_reset_close() causes http to
2600 * be freed and the above connNoteUseOfBuffer() would hit an
2601 * assertion, not to mention that we were accessing freed memory.
2603 if (http
->request
->flags
.resetTCP() && Comm::IsConnOpen(conn
->clientConn
)) {
2604 debugs(33, 3, HERE
<< "Sending TCP RST on " << conn
->clientConn
);
2605 conn
->flags
.readMoreRequests
= false;
2606 comm_reset_close(conn
->clientConn
);
2612 connStripBufferWhitespace (ConnStateData
* conn
)
2614 while (conn
->in
.notYetUsed
> 0 && xisspace(conn
->in
.buf
[0])) {
2615 xmemmove(conn
->in
.buf
, conn
->in
.buf
+ 1, conn
->in
.notYetUsed
- 1);
2616 --conn
->in
.notYetUsed
;
2621 connOkToAddRequest(ConnStateData
* conn
)
2623 int result
= conn
->getConcurrentRequestCount() < (Config
.onoff
.pipeline_prefetch
? 2 : 1);
2626 debugs(33, 3, HERE
<< conn
->clientConn
<< " max concurrent requests reached");
2627 debugs(33, 5, HERE
<< conn
->clientConn
<< " defering new request until one is done");
2634 * Attempt to parse one or more requests from the input buffer.
2635 * If a request is successfully parsed, even if the next request
2636 * is only partially parsed, it will return TRUE.
2637 * do_next_read is updated to indicate whether a read should be
2641 clientParseRequest(ConnStateData
* conn
, bool &do_next_read
)
2643 HttpRequestMethod method
;
2644 ClientSocketContext
*context
;
2645 bool parsed_req
= false;
2646 HttpVersion http_ver
;
2649 debugs(33, 5, HERE
<< conn
->clientConn
<< ": attempting to parse");
2651 // Loop while we have read bytes that are not needed for producing the body
2652 // On errors, bodyPipe may become nil, but readMoreRequests will be cleared
2653 while (conn
->in
.notYetUsed
> 0 && !conn
->bodyPipe
&&
2654 conn
->flags
.readMoreRequests
) {
2655 connStripBufferWhitespace (conn
);
2657 /* Don't try to parse if the buffer is empty */
2659 if (conn
->in
.notYetUsed
== 0)
2662 /* Limit the number of concurrent requests to 2 */
2664 if (!connOkToAddRequest(conn
)) {
2668 /* Should not be needed anymore */
2669 /* Terminate the string */
2670 conn
->in
.buf
[conn
->in
.notYetUsed
] = '\0';
2672 /* Begin the parsing */
2673 HttpParserInit(&hp
, conn
->in
.buf
, conn
->in
.notYetUsed
);
2675 /* Process request */
2676 PROF_start(parseHttpRequest
);
2678 context
= parseHttpRequest(conn
, &hp
, &method
, &http_ver
);
2680 PROF_stop(parseHttpRequest
);
2682 /* partial or incomplete request */
2684 // TODO: why parseHttpRequest can just return parseHttpRequestAbort
2685 // (which becomes context) but checkHeaderLimits cannot?
2686 conn
->checkHeaderLimits();
2690 /* status -1 or 1 */
2692 debugs(33, 5, HERE
<< conn
->clientConn
<< ": parsed a request");
2693 commSetTimeout(conn
->clientConn
->fd
, Config
.Timeout
.lifetime
, clientLifetimeTimeout
,
2696 clientProcessRequest(conn
, &hp
, context
, method
, http_ver
);
2700 if (context
->mayUseConnection()) {
2701 debugs(33, 3, "clientParseRequest: Not reading, as this request may need the connection");
2708 /* XXX where to 'finish' the parsing pass? */
2714 ConnStateData::clientReadRequest(const CommIoCbParams
&io
)
2716 debugs(33,5,HERE
<< io
.conn
<< " size " << io
.size
);
2719 bool do_next_read
= 1; /* the default _is_ to read data! - adrian */
2721 /* Bail out quickly on COMM_ERR_CLOSING - close handlers will tidy up */
2723 if (io
.flag
== COMM_ERR_CLOSING
) {
2724 debugs(33,5, HERE
<< io
.conn
<< " closing Bailout.");
2728 assert(Comm::IsConnOpen(clientConn
));
2729 assert(io
.conn
== clientConn
);
2732 * Don't reset the timeout value here. The timeout value will be
2733 * set to Config.Timeout.request by httpAccept() and
2734 * clientWriteComplete(), and should apply to the request as a
2735 * whole, not individual read() calls. Plus, it breaks our
2736 * lame half-close detection
2738 if (connReadWasError(io
.flag
, io
.size
, io
.xerrno
)) {
2739 notifyAllContexts(io
.xerrno
);
2744 if (io
.flag
== COMM_OK
) {
2746 kb_incr(&statCounter
.client_http
.kbytes_in
, io
.size
);
2748 // may comm_close or setReplyToError
2749 if (!handleReadData(io
.buf
, io
.size
))
2752 } else if (io
.size
== 0) {
2753 debugs(33, 5, HERE
<< io
.conn
<< " closed?");
2755 if (connFinishedWithConn(io
.size
)) {
2756 clientConn
->close();
2760 /* It might be half-closed, we can't tell */
2761 fd_table
[io
.conn
->fd
].flags
.socket_eof
= 1;
2763 commMarkHalfClosed(io
.conn
->fd
);
2767 fd_note(io
.conn
->fd
, "half-closed");
2769 /* There is one more close check at the end, to detect aborted
2770 * (partial) requests. At this point we can't tell if the request
2773 /* Continue to process previously read data */
2777 /* Process next request */
2778 if (getConcurrentRequestCount() == 0)
2779 fd_note(io
.fd
, "Reading next request");
2781 if (! clientParseRequest(this, do_next_read
)) {
2785 * If the client here is half closed and we failed
2786 * to parse a request, close the connection.
2787 * The above check with connFinishedWithConn() only
2788 * succeeds _if_ the buffer is empty which it won't
2789 * be if we have an incomplete request.
2790 * XXX: This duplicates ClientSocketContext::keepaliveNextRequest
2792 if (getConcurrentRequestCount() == 0 && commIsHalfClosed(io
.fd
)) {
2793 debugs(33, 5, HERE
<< io
.conn
<< ": half-closed connection, no completed request parsed, connection closing.");
2794 clientConn
->close();
2802 clientAfterReadingRequests(do_next_read
);
2806 * called when new request data has been read from the socket
2808 * \retval false called comm_close or setReplyToError (the caller should bail)
2809 * \retval true we did not call comm_close or setReplyToError
2812 ConnStateData::handleReadData(char *buf
, size_t size
)
2814 char *current_buf
= in
.addressToReadInto();
2816 if (buf
!= current_buf
)
2817 xmemmove(current_buf
, buf
, size
);
2819 in
.notYetUsed
+= size
;
2821 in
.buf
[in
.notYetUsed
] = '\0'; /* Terminate the string */
2823 // if we are reading a body, stuff data into the body pipe
2824 if (bodyPipe
!= NULL
)
2825 return handleRequestBodyData();
2830 * called when new request body data has been buffered in in.buf
2831 * may close the connection if we were closing and piped everything out
2833 * \retval false called comm_close or setReplyToError (the caller should bail)
2834 * \retval true we did not call comm_close or setReplyToError
2837 ConnStateData::handleRequestBodyData()
2839 assert(bodyPipe
!= NULL
);
2843 if (in
.bodyParser
) { // chunked encoding
2844 if (const err_type error
= handleChunkedRequestBody(putSize
)) {
2845 abortChunkedRequestBody(error
);
2848 } else { // identity encoding
2849 debugs(33,5, HERE
<< "handling plain request body for " << clientConn
);
2850 putSize
= bodyPipe
->putMoreData(in
.buf
, in
.notYetUsed
);
2851 if (!bodyPipe
->mayNeedMoreData()) {
2852 // BodyPipe will clear us automagically when we produced everything
2858 connNoteUseOfBuffer(this, putSize
);
2861 debugs(33,5, HERE
<< "produced entire request body for " << clientConn
);
2864 /* we've finished reading like good clients,
2865 * now do the close that initiateClose initiated.
2867 clientConn
->close();
2875 /// parses available chunked encoded body bytes, checks size, returns errors
2877 ConnStateData::handleChunkedRequestBody(size_t &putSize
)
2879 debugs(33,7, HERE
<< "chunked from " << clientConn
<< ": " << in
.notYetUsed
);
2881 try { // the parser will throw on errors
2883 if (!in
.notYetUsed
) // nothing to do (MemBuf::init requires this check)
2886 MemBuf raw
; // ChunkedCodingParser only works with MemBufs
2887 // add one because MemBuf will assert if it cannot 0-terminate
2888 raw
.init(in
.notYetUsed
, in
.notYetUsed
+1);
2889 raw
.append(in
.buf
, in
.notYetUsed
);
2891 const mb_size_t wasContentSize
= raw
.contentSize();
2892 BodyPipeCheckout
bpc(*bodyPipe
);
2893 const bool parsed
= in
.bodyParser
->parse(&raw
, &bpc
.buf
);
2895 putSize
= wasContentSize
- raw
.contentSize();
2897 // dechunk then check: the size limit applies to _dechunked_ content
2898 if (clientIsRequestBodyTooLargeForPolicy(bodyPipe
->producedSize()))
2902 finishDechunkingRequest(true);
2904 return ERR_NONE
; // nil bodyPipe implies body end for the caller
2907 // if chunk parser needs data, then the body pipe must need it too
2908 Must(!in
.bodyParser
->needsMoreData() || bodyPipe
->mayNeedMoreData());
2910 // if parser needs more space and we can consume nothing, we will stall
2911 Must(!in
.bodyParser
->needsMoreSpace() || bodyPipe
->buf().hasContent());
2912 } catch (...) { // TODO: be more specific
2913 debugs(33, 3, HERE
<< "malformed chunks" << bodyPipe
->status());
2914 return ERR_INVALID_REQ
;
2917 debugs(33, 7, HERE
<< "need more chunked data" << *bodyPipe
->status());
2921 /// quit on errors related to chunked request body handling
2923 ConnStateData::abortChunkedRequestBody(const err_type error
)
2925 finishDechunkingRequest(false);
2927 // XXX: The code below works if we fail during initial request parsing,
2928 // but if we fail when the server-side works already, the server may send
2929 // us its response too, causing various assertions. How to prevent that?
2930 #if WE_KNOW_HOW_TO_SEND_ERRORS
2931 ClientSocketContext::Pointer context
= getCurrentContext();
2932 if (context
!= NULL
&& !context
->http
->out
.offset
) { // output nothing yet
2933 clientStreamNode
*node
= context
->getClientReplyContext();
2934 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2936 const http_status scode
= (error
== ERR_TOO_BIG
) ?
2937 HTTP_REQUEST_ENTITY_TOO_LARGE
: HTTP_BAD_REQUEST
;
2938 repContext
->setReplyToError(error
, scode
,
2939 repContext
->http
->request
->method
,
2940 repContext
->http
->uri
,
2942 repContext
->http
->request
,
2944 context
->pullData();
2946 // close or otherwise we may get stuck as nobody will notice the error?
2947 comm_reset_close(clientConn
);
2950 debugs(33, 3, HERE
<< "aborting chunked request without error " << error
);
2951 comm_reset_close(clientConn
);
2953 flags
.readMoreRequests
= false;
2957 ConnStateData::noteMoreBodySpaceAvailable(BodyPipe::Pointer
)
2959 handleRequestBodyData();
2963 ConnStateData::noteBodyConsumerAborted(BodyPipe::Pointer
)
2966 startClosing("body consumer aborted");
2969 /** general lifetime handler for HTTP requests */
2971 ConnStateData::requestTimeout(const CommTimeoutCbParams
&io
)
2973 #if THIS_CONFUSES_PERSISTENT_CONNECTION_AWARE_BROWSERS_AND_USERS
2974 debugs(33, 3, "requestTimeout: FD " << io
.fd
<< ": lifetime is expired.");
2976 if (COMMIO_FD_WRITECB(io
.fd
)->active
) {
2977 /* FIXME: If this code is reinstated, check the conn counters,
2978 * not the fd table state
2981 * Some data has been sent to the client, just close the FD
2983 clientConn
->close();
2984 } else if (nrequests
) {
2986 * assume its a persistent connection; just close it
2988 clientConn
->close();
2993 ClientHttpRequest
**H
;
2994 clientStreamNode
*node
;
2995 ClientHttpRequest
*http
= parseHttpRequestAbort(this, "error:Connection%20lifetime%20expired");
2996 node
= http
->client_stream
.tail
->prev
->data
;
2997 clientReplyContext
*repContext
= dynamic_cast<clientReplyContext
*>(node
->data
.getRaw());
2998 assert (repContext
);
2999 repContext
->setReplyToError(ERR_LIFETIME_EXP
,
3000 HTTP_REQUEST_TIMEOUT
, METHOD_NONE
, "N/A", &peer
.sin_addr
,
3002 /* No requests can be outstanded */
3003 assert(chr
== NULL
);
3004 /* add to the client request queue */
3006 for (H
= &chr
; *H
; H
= &(*H
)->next
);
3009 clientStreamRead(http
->client_stream
.tail
->data
, http
, 0,
3010 HTTP_REQBUF_SZ
, context
->reqbuf
);
3013 * if we don't close() here, we still need a timeout handler!
3015 typedef CommCbMemFunT
<ConnStateData
, CommTimeoutCbParams
> TimeoutDialer
;
3016 AsyncCall::Pointer timeoutCall
= JobCallback(33, 5,
3017 TimeoutDialer
, this, ConnStateData::requestTimeout
);
3018 commSetTimeout(io
.fd
, 30, timeoutCall
);
3021 * Aha, but we don't want a read handler!
3023 commSetSelect(io
.fd
, COMM_SELECT_READ
, NULL
, NULL
, 0);
3028 * Just close the connection to not confuse browsers
3029 * using persistent connections. Some browsers opens
3030 * an connection and then does not use it until much
3031 * later (presumeably because the request triggering
3032 * the open has already been completed on another
3035 debugs(33, 3, "requestTimeout: FD " << io
.fd
<< ": lifetime is expired.");
3037 clientConn
->close();
3043 clientLifetimeTimeout(int fd
, void *data
)
3045 ClientHttpRequest
*http
= (ClientHttpRequest
*)data
;
3046 debugs(33, DBG_IMPORTANT
, "WARNING: Closing client connection due to lifetime timeout");
3047 debugs(33, DBG_IMPORTANT
, "\t" << http
->uri
);
3048 http
->al
.http
.timedout
= true;
3049 if (http
->getConn() && Comm::IsConnOpen(http
->getConn()->clientConn
))
3050 http
->getConn()->clientConn
->close();
3054 connStateCreate(const Comm::ConnectionPointer
&client
, http_port_list
*port
)
3056 ConnStateData
*result
= new ConnStateData
;
3058 result
->clientConn
= client
;
3059 result
->log_addr
= client
->remote
;
3060 result
->log_addr
.ApplyMask(Config
.Addrs
.client_netmask
);
3061 result
->in
.buf
= (char *)memAllocBuf(CLIENT_REQ_BUF_SZ
, &result
->in
.allocatedSize
);
3062 result
->port
= cbdataReference(port
);
3064 // XXX: move the NAT and TPROXY stuff into ConnAcceptor
3065 if (port
->intercepted
|| port
->spoof_client_ip
) {
3066 Ip::Address cl
, dst
;
3068 if (Ip::Interceptor
.NatLookup(client
->fd
, client
->local
, client
->remote
, cl
, dst
) == 0) {
3069 result
->clientConn
->local
= cl
;
3070 result
->clientConn
->remote
= dst
;
3071 result
->transparent(true);
3075 if (port
->disable_pmtu_discovery
!= DISABLE_PMTU_OFF
&&
3076 (result
->transparent() || port
->disable_pmtu_discovery
== DISABLE_PMTU_ALWAYS
)) {
3077 #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT)
3078 int i
= IP_PMTUDISC_DONT
;
3079 setsockopt(client
->fd
, SOL_IP
, IP_MTU_DISCOVER
, &i
, sizeof i
);
3083 static int reported
= 0;
3086 debugs(33, 1, "Notice: httpd_accel_no_pmtu_disc not supported on your platform");
3094 result
->flags
.readMoreRequests
= true;
3098 /** Handle a new connection on HTTP socket. */
3100 httpAccept(int sock
, const Comm::ConnectionPointer
&details
, comm_err_t flag
, int xerrno
, void *data
)
3102 http_port_list
*s
= (http_port_list
*)data
;
3103 ConnStateData
*connState
= NULL
;
3105 assert(flag
== COMM_OK
); // acceptor does not call us for anything bad.
3107 debugs(33, 4, HERE
<< details
<< ": accepted");
3108 fd_note(details
->fd
, "client http connect");
3109 connState
= connStateCreate(details
, s
);
3111 typedef CommCbMemFunT
<ConnStateData
, CommCloseCbParams
> Dialer
;
3112 AsyncCall::Pointer call
= JobCallback(33, 5, Dialer
, connState
, ConnStateData::connStateClosed
);
3113 comm_add_close_handler(details
->fd
, call
);
3115 if (Config
.onoff
.log_fqdn
)
3116 fqdncache_gethostbyaddr(details
->remote
, FQDN_LOOKUP_IF_MISS
);
3118 typedef CommCbMemFunT
<ConnStateData
, CommTimeoutCbParams
> TimeoutDialer
;
3119 AsyncCall::Pointer timeoutCall
= JobCallback(33, 5,
3120 TimeoutDialer
, connState
, ConnStateData::requestTimeout
);
3121 commSetTimeout(details
->fd
, Config
.Timeout
.read
, timeoutCall
);
3124 if (Ident::TheConfig
.identLookup
) {
3125 ACLFilledChecklist
identChecklist(Ident::TheConfig
.identLookup
, NULL
, NULL
);
3126 identChecklist
.src_addr
= details
->remote
;
3127 identChecklist
.my_addr
= details
->local
;
3128 if (identChecklist
.fastCheck())
3129 Ident::Start(details
, clientIdentDone
, connState
);
3134 if (Eui::TheConfig
.euiLookup
) {
3135 if (details
->remote
.IsIPv4()) {
3136 connState
->peer_eui48
.lookup(details
->remote
);
3137 } else if (details
->remote
.IsIPv6()) {
3138 connState
->peer_eui64
.lookup(details
->remote
);
3143 if (s
->tcp_keepalive
.enabled
) {
3144 commSetTcpKeepalive(details
->fd
, s
->tcp_keepalive
.idle
, s
->tcp_keepalive
.interval
, s
->tcp_keepalive
.timeout
);
3147 connState
->readSomeData();
3149 clientdbEstablished(details
->remote
, 1);
3152 fd_table
[newfd
].clientInfo
= NULL
;
3154 if (Config
.onoff
.client_db
) {
3155 /* it was said several times that client write limiter does not work if client_db is disabled */
3157 ClientDelayPools
& pools(Config
.ClientDelay
.pools
);
3158 for (unsigned int pool
= 0; pool
< pools
.size(); pool
++) {
3160 /* pools require explicit 'allow' to assign a client into them */
3161 if (!pools
[pool
].access
)
3162 continue; // warned in ClientDelayConfig::Finalize()
3164 ACLFilledChecklist
ch(pools
[pool
].access
, NULL
, NULL
);
3166 // TODO: we check early to limit error response bandwith but we
3167 // should recheck when we can honor delay_pool_uses_indirect
3169 ch
.src_addr
= details
->peer
;
3170 ch
.my_addr
= details
->me
;
3172 if (ch
.fastCheck()) {
3174 /* request client information from db after we did all checks
3175 this will save hash lookup if client failed checks */
3176 ClientInfo
* cli
= clientdbGetInfo(details
->peer
);
3179 /* put client info in FDE */
3180 fd_table
[newfd
].clientInfo
= cli
;
3182 /* setup write limiter for this request */
3183 const double burst
= floor(0.5 +
3184 (pools
[pool
].highwatermark
* Config
.ClientDelay
.initial
)/100.0);
3185 cli
->setWriteLimiter(pools
[pool
].rate
, burst
, pools
[pool
].highwatermark
);
3191 incoming_sockets_accepted
++;
3196 /** Create SSL connection structure and update fd_table */
3198 httpsCreate(const Comm::ConnectionPointer
&details
, SSL_CTX
*sslContext
)
3200 SSL
*ssl
= SSL_new(sslContext
);
3203 const int ssl_error
= ERR_get_error();
3204 debugs(83, 1, "httpsAccept: Error allocating handle: " << ERR_error_string(ssl_error
, NULL
) );
3209 SSL_set_fd(ssl
, details
->fd
);
3210 fd_table
[details
->fd
].ssl
= ssl
;
3211 fd_table
[details
->fd
].read_method
= &ssl_read_method
;
3212 fd_table
[details
->fd
].write_method
= &ssl_write_method
;
3214 debugs(33, 5, "httpsCreate: will negotate SSL on " << details
);
3215 fd_note(details
->fd
, "client https start");
3220 /** negotiate an SSL connection */
3222 clientNegotiateSSL(int fd
, void *data
)
3224 ConnStateData
*conn
= (ConnStateData
*)data
;
3226 SSL
*ssl
= fd_table
[fd
].ssl
;
3229 if ((ret
= SSL_accept(ssl
)) <= 0) {
3230 int ssl_error
= SSL_get_error(ssl
, ret
);
3232 switch (ssl_error
) {
3234 case SSL_ERROR_WANT_READ
:
3235 commSetSelect(fd
, COMM_SELECT_READ
, clientNegotiateSSL
, conn
, 0);
3238 case SSL_ERROR_WANT_WRITE
:
3239 commSetSelect(fd
, COMM_SELECT_WRITE
, clientNegotiateSSL
, conn
, 0);
3242 case SSL_ERROR_SYSCALL
:
3245 debugs(83, 2, "clientNegotiateSSL: Error negotiating SSL connection on FD " << fd
<< ": Aborted by client");
3251 if (errno
== ECONNRESET
)
3254 debugs(83, hard
? 1 : 2, "clientNegotiateSSL: Error negotiating SSL connection on FD " <<
3255 fd
<< ": " << strerror(errno
) << " (" << errno
<< ")");
3262 case SSL_ERROR_ZERO_RETURN
:
3263 debugs(83, 1, "clientNegotiateSSL: Error negotiating SSL connection on FD " << fd
<< ": Closed by client");
3268 debugs(83, 1, "clientNegotiateSSL: Error negotiating SSL connection on FD " <<
3269 fd
<< ": " << ERR_error_string(ERR_get_error(), NULL
) <<
3270 " (" << ssl_error
<< "/" << ret
<< ")");
3278 if (SSL_session_reused(ssl
)) {
3279 debugs(83, 2, "clientNegotiateSSL: Session " << SSL_get_session(ssl
) <<
3280 " reused on FD " << fd
<< " (" << fd_table
[fd
].ipaddr
<< ":" << (int)fd_table
[fd
].remote_port
<< ")");
3282 if (do_debug(83, 4)) {
3283 /* Write out the SSL session details.. actually the call below, but
3284 * OpenSSL headers do strange typecasts confusing GCC.. */
3285 /* PEM_write_SSL_SESSION(debug_log, SSL_get_session(ssl)); */
3286 #if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x00908000L
3287 PEM_ASN1_write((i2d_of_void
*)i2d_SSL_SESSION
, PEM_STRING_SSL_SESSION
, debug_log
, (char *)SSL_get_session(ssl
), NULL
,NULL
,0,NULL
,NULL
);
3289 #elif (ALLOW_ALWAYS_SSL_SESSION_DETAIL == 1)
3291 /* When using gcc 3.3.x and OpenSSL 0.9.7x sometimes a compile error can occur here.
3292 * This is caused by an unpredicatble gcc behaviour on a cast of the first argument
3293 * of PEM_ASN1_write(). For this reason this code section is disabled. To enable it,
3294 * define ALLOW_ALWAYS_SSL_SESSION_DETAIL=1.
3295 * Because there are two possible usable cast, if you get an error here, try the other
3296 * commented line. */
3298 PEM_ASN1_write((int(*)())i2d_SSL_SESSION
, PEM_STRING_SSL_SESSION
, debug_log
, (char *)SSL_get_session(ssl
), NULL
,NULL
,0,NULL
,NULL
);
3299 /* PEM_ASN1_write((int(*)(...))i2d_SSL_SESSION, PEM_STRING_SSL_SESSION, debug_log, (char *)SSL_get_session(ssl), NULL,NULL,0,NULL,NULL); */
3303 debugs(83, 4, "With " OPENSSL_VERSION_TEXT
", session details are available only defining ALLOW_ALWAYS_SSL_SESSION_DETAIL=1 in the source." );
3306 /* Note: This does not automatically fflush the log file.. */
3309 debugs(83, 2, "clientNegotiateSSL: New session " <<
3310 SSL_get_session(ssl
) << " on FD " << fd
<< " (" <<
3311 fd_table
[fd
].ipaddr
<< ":" << (int)fd_table
[fd
].remote_port
<<
3315 debugs(83, 3, "clientNegotiateSSL: FD " << fd
<< " negotiated cipher " <<
3316 SSL_get_cipher(ssl
));
3318 client_cert
= SSL_get_peer_certificate(ssl
);
3320 if (client_cert
!= NULL
) {
3321 debugs(83, 3, "clientNegotiateSSL: FD " << fd
<<
3322 " client certificate: subject: " <<
3323 X509_NAME_oneline(X509_get_subject_name(client_cert
), 0, 0));
3325 debugs(83, 3, "clientNegotiateSSL: FD " << fd
<<
3326 " client certificate: issuer: " <<
3327 X509_NAME_oneline(X509_get_issuer_name(client_cert
), 0, 0));
3330 X509_free(client_cert
);
3332 debugs(83, 5, "clientNegotiateSSL: FD " << fd
<<
3333 " has no certificate.");
3336 conn
->readSomeData();
3339 /** handle a new HTTPS connection */
3341 httpsAccept(int sock
, const Comm::ConnectionPointer
& details
, comm_err_t flag
, int xerrno
, void *data
)
3343 https_port_list
*s
= (https_port_list
*)data
;
3344 SSL_CTX
*sslContext
= s
->sslContext
;
3346 assert(flag
!= COMM_OK
); // Acceptor does not call un unless successful.
3349 if (!(ssl
= httpsCreate(details
, sslContext
)))
3352 debugs(33, 5, HERE
<< details
<< " accepted, starting SSL negotiation.");
3353 fd_note(details
->fd
, "client https connect");
3354 ConnStateData
*connState
= connStateCreate(details
, &s
->http
);
3355 typedef CommCbMemFunT
<ConnStateData
, CommCloseCbParams
> Dialer
;
3356 AsyncCall::Pointer call
= JobCallback(33, 5, Dialer
, connState
, ConnStateData::connStateClosed
);
3357 comm_add_close_handler(details
->fd
, call
);
3359 if (Config
.onoff
.log_fqdn
)
3360 fqdncache_gethostbyaddr(details
->remote
, FQDN_LOOKUP_IF_MISS
);
3362 typedef CommCbMemFunT
<ConnStateData
, CommTimeoutCbParams
> TimeoutDialer
;
3363 AsyncCall::Pointer timeoutCall
= JobCallback(33, 5,
3364 TimeoutDialer
, connState
, ConnStateData::requestTimeout
);
3365 commSetTimeout(details
->fd
, Config
.Timeout
.request
, timeoutCall
);
3368 if (Ident::TheConfig
.identLookup
) {
3369 ACLFilledChecklist
identChecklist(Ident::TheConfig
.identLookup
, NULL
, NULL
);
3370 identChecklist
.src_addr
= details
->remote
;
3371 identChecklist
.my_addr
= details
->local
;
3372 if (identChecklist
.fastCheck())
3373 Ident::Start(details
, clientIdentDone
, connState
);
3377 if (s
->http
.tcp_keepalive
.enabled
) {
3378 commSetTcpKeepalive(details
->fd
, s
->http
.tcp_keepalive
.idle
, s
->http
.tcp_keepalive
.interval
, s
->http
.tcp_keepalive
.timeout
);
3381 commSetSelect(details
->fd
, COMM_SELECT_READ
, clientNegotiateSSL
, connState
, 0);
3383 clientdbEstablished(details
->remote
, 1);
3385 incoming_sockets_accepted
++;
3389 ConnStateData::switchToHttps()
3391 assert(!switchedToHttps_
);
3393 //HTTPMSGLOCK(currentobject->http->request);
3394 assert(areAllContextsForThisConnection());
3396 //currentobject->connIsFinished();
3398 debugs(33, 5, HERE
<< "converting " << clientConn
<< " to SSL");
3400 #if 0 // use the actual clientConn now that we have it.
3401 // fake a Comm::Connection object; XXX: make ConnState a Comm::Connection?
3402 Comm::Connection detail
;
3404 detail
.remote
= peer
;
3407 SSL_CTX
*sslContext
= port
->sslContext
;
3409 if (!(ssl
= httpsCreate(clientConn
, sslContext
)))
3412 // commSetTimeout() was called for this request before we switched.
3414 // Disable the client read handler until peer selection is complete
3415 commSetSelect(clientConn
->fd
, COMM_SELECT_READ
, NULL
, NULL
, 0);
3417 commSetSelect(clientConn
->fd
, COMM_SELECT_READ
, clientNegotiateSSL
, this, 0);
3419 switchedToHttps_
= true;
3423 #endif /* USE_SSL */
3425 /// check FD after clientHttp[s]ConnectionOpened, adjust HttpSockets as needed
3427 OpenedHttpSocket(const Comm::ConnectionPointer
&clientConn
, const char *msgIfFail
)
3429 if (!Comm::IsConnOpen(clientConn
)) {
3430 Must(NHttpSockets
> 0); // we tried to open some
3431 --NHttpSockets
; // there will be fewer sockets than planned
3432 Must(HttpSockets
[NHttpSockets
] < 0); // no extra fds received
3434 if (!NHttpSockets
) // we could not open any listen sockets at all
3442 /// find any unused HttpSockets[] slot and store fd there or return false
3444 AddOpenedHttpSocket(const Comm::ConnectionPointer
&conn
)
3447 for (int i
= 0; i
< NHttpSockets
&& !found
; i
++) {
3448 if ((found
= HttpSockets
[i
] < 0))
3449 HttpSockets
[i
] = conn
->fd
;
3455 clientHttpConnectionsOpen(void)
3457 http_port_list
*s
= NULL
;
3459 int bumpCount
= 0; // counts http_ports with sslBump option
3462 for (s
= Config
.Sockaddr
.http
; s
; s
= s
->next
) {
3463 if (MAXHTTPPORTS
== NHttpSockets
) {
3464 debugs(1, 1, "WARNING: You have too many 'http_port' lines.");
3465 debugs(1, 1, " The limit is " << MAXHTTPPORTS
);
3470 if (s
->sslBump
&& s
->sslContext
== NULL
) {
3471 debugs(1, 1, "Will not bump SSL at http_port " <<
3472 s
->http
.s
<< " due to SSL initialization failure.");
3479 // NOTE: would the design here be better if we opened both the ConnAcceptor and IPC informative messages now?
3480 // that way we have at least one worker listening on the socket immediately with others joining in as
3481 // they receive the IPC message.
3483 // Fill out a Comm::Connection which IPC will open as a listener for us
3484 // then pass back so we can start a ConnAcceptor subscription.
3485 s
->listenConn
= new Comm::Connection
;
3486 s
->listenConn
->local
= s
->s
;
3487 s
->listenConn
->flags
= COMM_NONBLOCKING
| (s
->spoof_client_ip
? COMM_TRANSPARENT
: 0);
3489 // setup the subscriptions such that new connections accepted by listenConn are handled by HTTP
3490 typedef CommCbFunPtrCallT
<CommAcceptCbPtrFun
> AcceptCall
;
3491 RefCount
<AcceptCall
> subCall
= commCbCall(5, 5, "httpAccept", CommAcceptCbPtrFun(httpAccept
, s
));
3492 Subscription::Pointer sub
= new CallSubscription
<AcceptCall
>(subCall
);
3494 AsyncCall::Pointer listenCall
= asyncCall(33,2, "clientListenerConnectionOpened",
3495 ListeningStartedDialer(&clientListenerConnectionOpened
, s
, false));
3496 Ipc::StartListening(SOCK_STREAM
, IPPROTO_TCP
, s
->listenConn
, Ipc::fdnHttpSocket
, listenCall
, sub
);
3498 HttpSockets
[NHttpSockets
++] = -1; // set in clientListenerHttpConnectionOpened
3502 if (bumpCount
&& !Config
.accessList
.ssl_bump
)
3503 debugs(33, 1, "WARNING: http_port(s) with SslBump found, but no " <<
3504 std::endl
<< "\tssl_bump ACL configured. No requests will be " <<
3511 clientHttpsConnectionsOpen(void)
3513 // XXX: de-dupe clientHttpConnectionsOpened and clientHttpsConnectionsOpened
3517 for (s
= Config
.Sockaddr
.https
; s
; s
= (https_port_list
*)s
->http
.next
) {
3518 if (MAXHTTPPORTS
== NHttpSockets
) {
3519 debugs(1, 1, "Ignoring 'https_port' lines exceeding the limit.");
3520 debugs(1, 1, "The limit is " << MAXHTTPPORTS
<< " HTTPS ports.");
3524 if (s
->sslContext
== NULL
) {
3525 debugs(1, 1, "Ignoring https_port " << s
->http
.s
<<
3526 " due to SSL initialization failure.");
3530 // Fill out a Comm::Connection which IPC will open as a listener for us
3531 s
->http
.listenConn
= new Comm::Connection
;
3532 s
->http
.listenConn
->local
= s
->http
.s
;
3533 s
->http
.listenConn
->flags
= COMM_NONBLOCKING
| (s
->http
.spoof_client_ip
? COMM_TRANSPARENT
: 0);
3535 // setup the subscriptions such that new connections accepted by listenConn are handled by HTTPS
3536 typedef CommCbFunPtrCallT
<CommAcceptCbPtrFun
> AcceptCall
;
3537 RefCount
<AcceptCall
> subCall
= commCbCall(5, 5, "httpsAccept", CommAcceptCbPtrFun(httpsAccept
, s
));
3538 Subscription::Pointer sub
= new CallSubscription
<AcceptCall
>(subCall
);
3540 AsyncCall::Pointer listenCall
= asyncCall(33, 2, "clientListenerConnectionOpened",
3541 ListeningStartedDialer(&clientListenerConnectionOpened
, &s
->http
, true));
3543 Ipc::StartListening(SOCK_STREAM
, IPPROTO_TCP
, s
->listenConn
, Ipc::fdnHttpsSocket
, listenCall
, sub
);
3545 HttpSockets
[NHttpSockets
++] = -1;
3550 /// process clientHttpConnectionsOpen result
3552 clientListenerConnectionOpened(int, http_port_list
*s
, bool uses_ssl
)
3554 if (!OpenedHttpSocket(s
->listenConn
, (uses_ssl
?"Cannot open HTTP Port":"Cannot open HTTPS Port")))
3558 Must(Comm::IsConnOpen(s
->listenConn
));
3560 debugs(1, 1, "Accepting" <<
3561 (s
->intercepted
? " intercepted" : "") <<
3562 (s
->spoof_client_ip
? " spoofing" : "") <<
3563 (s
->sslBump
? " bumpy" : "") <<
3564 (s
->accel
? " accelerated" : "")
3565 << " HTTP" << (uses_ssl
?"S":"") << " connections at " << s
->listenConn
<< ".");
3567 Must(AddOpenedHttpSocket(s
->listenConn
)); // otherwise, we have received a fd we did not ask for
3571 clientOpenListenSockets(void)
3573 clientHttpConnectionsOpen();
3575 clientHttpsConnectionsOpen();
3578 if (NHttpSockets
< 1)
3579 fatal("No HTTP or HTTPS ports configured");
3583 clientHttpConnectionsClose(void)
3585 for (http_port_list
*s
= Config
.Sockaddr
.http
; s
; s
= s
->next
) {
3586 if (s
->listenConn
!= NULL
) {
3587 debugs(1, 1, "Closing HTTP port " << s
->listenConn
->local
);
3588 s
->listenConn
->close();
3589 s
->listenConn
= NULL
;
3594 for (http_port_list
*s
= Config
.Sockaddr
.https
; s
; s
= s
->next
) {
3595 if (s
->listenConn
!= NULL
) {
3596 debugs(1, 1, "Closing HTTPS port " << s
->listenConn
->local
);
3597 s
->listenConn
->close();
3598 s
->listenConn
= NULL
;
3603 // TODO see if we can drop HttpSockets array entirely */
3604 for (int i
= 0; i
< NHttpSockets
; i
++) {
3605 HttpSockets
[i
] = -1;
3612 varyEvaluateMatch(StoreEntry
* entry
, HttpRequest
* request
)
3614 const char *vary
= request
->vary_headers
;
3615 int has_vary
= entry
->getReply()->header
.has(HDR_VARY
);
3616 #if X_ACCELERATOR_VARY
3619 entry
->getReply()->header
.has(HDR_X_ACCELERATOR_VARY
);
3622 if (!has_vary
|| !entry
->mem_obj
->vary_headers
) {
3624 /* Oops... something odd is going on here.. */
3625 debugs(33, 1, "varyEvaluateMatch: Oops. Not a Vary object on second attempt, '" <<
3626 entry
->mem_obj
->url
<< "' '" << vary
<< "'");
3627 safe_free(request
->vary_headers
);
3632 /* This is not a varying object */
3636 /* virtual "vary" object found. Calculate the vary key and
3637 * continue the search
3639 vary
= httpMakeVaryMark(request
, entry
->getReply());
3642 request
->vary_headers
= xstrdup(vary
);
3645 /* Ouch.. we cannot handle this kind of variance */
3646 /* XXX This cannot really happen, but just to be complete */
3651 vary
= httpMakeVaryMark(request
, entry
->getReply());
3654 request
->vary_headers
= xstrdup(vary
);
3658 /* Ouch.. we cannot handle this kind of variance */
3659 /* XXX This cannot really happen, but just to be complete */
3661 } else if (strcmp(vary
, entry
->mem_obj
->vary_headers
) == 0) {
3664 /* Oops.. we have already been here and still haven't
3665 * found the requested variant. Bail out
3667 debugs(33, 1, "varyEvaluateMatch: Oops. Not a Vary match on second attempt, '" <<
3668 entry
->mem_obj
->url
<< "' '" << vary
<< "'");
3674 ACLFilledChecklist
*
3675 clientAclChecklistCreate(const acl_access
* acl
, ClientHttpRequest
* http
)
3677 ConnStateData
* conn
= http
->getConn();
3678 ACLFilledChecklist
*ch
= new ACLFilledChecklist(acl
, http
->request
,
3679 cbdataReferenceValid(conn
) && conn
!= NULL
? conn
->rfc931
: dash_str
);
3682 * hack for ident ACL. It needs to get full addresses, and a place to store
3683 * the ident result on persistent connections...
3685 /* connection oriented auth also needs these two lines for it's operation. */
3687 * Internal requests do not have a connection reference, because: A) their
3688 * byte count may be transformed before being applied to an outbound
3689 * connection B) they are internal - any limiting on them should be done on
3694 ch
->conn(conn
); /* unreferenced in FilledCheckList.cc */
3699 CBDATA_CLASS_INIT(ConnStateData
);
3701 ConnStateData::ConnStateData() :
3702 AsyncJob("ConnStateData"),
3703 transparent_(false),
3707 pinning
.pinned
= false;
3708 pinning
.auth
= false;
3712 ConnStateData::transparent() const
3714 return transparent_
;
3718 ConnStateData::transparent(bool const anInt
)
3720 transparent_
= anInt
;
3724 ConnStateData::reading() const
3726 return reader
!= NULL
;
3730 ConnStateData::stopReading()
3733 comm_read_cancel(clientConn
->fd
, reader
);
3740 ConnStateData::expectRequestBody(int64_t size
)
3742 bodyPipe
= new BodyPipe(this);
3744 bodyPipe
->setBodySize(size
);
3746 startDechunkingRequest();
3751 ConnStateData::mayNeedToReadMoreBody() const
3754 return 0; // request without a body or read/produced all body bytes
3756 if (!bodyPipe
->bodySizeKnown())
3757 return -1; // probably need to read more, but we cannot be sure
3759 const int64_t needToProduce
= bodyPipe
->unproducedSize();
3760 const int64_t haveAvailable
= static_cast<int64_t>(in
.notYetUsed
);
3762 if (needToProduce
<= haveAvailable
)
3763 return 0; // we have read what we need (but are waiting for pipe space)
3765 return needToProduce
- haveAvailable
;
3769 ConnStateData::closing() const
3775 * Called by ClientSocketContext to give the connection a chance to read
3776 * the entire body before closing the socket.
3779 ConnStateData::startClosing(const char *reason
)
3781 debugs(33, 5, HERE
<< "startClosing " << this << " for " << reason
);
3785 assert(bodyPipe
!= NULL
);
3787 // We do not have to abort the body pipeline because we are going to
3788 // read the entire body anyway.
3789 // Perhaps an ICAP server wants to log the complete request.
3791 // If a consumer abort have caused this closing, we may get stuck
3792 // as nobody is consuming our data. Allow auto-consumption.
3793 bodyPipe
->enableAutoConsumption();
3796 /// initialize dechunking state
3798 ConnStateData::startDechunkingRequest()
3800 Must(bodyPipe
!= NULL
);
3801 debugs(33, 5, HERE
<< "start dechunking" << bodyPipe
->status());
3802 assert(!in
.bodyParser
);
3803 in
.bodyParser
= new ChunkedCodingParser
;
3806 /// put parsed content into input buffer and clean up
3808 ConnStateData::finishDechunkingRequest(bool withSuccess
)
3810 debugs(33, 5, HERE
<< "finish dechunking: " << withSuccess
);
3812 if (bodyPipe
!= NULL
) {
3813 debugs(33, 7, HERE
<< "dechunked tail: " << bodyPipe
->status());
3814 BodyPipe::Pointer myPipe
= bodyPipe
;
3815 stopProducingFor(bodyPipe
, withSuccess
); // sets bodyPipe->bodySize()
3816 Must(!bodyPipe
); // we rely on it being nil after we are done with body
3818 Must(myPipe
->bodySizeKnown());
3819 ClientSocketContext::Pointer context
= getCurrentContext();
3820 if (context
!= NULL
&& context
->http
&& context
->http
->request
)
3821 context
->http
->request
->setContentLength(myPipe
->bodySize());
3825 delete in
.bodyParser
;
3826 in
.bodyParser
= NULL
;
3830 ConnStateData::In::addressToReadInto() const
3832 return buf
+ notYetUsed
;
3835 ConnStateData::In::In() : bodyParser(NULL
),
3836 buf (NULL
), notYetUsed (0), allocatedSize (0)
3839 ConnStateData::In::~In()
3842 memFreeBuf(allocatedSize
, buf
);
3843 delete bodyParser
; // TODO: pool
3847 ConnStateData::sendControlMsg(HttpControlMsg msg
)
3850 debugs(33, 3, HERE
<< "ignoring 1xx due to earlier closure");
3854 ClientSocketContext::Pointer context
= getCurrentContext();
3855 if (context
!= NULL
) {
3856 context
->writeControlMsg(msg
); // will call msg.cbSuccess
3860 debugs(33, 3, HERE
<< " closing due to missing context for 1xx");
3861 clientConn
->close();
3864 /* This is a comm call normally scheduled by comm_close() */
3866 ConnStateData::clientPinnedConnectionClosed(const CommCloseCbParams
&io
)
3870 cbdataReferenceDone(pinning
.peer
);
3872 safe_free(pinning
.host
);
3873 /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host
3874 * connection has gone away */
3877 void ConnStateData::pinConnection(int pinning_fd
, HttpRequest
*request
, struct peer
*aPeer
, bool auth
)
3879 char desc
[FD_DESC_SZ
];
3881 if (pinning
.fd
== pinning_fd
)
3883 else if (pinning
.fd
!= -1)
3884 comm_close(pinning
.fd
);
3887 safe_free(pinning
.host
);
3889 pinning
.fd
= pinning_fd
;
3890 pinning
.host
= xstrdup(request
->GetHost());
3891 pinning
.port
= request
->port
;
3892 pinning
.pinned
= true;
3894 cbdataReferenceDone(pinning
.peer
);
3896 pinning
.peer
= cbdataReference(aPeer
);
3897 pinning
.auth
= auth
;
3898 snprintf(desc
, FD_DESC_SZ
, "%s pinned connection for %s:%d (%d)",
3899 (auth
|| !aPeer
) ? request
->GetHost() : aPeer
->name
, fd_table
[clientConn
->fd
].ipaddr
,
3900 clientConn
->remote
.GetPort(), clientConn
->fd
);
3901 fd_note(pinning_fd
, desc
);
3903 typedef CommCbMemFunT
<ConnStateData
, CommCloseCbParams
> Dialer
;
3904 pinning
.closeHandler
= JobCallback(33, 5,
3905 Dialer
, this, ConnStateData::clientPinnedConnectionClosed
);
3906 comm_add_close_handler(pinning_fd
, pinning
.closeHandler
);
3910 int ConnStateData::validatePinnedConnection(HttpRequest
*request
, const struct peer
*aPeer
)
3916 if (pinning
.auth
&& request
&& strcasecmp(pinning
.host
, request
->GetHost()) != 0) {
3919 if (request
&& pinning
.port
!= request
->port
) {
3922 if (pinning
.peer
&& !cbdataReferenceValid(pinning
.peer
)) {
3925 if (aPeer
!= pinning
.peer
) {
3930 int pinning_fd
=pinning
.fd
;
3931 /* The pinning info is not safe, remove any pinning info*/
3934 /* also close the server side socket, we should not use it for invalid/unauthenticated
3937 comm_close(pinning_fd
);
3944 void ConnStateData::unpinConnection()
3947 cbdataReferenceDone(pinning
.peer
);
3949 if (pinning
.closeHandler
!= NULL
) {
3950 comm_remove_close_handler(pinning
.fd
, pinning
.closeHandler
);
3951 pinning
.closeHandler
= NULL
;
3954 safe_free(pinning
.host
);