2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 93 ICAP (RFC 3507) Client */
12 #include "AccessLogEntry.h"
13 #include "adaptation/Answer.h"
14 #include "adaptation/History.h"
15 #include "adaptation/icap/Client.h"
16 #include "adaptation/icap/Config.h"
17 #include "adaptation/icap/History.h"
18 #include "adaptation/icap/Launcher.h"
19 #include "adaptation/icap/ModXact.h"
20 #include "adaptation/icap/ServiceRep.h"
21 #include "adaptation/Initiator.h"
22 #include "auth/UserRequest.h"
23 #include "base/TextException.h"
25 #include "ChunkedCodingParser.h"
27 #include "comm/Connection.h"
28 #include "err_detail_type.h"
29 #include "HttpHeaderTools.h"
31 #include "HttpReply.h"
32 #include "HttpRequest.h"
33 #include "SquidTime.h"
36 // flow and terminology:
37 // HTTP| --> receive --> encode --> write --> |network
38 // end | <-- send <-- parse <-- read <-- |end
40 // TODO: replace gotEncapsulated() with something faster; we call it often
42 CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap
, ModXact
);
43 CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap
, ModXactLauncher
);
45 static const size_t TheBackupLimit
= BodyPipe::MaxCapacity
;
47 Adaptation::Icap::ModXact::State::State()
49 memset(this, 0, sizeof(*this));
52 Adaptation::Icap::ModXact::ModXact(HttpMsg
*virginHeader
,
53 HttpRequest
*virginCause
, AccessLogEntry::Pointer
&alp
, Adaptation::Icap::ServiceRep::Pointer
&aService
):
54 AsyncJob("Adaptation::Icap::ModXact"),
55 Adaptation::Icap::Xaction("Adaptation::Icap::ModXact", aService
),
58 canStartBypass(false), // too early
59 protectGroupBypass(true),
60 replyHttpHeaderSize(-1),
61 replyHttpBodySize(-1),
67 virgin
.setHeader(virginHeader
); // sets virgin.body_pipe if needed
68 virgin
.setCause(virginCause
); // may be NULL
70 // adapted header and body are initialized when we parse them
72 // writing and reading ends are handled by Adaptation::Icap::Xaction
75 // nothing to do because we are using temporary buffers
77 // parsing; TODO: do not set until we parse, see ICAPOptXact
78 icapReply
= new HttpReply
;
79 icapReply
->protoPrefix
= "ICAP/"; // TODO: make an IcapReply class?
81 debugs(93,7, HERE
<< "initialized." << status());
84 // initiator wants us to start
85 void Adaptation::Icap::ModXact::start()
87 Adaptation::Icap::Xaction::start();
89 // reserve an adaptation history slot (attempts are known at this time)
90 Adaptation::History::Pointer ah
= virginRequest().adaptLogHistory();
92 adaptHistoryId
= ah
->recordXactStart(service().cfg().key
, icap_tr_start
, attempts
> 1);
94 estimateVirginBody(); // before virgin disappears!
96 canStartBypass
= service().cfg().bypass
;
98 // it is an ICAP violation to send request to a service w/o known OPTIONS
99 // and the service may is too busy for us: honor Max-Connections and such
100 if (service().up() && service().availableForNew())
106 void Adaptation::Icap::ModXact::waitForService()
109 Must(!state
.serviceWaiting
);
111 if (!service().up()) {
112 AsyncCall::Pointer call
= JobCallback(93,5,
113 ConnWaiterDialer
, this, Adaptation::Icap::ModXact::noteServiceReady
);
115 service().callWhenReady(call
);
116 comment
= "to be up";
118 //The service is unavailable because of max-connection or other reason
120 if (service().cfg().onOverload
!= srvWait
) {
121 // The service is overloaded, but waiting to be available prohibited by
122 // user configuration (onOverload is set to "block" or "bypass")
123 if (service().cfg().onOverload
== srvBlock
)
124 disableBypass("not available", true);
125 else //if (service().cfg().onOverload == srvBypass)
126 canStartBypass
= true;
129 disableRepeats("ICAP service is not available");
131 debugs(93, 7, HERE
<< "will not wait for the service to be available" <<
134 throw TexcHere("ICAP service is not available");
137 AsyncCall::Pointer call
= JobCallback(93,5,
138 ConnWaiterDialer
, this, Adaptation::Icap::ModXact::noteServiceAvailable
);
139 service().callWhenAvailable(call
, state
.waitedForService
);
140 comment
= "to be available";
143 debugs(93, 7, HERE
<< "will wait for the service " << comment
<< status());
144 state
.serviceWaiting
= true; // after callWhenReady() which may throw
145 state
.waitedForService
= true;
148 void Adaptation::Icap::ModXact::noteServiceReady()
150 Must(state
.serviceWaiting
);
151 state
.serviceWaiting
= false;
153 if (!service().up()) {
155 disableRepeats("ICAP service is unusable");
156 throw TexcHere("ICAP service is unusable");
159 if (service().availableForOld())
165 void Adaptation::Icap::ModXact::noteServiceAvailable()
167 Must(state
.serviceWaiting
);
168 state
.serviceWaiting
= false;
170 if (service().up() && service().availableForOld())
176 void Adaptation::Icap::ModXact::startWriting()
178 state
.writing
= State::writingConnect
;
180 decideOnPreview(); // must be decided before we decideOnRetries
186 // connection with the ICAP service established
187 void Adaptation::Icap::ModXact::handleCommConnected()
189 Must(state
.writing
== State::writingConnect
);
191 startReading(); // wait for early errors from the ICAP server
196 makeRequestHeaders(requestBuf
);
197 debugs(93, 9, HERE
<< "will write" << status() << ":\n" <<
198 (requestBuf
.terminate(), requestBuf
.content()));
201 state
.writing
= State::writingHeaders
;
202 icap_tio_start
= current_time
;
203 scheduleWrite(requestBuf
);
206 void Adaptation::Icap::ModXact::handleCommWrote(size_t sz
)
208 debugs(93, 5, HERE
<< "Wrote " << sz
<< " bytes");
210 if (state
.writing
== State::writingHeaders
)
211 handleCommWroteHeaders();
213 handleCommWroteBody();
216 void Adaptation::Icap::ModXact::handleCommWroteHeaders()
218 Must(state
.writing
== State::writingHeaders
);
220 // determine next step
221 if (preview
.enabled()) {
223 decideWritingAfterPreview("zero-size");
225 state
.writing
= State::writingPreview
;
226 } else if (virginBody
.expected()) {
227 state
.writing
= State::writingPrime
;
236 void Adaptation::Icap::ModXact::writeMore()
238 debugs(93, 5, HERE
<< "checking whether to write more" << status());
240 if (writer
!= NULL
) // already writing something
243 switch (state
.writing
) {
245 case State::writingInit
: // waiting for service OPTIONS
246 Must(state
.serviceWaiting
);
248 case State::writingConnect
: // waiting for the connection to establish
250 case State::writingHeaders
: // waiting for the headers to be written
252 case State::writingPaused
: // waiting for the ICAP server response
254 case State::writingReallyDone
: // nothing more to write
257 case State::writingAlmostDone
: // was waiting for the last write
261 case State::writingPreview
:
265 case State::writingPrime
:
270 throw TexcHere("Adaptation::Icap::ModXact in bad writing state");
274 void Adaptation::Icap::ModXact::writePreviewBody()
276 debugs(93, 8, HERE
<< "will write Preview body from " <<
277 virgin
.body_pipe
<< status());
278 Must(state
.writing
== State::writingPreview
);
279 Must(virgin
.body_pipe
!= NULL
);
281 const size_t sizeMax
= (size_t)virgin
.body_pipe
->buf().contentSize();
282 const size_t size
= min(preview
.debt(), sizeMax
);
283 writeSomeBody("preview body", size
);
285 // change state once preview is written
288 decideWritingAfterPreview("body");
291 /// determine state.writing after we wrote the entire preview
292 void Adaptation::Icap::ModXact::decideWritingAfterPreview(const char *kind
)
294 if (preview
.ieof()) // nothing more to write
296 else if (state
.parsing
== State::psIcapHeader
) // did not get a reply yet
297 state
.writing
= State::writingPaused
; // wait for the ICAP server reply
299 stopWriting(true); // ICAP server reply implies no post-preview writing
301 debugs(93, 6, HERE
<< "decided on writing after " << kind
<< " preview" <<
305 void Adaptation::Icap::ModXact::writePrimeBody()
307 Must(state
.writing
== State::writingPrime
);
308 Must(virginBodyWriting
.active());
310 const size_t size
= (size_t)virgin
.body_pipe
->buf().contentSize();
311 writeSomeBody("prime virgin body", size
);
313 if (virginBodyEndReached(virginBodyWriting
)) {
314 debugs(93, 5, HERE
<< "wrote entire body");
319 void Adaptation::Icap::ModXact::writeSomeBody(const char *label
, size_t size
)
321 Must(!writer
&& state
.writing
< state
.writingAlmostDone
);
322 Must(virgin
.body_pipe
!= NULL
);
323 debugs(93, 8, HERE
<< "will write up to " << size
<< " bytes of " <<
326 MemBuf writeBuf
; // TODO: suggest a min size based on size and lastChunk
328 writeBuf
.init(); // note: we assume that last-chunk will fit
330 const size_t writableSize
= virginContentSize(virginBodyWriting
);
331 const size_t chunkSize
= min(writableSize
, size
);
334 debugs(93, 7, HERE
<< "will write " << chunkSize
<<
335 "-byte chunk of " << label
);
337 openChunk(writeBuf
, chunkSize
, false);
338 writeBuf
.append(virginContentData(virginBodyWriting
), chunkSize
);
339 closeChunk(writeBuf
);
341 virginBodyWriting
.progress(chunkSize
);
344 debugs(93, 7, HERE
<< "has no writable " << label
<< " content");
347 const bool wroteEof
= virginBodyEndReached(virginBodyWriting
);
348 bool lastChunk
= wroteEof
;
349 if (state
.writing
== State::writingPreview
) {
350 preview
.wrote(chunkSize
, wroteEof
); // even if wrote nothing
351 lastChunk
= lastChunk
|| preview
.done();
355 debugs(93, 8, HERE
<< "will write last-chunk of " << label
);
356 addLastRequestChunk(writeBuf
);
359 debugs(93, 7, HERE
<< "will write " << writeBuf
.contentSize()
360 << " raw bytes of " << label
);
362 if (writeBuf
.hasContent()) {
363 scheduleWrite(writeBuf
); // comm will free the chunk
369 void Adaptation::Icap::ModXact::addLastRequestChunk(MemBuf
&buf
)
371 const bool ieof
= state
.writing
== State::writingPreview
&& preview
.ieof();
372 openChunk(buf
, 0, ieof
);
376 void Adaptation::Icap::ModXact::openChunk(MemBuf
&buf
, size_t chunkSize
, bool ieof
)
378 buf
.Printf((ieof
? "%x; ieof\r\n" : "%x\r\n"), (int) chunkSize
);
381 void Adaptation::Icap::ModXact::closeChunk(MemBuf
&buf
)
383 buf
.append(ICAP::crlf
, 2); // chunk-terminating CRLF
386 const HttpRequest
&Adaptation::Icap::ModXact::virginRequest() const
388 const HttpRequest
*request
= virgin
.cause
?
389 virgin
.cause
: dynamic_cast<const HttpRequest
*>(virgin
.header
);
394 // did the activity reached the end of the virgin body?
395 bool Adaptation::Icap::ModXact::virginBodyEndReached(const Adaptation::Icap::VirginBodyAct
&act
) const
398 !act
.active() || // did all (assuming it was originally planned)
399 !virgin
.body_pipe
->expectMoreAfter(act
.offset()); // wont have more
402 // the size of buffered virgin body data available for the specified activity
403 // if this size is zero, we may be done or may be waiting for more data
404 size_t Adaptation::Icap::ModXact::virginContentSize(const Adaptation::Icap::VirginBodyAct
&act
) const
407 // asbolute start of unprocessed data
408 const uint64_t dataStart
= act
.offset();
409 // absolute end of buffered data
410 const uint64_t dataEnd
= virginConsumed
+ virgin
.body_pipe
->buf().contentSize();
411 Must(virginConsumed
<= dataStart
&& dataStart
<= dataEnd
);
412 return static_cast<size_t>(dataEnd
- dataStart
);
415 // pointer to buffered virgin body data available for the specified activity
416 const char *Adaptation::Icap::ModXact::virginContentData(const Adaptation::Icap::VirginBodyAct
&act
) const
419 const uint64_t dataStart
= act
.offset();
420 Must(virginConsumed
<= dataStart
);
421 return virgin
.body_pipe
->buf().content() + static_cast<size_t>(dataStart
-virginConsumed
);
424 void Adaptation::Icap::ModXact::virginConsume()
426 debugs(93, 9, HERE
<< "consumption guards: " << !virgin
.body_pipe
<< isRetriable
<<
427 isRepeatable
<< canStartBypass
<< protectGroupBypass
);
429 if (!virgin
.body_pipe
)
430 return; // nothing to consume
433 return; // do not consume if we may have to retry later
435 BodyPipe
&bp
= *virgin
.body_pipe
;
436 const bool wantToPostpone
= isRepeatable
|| canStartBypass
|| protectGroupBypass
;
438 // Why > 2? HttpState does not use the last bytes in the buffer
439 // because delayAwareRead() is arguably broken. See
440 // HttpStateData::maybeReadVirginBody for more details.
441 if (wantToPostpone
&& bp
.buf().spaceSize() > 2) {
442 // Postponing may increase memory footprint and slow the HTTP side
443 // down. Not postponing may increase the number of ICAP errors
444 // if the ICAP service fails. We may also use "potential" space to
445 // postpone more aggressively. Should the trade-off be configurable?
446 debugs(93, 8, HERE
<< "postponing consumption from " << bp
.status());
450 const size_t have
= static_cast<size_t>(bp
.buf().contentSize());
451 const uint64_t end
= virginConsumed
+ have
;
452 uint64_t offset
= end
;
454 debugs(93, 9, HERE
<< "max virgin consumption offset=" << offset
<<
455 " acts " << virginBodyWriting
.active() << virginBodySending
.active() <<
456 " consumed=" << virginConsumed
<<
457 " from " << virgin
.body_pipe
->status());
459 if (virginBodyWriting
.active())
460 offset
= min(virginBodyWriting
.offset(), offset
);
462 if (virginBodySending
.active())
463 offset
= min(virginBodySending
.offset(), offset
);
465 Must(virginConsumed
<= offset
&& offset
<= end
);
467 if (const size_t size
= static_cast<size_t>(offset
- virginConsumed
)) {
468 debugs(93, 8, HERE
<< "consuming " << size
<< " out of " << have
<<
469 " virgin body bytes");
471 virginConsumed
+= size
;
472 Must(!isRetriable
); // or we should not be consuming
473 disableRepeats("consumed content");
474 disableBypass("consumed content", true);
478 void Adaptation::Icap::ModXact::handleCommWroteBody()
483 // Called when we do not expect to call comm_write anymore.
484 // We may have a pending write though.
485 // If stopping nicely, we will just wait for that pending write, if any.
486 void Adaptation::Icap::ModXact::stopWriting(bool nicely
)
488 if (state
.writing
== State::writingReallyDone
)
491 if (writer
!= NULL
) {
493 debugs(93, 7, HERE
<< "will wait for the last write" << status());
494 state
.writing
= State::writingAlmostDone
; // may already be set
498 debugs(93, 3, HERE
<< "will NOT wait for the last write" << status());
500 // Comm does not have an interface to clear the writer callback nicely,
501 // but without clearing the writer we cannot recycle the connection.
502 // We prevent connection reuse and hope that we can handle a callback
503 // call at any time, usually in the middle of the destruction sequence!
504 // Somebody should add comm_remove_write_handler() to comm API.
505 reuseConnection
= false;
506 ignoreLastWrite
= true;
509 debugs(93, 7, HERE
<< "will no longer write" << status());
510 if (virginBodyWriting
.active()) {
511 virginBodyWriting
.disable();
514 state
.writing
= State::writingReallyDone
;
518 void Adaptation::Icap::ModXact::stopBackup()
520 if (!virginBodySending
.active())
523 debugs(93, 7, HERE
<< "will no longer backup" << status());
524 virginBodySending
.disable();
528 bool Adaptation::Icap::ModXact::doneAll() const
530 return Adaptation::Icap::Xaction::doneAll() && !state
.serviceWaiting
&&
532 doneReading() && state
.doneWriting();
535 void Adaptation::Icap::ModXact::startReading()
537 Must(haveConnection());
539 Must(!adapted
.header
);
540 Must(!adapted
.body_pipe
);
542 // we use the same buffer for headers and body and then consume headers
546 void Adaptation::Icap::ModXact::readMore()
548 if (reader
!= NULL
|| doneReading()) {
549 debugs(93,3,HERE
<< "returning from readMore because reader or doneReading()");
553 // do not fill readBuf if we have no space to store the result
554 if (adapted
.body_pipe
!= NULL
&&
555 !adapted
.body_pipe
->buf().hasPotentialSpace()) {
556 debugs(93,3,HERE
<< "not reading because ICAP reply pipe is full");
560 if (readBuf
.spaceSize())
563 debugs(93,3,HERE
<< "nothing to do because !readBuf.spaceSize()");
566 // comm module read a portion of the ICAP response for us
567 void Adaptation::Icap::ModXact::handleCommRead(size_t)
569 Must(!state
.doneParsing());
570 icap_tio_finish
= current_time
;
575 void Adaptation::Icap::ModXact::echoMore()
577 Must(state
.sending
== State::sendingVirgin
);
578 Must(adapted
.body_pipe
!= NULL
);
579 Must(virginBodySending
.active());
581 const size_t sizeMax
= virginContentSize(virginBodySending
);
582 debugs(93,5, HERE
<< "will echo up to " << sizeMax
<< " bytes from " <<
583 virgin
.body_pipe
->status());
584 debugs(93,5, HERE
<< "will echo up to " << sizeMax
<< " bytes to " <<
585 adapted
.body_pipe
->status());
588 const size_t size
= adapted
.body_pipe
->putMoreData(virginContentData(virginBodySending
), sizeMax
);
589 debugs(93,5, HERE
<< "echoed " << size
<< " out of " << sizeMax
<<
591 virginBodySending
.progress(size
);
592 disableRepeats("echoed content");
593 disableBypass("echoed content", true);
597 if (virginBodyEndReached(virginBodySending
)) {
598 debugs(93, 5, HERE
<< "echoed all" << status());
601 debugs(93, 5, HERE
<< "has " <<
602 virgin
.body_pipe
->buf().contentSize() << " bytes " <<
603 "and expects more to echo" << status());
604 // TODO: timeout if virgin or adapted pipes are broken
608 bool Adaptation::Icap::ModXact::doneSending() const
610 return state
.sending
== State::sendingDone
;
613 // stop (or do not start) sending adapted message body
614 void Adaptation::Icap::ModXact::stopSending(bool nicely
)
616 debugs(93, 7, HERE
<< "Enter stop sending ");
619 debugs(93, 7, HERE
<< "Proceed with stop sending ");
621 if (state
.sending
!= State::sendingUndecided
) {
622 debugs(93, 7, HERE
<< "will no longer send" << status());
623 if (adapted
.body_pipe
!= NULL
) {
624 virginBodySending
.disable();
625 // we may leave debts if we were echoing and the virgin
626 // body_pipe got exhausted before we echoed all planned bytes
627 const bool leftDebts
= adapted
.body_pipe
->needsMoreData();
628 stopProducingFor(adapted
.body_pipe
, nicely
&& !leftDebts
);
631 debugs(93, 7, HERE
<< "will not start sending" << status());
632 Must(!adapted
.body_pipe
);
635 state
.sending
= State::sendingDone
;
639 // should be called after certain state.writing or state.sending changes
640 void Adaptation::Icap::ModXact::checkConsuming()
642 // quit if we already stopped or are still using the pipe
643 if (!virgin
.body_pipe
|| !state
.doneConsumingVirgin())
646 debugs(93, 7, HERE
<< "will stop consuming" << status());
647 stopConsumingFrom(virgin
.body_pipe
);
650 void Adaptation::Icap::ModXact::parseMore()
652 debugs(93, 5, "have " << readBuf
.length() << " bytes to parse" << status());
653 debugs(93, 5, "\n" << readBuf
);
655 if (state
.parsingHeaders())
658 if (state
.parsing
== State::psBody
)
662 void Adaptation::Icap::ModXact::callException(const std::exception
&e
)
664 if (!canStartBypass
|| isRetriable
) {
666 if (const TextException
*te
= dynamic_cast<const TextException
*>(&e
))
667 detailError(ERR_DETAIL_EXCEPTION_START
+ te
->id());
669 detailError(ERR_DETAIL_EXCEPTION_OTHER
);
671 Adaptation::Icap::Xaction::callException(e
);
676 debugs(93, 3, HERE
<< "bypassing " << inCall
<< " exception: " <<
677 e
.what() << ' ' << status());
679 } catch (const TextException
&bypassTe
) {
680 detailError(ERR_DETAIL_EXCEPTION_START
+ bypassTe
.id());
681 Adaptation::Icap::Xaction::callException(bypassTe
);
682 } catch (const std::exception
&bypassE
) {
683 detailError(ERR_DETAIL_EXCEPTION_OTHER
);
684 Adaptation::Icap::Xaction::callException(bypassE
);
688 void Adaptation::Icap::ModXact::bypassFailure()
690 disableBypass("already started to bypass", false);
692 Must(!isRetriable
); // or we should not be bypassing
693 // TODO: should the same be enforced for isRepeatable? Check icap_repeat??
699 // end all activities associated with the ICAP server
703 stopWriting(true); // or should we force it?
704 if (haveConnection()) {
705 reuseConnection
= false; // be conservative
706 cancelRead(); // may not work; and we cannot stop connecting either
708 debugs(93, 7, HERE
<< "Warning: bypass failed to stop I/O" << status());
711 service().noteFailure(); // we are bypassing, but this is still a failure
714 void Adaptation::Icap::ModXact::disableBypass(const char *reason
, bool includingGroupBypass
)
716 if (canStartBypass
) {
717 debugs(93,7, HERE
<< "will never start bypass because " << reason
);
718 canStartBypass
= false;
720 if (protectGroupBypass
&& includingGroupBypass
) {
721 debugs(93,7, HERE
<< "not protecting group bypass because " << reason
);
722 protectGroupBypass
= false;
726 // note that allocation for echoing is done in handle204NoContent()
727 void Adaptation::Icap::ModXact::maybeAllocateHttpMsg()
729 if (adapted
.header
) // already allocated
732 if (gotEncapsulated("res-hdr")) {
733 adapted
.setHeader(new HttpReply
);
734 setOutcome(service().cfg().method
== ICAP::methodReqmod
?
735 xoSatisfied
: xoModified
);
736 } else if (gotEncapsulated("req-hdr")) {
737 adapted
.setHeader(new HttpRequest
);
738 setOutcome(xoModified
);
740 throw TexcHere("Neither res-hdr nor req-hdr in maybeAllocateHttpMsg()");
743 void Adaptation::Icap::ModXact::parseHeaders()
745 Must(state
.parsingHeaders());
747 if (state
.parsing
== State::psIcapHeader
) {
748 debugs(93, 5, HERE
<< "parse ICAP headers");
752 if (state
.parsing
== State::psHttpHeader
) {
753 debugs(93, 5, HERE
<< "parse HTTP headers");
757 if (state
.parsingHeaders()) { // need more data
765 // called after parsing all headers or when bypassing an exception
766 void Adaptation::Icap::ModXact::startSending()
768 disableRepeats("sent headers");
769 disableBypass("sent headers", true);
770 sendAnswer(Answer::Forward(adapted
.header
));
772 if (state
.sending
== State::sendingVirgin
)
776 void Adaptation::Icap::ModXact::parseIcapHead()
778 Must(state
.sending
== State::sendingUndecided
);
780 if (!parseHead(icapReply
.getRaw()))
783 if (httpHeaderHasConnDir(&icapReply
->header
, "close")) {
784 debugs(93, 5, HERE
<< "found connection close");
785 reuseConnection
= false;
788 switch (icapReply
->sline
.status()) {
790 case Http::scContinue
:
795 case Http::scCreated
: // Symantec Scan Engine 5.0 and later when modifying HTTP msg
797 if (!validate200Ok()) {
798 throw TexcHere("Invalid ICAP Response");
805 case Http::scNoContent
:
806 handle204NoContent();
809 case Http::scPartialContent
:
810 handle206PartialContent();
814 debugs(93, 5, "ICAP status " << icapReply
->sline
.status());
815 handleUnknownScode();
819 const HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
821 request
= &virginRequest();
823 // update the cross-transactional database if needed (all status codes!)
824 if (const char *xxName
= Adaptation::Config::masterx_shared_name
) {
825 Adaptation::History::Pointer ah
= request
->adaptHistory(true);
826 if (ah
!= NULL
) { // TODO: reorder checks to avoid creating history
827 const String val
= icapReply
->header
.getByName(xxName
);
828 if (val
.size() > 0) // XXX: HttpHeader lacks empty value detection
829 ah
->updateXxRecord(xxName
, val
);
833 // update the adaptation plan if needed (all status codes!)
834 if (service().cfg().routing
) {
836 if (icapReply
->header
.getList(HDR_X_NEXT_SERVICES
, &services
)) {
837 Adaptation::History::Pointer ah
= request
->adaptHistory(true);
839 ah
->updateNextServices(services
);
841 } // TODO: else warn (occasionally!) if we got HDR_X_NEXT_SERVICES
843 // We need to store received ICAP headers for <icapLastHeader logformat option.
844 // If we already have stored headers from previous ICAP transaction related to this
845 // request, old headers will be replaced with the new one.
847 Adaptation::History::Pointer ah
= request
->adaptLogHistory();
849 ah
->recordMeta(&icapReply
->header
);
851 // handle100Continue() manages state.writing on its own.
852 // Non-100 status means the server needs no postPreview data from us.
853 if (state
.writing
== State::writingPaused
)
857 bool Adaptation::Icap::ModXact::validate200Ok()
859 if (ICAP::methodRespmod
== service().cfg().method
) {
860 if (!gotEncapsulated("res-hdr"))
866 if (ICAP::methodReqmod
== service().cfg().method
) {
867 if (!gotEncapsulated("res-hdr") && !gotEncapsulated("req-hdr"))
876 void Adaptation::Icap::ModXact::handle100Continue()
878 Must(state
.writing
== State::writingPaused
);
879 // server must not respond before the end of preview: we may send ieof
880 Must(preview
.enabled() && preview
.done() && !preview
.ieof());
882 // 100 "Continue" cancels our Preview commitment,
883 // but not commitment to handle 204 or 206 outside Preview
884 if (!state
.allowedPostview204
&& !state
.allowedPostview206
)
887 state
.parsing
= State::psIcapHeader
; // eventually
890 state
.writing
= State::writingPrime
;
895 void Adaptation::Icap::ModXact::handle200Ok()
897 state
.parsing
= State::psHttpHeader
;
898 state
.sending
= State::sendingAdapted
;
903 void Adaptation::Icap::ModXact::handle204NoContent()
909 void Adaptation::Icap::ModXact::handle206PartialContent()
911 if (state
.writing
== State::writingPaused
) {
912 Must(preview
.enabled());
913 Must(state
.allowedPreview206
);
914 debugs(93, 7, HERE
<< "206 inside preview");
916 Must(state
.writing
> State::writingPaused
);
917 Must(state
.allowedPostview206
);
918 debugs(93, 7, HERE
<< "206 outside preview");
920 state
.parsing
= State::psHttpHeader
;
921 state
.sending
= State::sendingAdapted
;
922 state
.readyForUob
= true;
926 // Called when we receive a 204 No Content response and
927 // when we are trying to bypass a service failure.
928 // We actually start sending (echoig or not) in startSending.
929 void Adaptation::Icap::ModXact::prepEchoing()
931 disableRepeats("preparing to echo content");
932 disableBypass("preparing to echo content", true);
935 // We want to clone the HTTP message, but we do not want
936 // to copy some non-HTTP state parts that HttpMsg kids carry in them.
937 // Thus, we cannot use a smart pointer, copy constructor, or equivalent.
938 // Instead, we simply write the HTTP message and "clone" it by parsing.
939 // TODO: use HttpMsg::clone()!
941 HttpMsg
*oldHead
= virgin
.header
;
942 debugs(93, 7, HERE
<< "cloning virgin message " << oldHead
);
946 // write the virgin message into a memory buffer
948 packHead(httpBuf
, oldHead
);
950 // allocate the adapted message and copy metainfo
951 Must(!adapted
.header
);
953 HttpMsg::Pointer newHead
;
954 if (dynamic_cast<const HttpRequest
*>(oldHead
)) {
955 newHead
= new HttpRequest
;
956 } else if (dynamic_cast<const HttpReply
*>(oldHead
)) {
957 newHead
= new HttpReply
;
959 Must(newHead
!= NULL
);
961 newHead
->inheritProperties(oldHead
);
963 adapted
.setHeader(newHead
.getRaw());
966 // parse the buffer back
967 Http::StatusCode error
= Http::scNone
;
969 httpBuf
.terminate(); // HttpMsg::parse requires nil-terminated buffer
970 Must(adapted
.header
->parse(httpBuf
.content(), httpBuf
.contentSize(), true, &error
));
972 if (HttpRequest
*r
= dynamic_cast<HttpRequest
*>(adapted
.header
))
973 urlCanonical(r
); // parse does not set HttpRequest::canonical
975 Must(adapted
.header
->hdr_sz
== httpBuf
.contentSize()); // no leftovers
979 debugs(93, 7, HERE
<< "cloned virgin message " << oldHead
<< " to " <<
982 // setup adapted body pipe if needed
983 if (oldHead
->body_pipe
!= NULL
) {
984 debugs(93, 7, HERE
<< "will echo virgin body from " <<
986 if (!virginBodySending
.active())
987 virginBodySending
.plan(); // will throw if not possible
988 state
.sending
= State::sendingVirgin
;
991 // TODO: optimize: is it possible to just use the oldHead pipe and
992 // remove ICAP from the loop? This echoing is probably a common case!
993 makeAdaptedBodyPipe("echoed virgin response");
994 if (oldHead
->body_pipe
->bodySizeKnown())
995 adapted
.body_pipe
->setBodySize(oldHead
->body_pipe
->bodySize());
996 debugs(93, 7, HERE
<< "will echo virgin body to " <<
999 debugs(93, 7, HERE
<< "no virgin body to echo");
1004 /// Called when we received use-original-body chunk extension in 206 response.
1005 /// We actually start sending (echoing or not) in startSending().
1006 void Adaptation::Icap::ModXact::prepPartialBodyEchoing(uint64_t pos
)
1008 Must(virginBodySending
.active());
1009 Must(virgin
.header
->body_pipe
!= NULL
);
1011 setOutcome(xoPartEcho
);
1013 debugs(93, 7, HERE
<< "will echo virgin body suffix from " <<
1014 virgin
.header
->body_pipe
<< " offset " << pos
);
1016 // check that use-original-body=N does not point beyond buffered data
1017 const uint64_t virginDataEnd
= virginConsumed
+
1018 virgin
.body_pipe
->buf().contentSize();
1019 Must(pos
<= virginDataEnd
);
1020 virginBodySending
.progress(static_cast<size_t>(pos
));
1022 state
.sending
= State::sendingVirgin
;
1025 if (virgin
.header
->body_pipe
->bodySizeKnown())
1026 adapted
.body_pipe
->expectProductionEndAfter(virgin
.header
->body_pipe
->bodySize() - pos
);
1028 debugs(93, 7, HERE
<< "will echo virgin body suffix to " <<
1031 // Start echoing data
1035 void Adaptation::Icap::ModXact::handleUnknownScode()
1039 // TODO: mark connection as "bad"
1041 // Terminate the transaction; we do not know how to handle this response.
1042 throw TexcHere("Unsupported ICAP status code");
1045 void Adaptation::Icap::ModXact::parseHttpHead()
1047 if (gotEncapsulated("res-hdr") || gotEncapsulated("req-hdr")) {
1048 replyHttpHeaderSize
= 0;
1049 maybeAllocateHttpMsg();
1051 if (!parseHead(adapted
.header
))
1052 return; // need more header data
1055 replyHttpHeaderSize
= adapted
.header
->hdr_sz
;
1057 if (dynamic_cast<HttpRequest
*>(adapted
.header
)) {
1058 const HttpRequest
*oldR
= dynamic_cast<const HttpRequest
*>(virgin
.header
);
1060 // TODO: the adapted request did not really originate from the
1061 // client; give proxy admin an option to prevent copying of
1062 // sensitive client information here. See the following thread:
1063 // http://www.squid-cache.org/mail-archive/squid-dev/200703/0040.html
1066 // Maybe adapted.header==NULL if HttpReply and have Http 0.9 ....
1068 adapted
.header
->inheritProperties(virgin
.header
);
1071 decideOnParsingBody();
1074 // parses both HTTP and ICAP headers
1075 bool Adaptation::Icap::ModXact::parseHead(HttpMsg
*head
)
1078 debugs(93, 5, "have " << readBuf
.length() << " head bytes to parse; state: " << state
.parsing
);
1080 Http::StatusCode error
= Http::scNone
;
1081 // XXX: performance regression. c_str() data copies
1082 // XXX: HttpMsg::parse requires a terminated string buffer
1083 const char *tmpBuf
= readBuf
.c_str();
1084 const bool parsed
= head
->parse(tmpBuf
, readBuf
.length(), commEof
, &error
);
1085 Must(parsed
|| !error
); // success or need more data
1087 if (!parsed
) { // need more data
1088 debugs(93, 5, HERE
<< "parse failed, need more data, return false");
1093 if (HttpRequest
*r
= dynamic_cast<HttpRequest
*>(head
))
1094 urlCanonical(r
); // parse does not set HttpRequest::canonical
1096 debugs(93, 5, HERE
<< "parse success, consume " << head
->hdr_sz
<< " bytes, return true");
1097 readBuf
.consume(head
->hdr_sz
);
1101 void Adaptation::Icap::ModXact::decideOnParsingBody()
1103 if (gotEncapsulated("res-body") || gotEncapsulated("req-body")) {
1104 debugs(93, 5, HERE
<< "expecting a body");
1105 state
.parsing
= State::psBody
;
1106 replyHttpBodySize
= 0;
1107 bodyParser
= new ChunkedCodingParser
;
1108 makeAdaptedBodyPipe("adapted response from the ICAP server");
1109 Must(state
.sending
== State::sendingAdapted
);
1111 debugs(93, 5, HERE
<< "not expecting a body");
1117 void Adaptation::Icap::ModXact::parseBody()
1119 Must(state
.parsing
== State::psBody
);
1122 debugs(93, 5, "have " << readBuf
.length() << " body bytes to parse");
1124 // the parser will throw on errors
1125 BodyPipeCheckout
bpc(*adapted
.body_pipe
);
1126 // XXX: performance regression. SBuf-convert (or Parser-convert?) the chunked decoder.
1129 // NP: we must do this instead of pointing encodedData at the SBuf::rawContent
1130 // because chunked decoder uses MemBuf::consume, which shuffles buffer bytes around.
1131 encodedData
.append(readBuf
.rawContent(), readBuf
.length());
1132 const bool parsed
= bodyParser
->parse(&encodedData
, &bpc
.buf
);
1133 // XXX: httpChunkDecoder has consumed from MemBuf.
1134 readBuf
.consume(readBuf
.length() - encodedData
.contentSize());
1137 debugs(93, 5, "have " << readBuf
.length() << " body bytes after parsed all: " << parsed
);
1138 replyHttpBodySize
+= adapted
.body_pipe
->buf().contentSize();
1140 // TODO: expose BodyPipe::putSize() to make this check simpler and clearer
1141 // TODO: do we really need this if we disable when sending headers?
1142 if (adapted
.body_pipe
->buf().contentSize() > 0) { // parsed something sometime
1143 disableRepeats("sent adapted content");
1144 disableBypass("sent adapted content", true);
1148 if (state
.readyForUob
&& bodyParser
->useOriginBody
>= 0) {
1149 prepPartialBodyEchoing(
1150 static_cast<uint64_t>(bodyParser
->useOriginBody
));
1156 stopSending(true); // the parser succeeds only if all parsed data fits
1160 debugs(93,3,HERE
<< this << " needsMoreData = " << bodyParser
->needsMoreData());
1162 if (bodyParser
->needsMoreData()) {
1163 debugs(93,3,HERE
<< this);
1164 Must(mayReadMore());
1168 if (bodyParser
->needsMoreSpace()) {
1169 Must(!doneSending()); // can hope for more space
1170 Must(adapted
.body_pipe
->buf().contentSize() > 0); // paranoid
1171 // TODO: there should be a timeout in case the sink is broken
1172 // or cannot consume partial content (while we need more space)
1176 void Adaptation::Icap::ModXact::stopParsing()
1178 if (state
.parsing
== State::psDone
)
1181 debugs(93, 7, HERE
<< "will no longer parse" << status());
1187 state
.parsing
= State::psDone
;
1190 // HTTP side added virgin body data
1191 void Adaptation::Icap::ModXact::noteMoreBodyDataAvailable(BodyPipe::Pointer
)
1195 if (state
.sending
== State::sendingVirgin
)
1199 // HTTP side sent us all virgin info
1200 void Adaptation::Icap::ModXact::noteBodyProductionEnded(BodyPipe::Pointer
)
1202 Must(virgin
.body_pipe
->productionEnded());
1204 // push writer and sender in case we were waiting for the last-chunk
1207 if (state
.sending
== State::sendingVirgin
)
1211 // body producer aborted, but the initiator may still want to know
1212 // the answer, even though the HTTP message has been truncated
1213 void Adaptation::Icap::ModXact::noteBodyProducerAborted(BodyPipe::Pointer
)
1215 Must(virgin
.body_pipe
->productionEnded());
1217 // push writer and sender in case we were waiting for the last-chunk
1220 if (state
.sending
== State::sendingVirgin
)
1224 // adapted body consumer wants more adapted data and
1225 // possibly freed some buffer space
1226 void Adaptation::Icap::ModXact::noteMoreBodySpaceAvailable(BodyPipe::Pointer
)
1228 if (state
.sending
== State::sendingVirgin
)
1230 else if (state
.sending
== State::sendingAdapted
)
1233 Must(state
.sending
== State::sendingUndecided
);
1236 // adapted body consumer aborted
1237 void Adaptation::Icap::ModXact::noteBodyConsumerAborted(BodyPipe::Pointer
)
1239 detailError(ERR_DETAIL_ICAP_XACT_BODY_CONSUMER_ABORT
);
1240 mustStop("adapted body consumer aborted");
1243 Adaptation::Icap::ModXact::~ModXact()
1249 void Adaptation::Icap::ModXact::swanSong()
1251 debugs(93, 5, HERE
<< "swan sings" << status());
1256 if (theInitiator
.set()) // we have not sent the answer to the initiator
1257 detailError(ERR_DETAIL_ICAP_XACT_OTHER
);
1259 // update adaptation history if start was called and we reserved a slot
1260 Adaptation::History::Pointer ah
= virginRequest().adaptLogHistory();
1261 if (ah
!= NULL
&& adaptHistoryId
>= 0)
1262 ah
->recordXactFinish(adaptHistoryId
);
1264 Adaptation::Icap::Xaction::swanSong();
1267 void prepareLogWithRequestDetails(HttpRequest
*, AccessLogEntry::Pointer
&);
1269 void Adaptation::Icap::ModXact::finalizeLogInfo()
1271 HttpRequest
* request_
= NULL
;
1272 HttpRequest
* adapted_request_
= NULL
;
1273 HttpReply
* reply_
= NULL
;
1274 request_
= (virgin
.cause
? virgin
.cause
: dynamic_cast<HttpRequest
*>(virgin
.header
));
1275 if (!(adapted_request_
= dynamic_cast<HttpRequest
*>(adapted
.header
))) {
1276 adapted_request_
= request_
;
1277 reply_
= dynamic_cast<HttpReply
*>(adapted
.header
);
1280 Adaptation::Icap::History::Pointer h
= (request_
? request_
->icapHistory() : NULL
);
1281 Must(h
!= NULL
); // ICAPXaction::maybeLog calls only if there is a log
1282 al
.icp
.opcode
= ICP_INVALID
;
1283 al
.url
= h
->log_uri
.termedBuf();
1284 const Adaptation::Icap::ServiceRep
&s
= service();
1285 al
.icap
.reqMethod
= s
.cfg().method
;
1287 al
.cache
.caddr
= request_
->client_addr
;
1289 al
.request
= request_
;
1290 HTTPMSGLOCK(al
.request
);
1291 al
.adapted_request
= adapted_request_
;
1292 HTTPMSGLOCK(al
.adapted_request
);
1296 HTTPMSGLOCK(al
.reply
);
1300 if (h
->rfc931
.size())
1301 al
.cache
.rfc931
= h
->rfc931
.termedBuf();
1304 if (h
->ssluser
.size())
1305 al
.cache
.ssluser
= h
->ssluser
.termedBuf();
1307 al
.cache
.code
= h
->logType
;
1308 // XXX: should use icap-specific counters instead ?
1309 al
.http
.clientRequestSz
.payloadData
= h
->req_sz
;
1311 // leave al.icap.bodyBytesRead negative if no body
1312 if (replyHttpHeaderSize
>= 0 || replyHttpBodySize
>= 0) {
1313 const int64_t zero
= 0; // to make max() argument types the same
1314 al
.icap
.bodyBytesRead
=
1315 max(zero
, replyHttpHeaderSize
) + max(zero
, replyHttpBodySize
);
1319 al
.http
.code
= reply_
->sline
.status();
1320 al
.http
.content_type
= reply_
->content_type
.termedBuf();
1321 if (replyHttpBodySize
>= 0) {
1322 // XXX: should use icap-specific counters instead ?
1323 al
.http
.clientReplySz
.payloadData
= replyHttpBodySize
;
1324 al
.http
.clientReplySz
.header
= reply_
->hdr_sz
;
1325 al
.cache
.highOffset
= replyHttpBodySize
;
1327 //don't set al.cache.objectSize because it hasn't exist yet
1333 packerToMemInit(&p
, &mb
);
1335 reply_
->header
.packInto(&p
);
1336 al
.headers
.reply
= xstrdup(mb
.buf
);
1341 prepareLogWithRequestDetails(adapted_request_
, alep
);
1342 Xaction::finalizeLogInfo();
1345 void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf
&buf
)
1347 char ntoabuf
[MAX_IPSTRLEN
];
1349 * XXX These should use HttpHdr interfaces instead of Printfs
1351 const Adaptation::ServiceConfig
&s
= service().cfg();
1352 buf
.Printf("%s " SQUIDSTRINGPH
" ICAP/1.0\r\n", s
.methodStr(), SQUIDSTRINGPRINT(s
.uri
));
1353 buf
.Printf("Host: " SQUIDSTRINGPH
":%d\r\n", SQUIDSTRINGPRINT(s
.host
), s
.port
);
1354 buf
.Printf("Date: %s\r\n", mkrfc1123(squid_curtime
));
1356 if (!TheConfig
.reuse_connections
)
1357 buf
.Printf("Connection: close\r\n");
1359 const HttpRequest
*request
= &virginRequest();
1361 // we must forward "Proxy-Authenticate" and "Proxy-Authorization"
1363 if (virgin
.header
->header
.has(HDR_PROXY_AUTHENTICATE
)) {
1364 String vh
=virgin
.header
->header
.getByName("Proxy-Authenticate");
1365 buf
.Printf("Proxy-Authenticate: " SQUIDSTRINGPH
"\r\n",SQUIDSTRINGPRINT(vh
));
1368 if (virgin
.header
->header
.has(HDR_PROXY_AUTHORIZATION
)) {
1369 String vh
=virgin
.header
->header
.getByName("Proxy-Authorization");
1370 buf
.Printf("Proxy-Authorization: " SQUIDSTRINGPH
"\r\n", SQUIDSTRINGPRINT(vh
));
1371 } else if (request
->extacl_user
.size() > 0 && request
->extacl_passwd
.size() > 0) {
1372 struct base64_encode_ctx ctx
;
1373 base64_encode_init(&ctx
);
1374 uint8_t base64buf
[base64_encode_len(MAX_LOGIN_SZ
)];
1375 size_t resultLen
= base64_encode_update(&ctx
, base64buf
, request
->extacl_user
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_user
.rawBuf()));
1376 resultLen
+= base64_encode_update(&ctx
, base64buf
+resultLen
, 1, reinterpret_cast<const uint8_t*>(":"));
1377 resultLen
+= base64_encode_update(&ctx
, base64buf
+resultLen
, request
->extacl_passwd
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_passwd
.rawBuf()));
1378 resultLen
+= base64_encode_final(&ctx
, base64buf
+resultLen
);
1379 buf
.Printf("Proxy-Authorization: Basic %.*s\r\n", (int)resultLen
, base64buf
);
1382 // share the cross-transactional database records if needed
1383 if (Adaptation::Config::masterx_shared_name
) {
1384 Adaptation::History::Pointer ah
= request
->adaptHistory(false);
1387 if (ah
->getXxRecord(name
, value
)) {
1388 buf
.Printf(SQUIDSTRINGPH
": " SQUIDSTRINGPH
"\r\n",
1389 SQUIDSTRINGPRINT(name
), SQUIDSTRINGPRINT(value
));
1394 buf
.Printf("Encapsulated: ");
1400 // build HTTP request header, if any
1401 ICAP::Method m
= s
.method
;
1403 // to simplify, we could assume that request is always available
1407 urlPath
= request
->urlpath
;
1408 if (ICAP::methodRespmod
== m
)
1409 encapsulateHead(buf
, "req-hdr", httpBuf
, request
);
1410 else if (ICAP::methodReqmod
== m
)
1411 encapsulateHead(buf
, "req-hdr", httpBuf
, virgin
.header
);
1414 if (ICAP::methodRespmod
== m
)
1415 if (const HttpMsg
*prime
= virgin
.header
)
1416 encapsulateHead(buf
, "res-hdr", httpBuf
, prime
);
1418 if (!virginBody
.expected())
1419 buf
.Printf("null-body=%d", (int) httpBuf
.contentSize());
1420 else if (ICAP::methodReqmod
== m
)
1421 buf
.Printf("req-body=%d", (int) httpBuf
.contentSize());
1423 buf
.Printf("res-body=%d", (int) httpBuf
.contentSize());
1425 buf
.append(ICAP::crlf
, 2); // terminate Encapsulated line
1427 if (preview
.enabled()) {
1428 buf
.Printf("Preview: %d\r\n", (int)preview
.ad());
1429 if (!virginBody
.expected()) // there is no body to preview
1430 finishNullOrEmptyBodyPreview(httpBuf
);
1433 makeAllowHeader(buf
);
1435 if (TheConfig
.send_client_ip
&& request
) {
1436 Ip::Address client_addr
;
1437 #if FOLLOW_X_FORWARDED_FOR
1438 if (TheConfig
.use_indirect_client
) {
1439 client_addr
= request
->indirect_client_addr
;
1442 client_addr
= request
->client_addr
;
1443 if (!client_addr
.isAnyAddr() && !client_addr
.isNoAddr())
1444 buf
.Printf("X-Client-IP: %s\r\n", client_addr
.toStr(ntoabuf
,MAX_IPSTRLEN
));
1447 if (TheConfig
.send_username
&& request
)
1448 makeUsernameHeader(request
, buf
);
1450 // Adaptation::Config::metaHeaders
1451 typedef Notes::iterator ACAMLI
;
1452 for (ACAMLI i
= Adaptation::Config::metaHeaders
.begin(); i
!= Adaptation::Config::metaHeaders
.end(); ++i
) {
1453 HttpRequest
*r
= virgin
.cause
?
1454 virgin
.cause
: dynamic_cast<HttpRequest
*>(virgin
.header
);
1457 HttpReply
*reply
= dynamic_cast<HttpReply
*>(virgin
.header
);
1459 if (const char *value
= (*i
)->match(r
, reply
, alMaster
)) {
1460 buf
.Printf("%s: %s\r\n", (*i
)->key
.termedBuf(), value
);
1461 Adaptation::History::Pointer ah
= request
->adaptHistory(false);
1463 if (ah
->metaHeaders
== NULL
)
1464 ah
->metaHeaders
= new NotePairs
;
1465 if (!ah
->metaHeaders
->hasPair((*i
)->key
.termedBuf(), value
))
1466 ah
->metaHeaders
->add((*i
)->key
.termedBuf(), value
);
1471 // fprintf(stderr, "%s\n", buf.content());
1473 buf
.append(ICAP::crlf
, 2); // terminate ICAP header
1475 // fill icapRequest for logging
1476 Must(icapRequest
->parseCharBuf(buf
.content(), buf
.contentSize()));
1478 // start ICAP request body with encapsulated HTTP headers
1479 buf
.append(httpBuf
.content(), httpBuf
.contentSize());
1484 // decides which Allow values to write and updates the request buffer
1485 void Adaptation::Icap::ModXact::makeAllowHeader(MemBuf
&buf
)
1487 const bool allow204in
= preview
.enabled(); // TODO: add shouldAllow204in()
1488 const bool allow204out
= state
.allowedPostview204
= shouldAllow204();
1489 const bool allow206in
= state
.allowedPreview206
= shouldAllow206in();
1490 const bool allow206out
= state
.allowedPostview206
= shouldAllow206out();
1492 debugs(93,9, HERE
<< "Allows: " << allow204in
<< allow204out
<<
1493 allow206in
<< allow206out
);
1495 const bool allow204
= allow204in
|| allow204out
;
1496 const bool allow206
= allow206in
|| allow206out
;
1498 if (!allow204
&& !allow206
)
1499 return; // nothing to do
1501 if (virginBody
.expected()) // if there is a virgin body, plan to send it
1502 virginBodySending
.plan();
1504 // writing Preview:... means we will honor 204 inside preview
1505 // writing Allow/204 means we will honor 204 outside preview
1506 // writing Allow:206 means we will honor 206 inside preview
1507 // writing Allow:204,206 means we will honor 206 outside preview
1508 const char *allowHeader
= NULL
;
1509 if (allow204out
&& allow206
)
1510 allowHeader
= "Allow: 204, 206\r\n";
1511 else if (allow204out
)
1512 allowHeader
= "Allow: 204\r\n";
1514 allowHeader
= "Allow: 206\r\n";
1516 if (allowHeader
) { // may be nil if only allow204in is true
1517 buf
.append(allowHeader
, strlen(allowHeader
));
1518 debugs(93,5, HERE
<< "Will write " << allowHeader
);
1522 void Adaptation::Icap::ModXact::makeUsernameHeader(const HttpRequest
*request
, MemBuf
&buf
)
1525 struct base64_encode_ctx ctx
;
1526 base64_encode_init(&ctx
);
1528 const char *value
= NULL
;
1529 if (request
->auth_user_request
!= NULL
) {
1530 value
= request
->auth_user_request
->username();
1531 } else if (request
->extacl_user
.size() > 0) {
1532 value
= request
->extacl_user
.termedBuf();
1536 if (TheConfig
.client_username_encode
) {
1537 uint8_t base64buf
[base64_encode_len(MAX_LOGIN_SZ
)];
1538 size_t resultLen
= base64_encode_update(&ctx
, base64buf
, strlen(value
), reinterpret_cast<const uint8_t*>(value
));
1539 resultLen
+= base64_encode_final(&ctx
, base64buf
+resultLen
);
1540 buf
.Printf("%s: %.*s\r\n", TheConfig
.client_username_header
, (int)resultLen
, base64buf
);
1542 buf
.Printf("%s: %s\r\n", TheConfig
.client_username_header
, value
);
1547 void Adaptation::Icap::ModXact::encapsulateHead(MemBuf
&icapBuf
, const char *section
, MemBuf
&httpBuf
, const HttpMsg
*head
)
1549 // update ICAP header
1550 icapBuf
.Printf("%s=%d, ", section
, (int) httpBuf
.contentSize());
1553 HttpMsg::Pointer headClone
;
1555 if (const HttpRequest
* old_request
= dynamic_cast<const HttpRequest
*>(head
)) {
1556 HttpRequest::Pointer
new_request(new HttpRequest
);
1557 Must(old_request
->canonical
);
1558 urlParse(old_request
->method
, old_request
->canonical
, new_request
.getRaw());
1559 new_request
->http_ver
= old_request
->http_ver
;
1560 headClone
= new_request
.getRaw();
1561 } else if (const HttpReply
*old_reply
= dynamic_cast<const HttpReply
*>(head
)) {
1562 HttpReply::Pointer
new_reply(new HttpReply
);
1563 new_reply
->sline
= old_reply
->sline
;
1564 headClone
= new_reply
.getRaw();
1566 Must(headClone
!= NULL
);
1567 headClone
->inheritProperties(head
);
1569 HttpHeaderPos pos
= HttpHeaderInitPos
;
1570 HttpHeaderEntry
* p_head_entry
= NULL
;
1571 while (NULL
!= (p_head_entry
= head
->header
.getEntry(&pos
)) )
1572 headClone
->header
.addEntry(p_head_entry
->clone());
1576 // remove all hop-by-hop headers from the clone
1577 headClone
->header
.delById(HDR_PROXY_AUTHENTICATE
);
1578 headClone
->header
.removeHopByHopEntries();
1580 // pack polished HTTP header
1581 packHead(httpBuf
, headClone
.getRaw());
1583 // headClone unlocks and, hence, deletes the message we packed
1586 void Adaptation::Icap::ModXact::packHead(MemBuf
&httpBuf
, const HttpMsg
*head
)
1589 packerToMemInit(&p
, &httpBuf
);
1590 head
->packInto(&p
, true);
1594 // decides whether to offer a preview and calculates its size
1595 void Adaptation::Icap::ModXact::decideOnPreview()
1597 if (!TheConfig
.preview_enable
) {
1598 debugs(93, 5, HERE
<< "preview disabled by squid.conf");
1602 const String urlPath
= virginRequest().urlpath
;
1604 if (!service().wantsPreview(urlPath
, wantedSize
)) {
1605 debugs(93, 5, HERE
<< "should not offer preview for " << urlPath
);
1609 // we decided to do preview, now compute its size
1611 // cannot preview more than we can backup
1612 size_t ad
= min(wantedSize
, TheBackupLimit
);
1614 if (!virginBody
.expected())
1616 else if (virginBody
.knownSize())
1617 ad
= min(static_cast<uint64_t>(ad
), virginBody
.size()); // not more than we have
1619 debugs(93, 5, HERE
<< "should offer " << ad
<< "-byte preview " <<
1620 "(service wanted " << wantedSize
<< ")");
1623 Must(preview
.enabled());
1626 // decides whether to allow 204 responses
1627 bool Adaptation::Icap::ModXact::shouldAllow204()
1629 if (!service().allows204())
1632 return canBackupEverything();
1635 // decides whether to allow 206 responses in some mode
1636 bool Adaptation::Icap::ModXact::shouldAllow206any()
1638 return TheConfig
.allow206_enable
&& service().allows206() &&
1639 virginBody
.expected(); // no need for 206 without a body
1642 // decides whether to allow 206 responses in preview mode
1643 bool Adaptation::Icap::ModXact::shouldAllow206in()
1645 return shouldAllow206any() && preview
.enabled();
1648 // decides whether to allow 206 responses outside of preview
1649 bool Adaptation::Icap::ModXact::shouldAllow206out()
1651 return shouldAllow206any() && canBackupEverything();
1654 // used by shouldAllow204 and decideOnRetries
1655 bool Adaptation::Icap::ModXact::canBackupEverything() const
1657 if (!virginBody
.expected())
1658 return true; // no body means no problems with backup
1660 // if there is a body, check whether we can backup it all
1662 if (!virginBody
.knownSize())
1665 // or should we have a different backup limit?
1666 // note that '<' allows for 0-termination of the "full" backup buffer
1667 return virginBody
.size() < TheBackupLimit
;
1670 // Decide whether this transaction can be retried if pconn fails
1671 // Must be called after decideOnPreview and before openConnection()
1672 void Adaptation::Icap::ModXact::decideOnRetries()
1675 return; // no, already decided
1677 if (preview
.enabled())
1678 return; // yes, because preview provides enough guarantees
1680 if (canBackupEverything())
1681 return; // yes, because we can back everything up
1683 disableRetries(); // no, because we cannot back everything up
1686 // Normally, the body-writing code handles preview body. It can deal with
1687 // bodies of unexpected size, including those that turn out to be empty.
1688 // However, that code assumes that the body was expected and body control
1689 // structures were initialized. This is not the case when there is no body
1690 // or the body is known to be empty, because the virgin message will lack a
1691 // body_pipe. So we handle preview of null-body and zero-size bodies here.
1692 void Adaptation::Icap::ModXact::finishNullOrEmptyBodyPreview(MemBuf
&)
1694 Must(!virginBodyWriting
.active()); // one reason we handle it here
1695 Must(!virgin
.body_pipe
); // another reason we handle it here
1696 Must(!preview
.ad());
1698 // do not add last-chunk because our Encapsulated header says null-body
1699 // addLastRequestChunk(buf);
1700 preview
.wrote(0, true);
1702 Must(preview
.done());
1703 Must(preview
.ieof());
1706 void Adaptation::Icap::ModXact::fillPendingStatus(MemBuf
&buf
) const
1708 Adaptation::Icap::Xaction::fillPendingStatus(buf
);
1710 if (state
.serviceWaiting
)
1713 if (virgin
.body_pipe
!= NULL
)
1716 if (haveConnection() && !doneReading())
1719 if (!state
.doneWriting() && state
.writing
!= State::writingInit
)
1720 buf
.Printf("w(%d)", state
.writing
);
1722 if (preview
.enabled()) {
1723 if (!preview
.done())
1724 buf
.Printf("P(%d)", (int) preview
.debt());
1727 if (virginBodySending
.active())
1730 if (!state
.doneParsing() && state
.parsing
!= State::psIcapHeader
)
1731 buf
.Printf("p(%d)", state
.parsing
);
1733 if (!doneSending() && state
.sending
!= State::sendingUndecided
)
1734 buf
.Printf("S(%d)", state
.sending
);
1736 if (state
.readyForUob
)
1742 if (protectGroupBypass
)
1746 void Adaptation::Icap::ModXact::fillDoneStatus(MemBuf
&buf
) const
1748 Adaptation::Icap::Xaction::fillDoneStatus(buf
);
1750 if (!virgin
.body_pipe
)
1753 if (state
.doneWriting())
1756 if (preview
.enabled()) {
1758 buf
.Printf("P%s", preview
.ieof() ? "(ieof)" : "");
1764 if (state
.doneParsing())
1771 bool Adaptation::Icap::ModXact::gotEncapsulated(const char *section
) const
1773 return icapReply
->header
.getByNameListMember("Encapsulated",
1774 section
, ',').size() > 0;
1777 // calculate whether there is a virgin HTTP body and
1778 // whether its expected size is known
1779 // TODO: rename because we do not just estimate
1780 void Adaptation::Icap::ModXact::estimateVirginBody()
1782 // note: lack of size info may disable previews and 204s
1784 HttpMsg
*msg
= virgin
.header
;
1787 HttpRequestMethod method
;
1790 method
= virgin
.cause
->method
;
1791 else if (HttpRequest
*req
= dynamic_cast<HttpRequest
*>(msg
))
1792 method
= req
->method
;
1794 method
= Http::METHOD_NONE
;
1797 // expectingBody returns true for zero-sized bodies, but we will not
1798 // get a pipe for that body, so we treat the message as bodyless
1799 if (method
!= Http::METHOD_NONE
&& msg
->expectingBody(method
, size
) && size
) {
1800 debugs(93, 6, HERE
<< "expects virgin body from " <<
1801 virgin
.body_pipe
<< "; size: " << size
);
1803 virginBody
.expect(size
);
1804 virginBodyWriting
.plan();
1806 // sign up as a body consumer
1807 Must(msg
->body_pipe
!= NULL
);
1808 Must(msg
->body_pipe
== virgin
.body_pipe
);
1809 Must(virgin
.body_pipe
->setConsumerIfNotLate(this));
1811 // make sure TheBackupLimit is in-sync with the buffer size
1812 Must(TheBackupLimit
<= static_cast<size_t>(msg
->body_pipe
->buf().max_capacity
));
1814 debugs(93, 6, HERE
<< "does not expect virgin body");
1815 Must(msg
->body_pipe
== NULL
);
1820 void Adaptation::Icap::ModXact::makeAdaptedBodyPipe(const char *what
)
1822 Must(!adapted
.body_pipe
);
1823 Must(!adapted
.header
->body_pipe
);
1824 adapted
.header
->body_pipe
= new BodyPipe(this);
1825 adapted
.body_pipe
= adapted
.header
->body_pipe
;
1826 debugs(93, 7, HERE
<< "will supply " << what
<< " via " <<
1827 adapted
.body_pipe
<< " pipe");
1830 // TODO: Move SizedEstimate and Preview elsewhere
1832 Adaptation::Icap::SizedEstimate::SizedEstimate()
1833 : theData(dtUnexpected
)
1836 void Adaptation::Icap::SizedEstimate::expect(int64_t aSize
)
1838 theData
= (aSize
>= 0) ? aSize
: (int64_t)dtUnknown
;
1841 bool Adaptation::Icap::SizedEstimate::expected() const
1843 return theData
!= dtUnexpected
;
1846 bool Adaptation::Icap::SizedEstimate::knownSize() const
1849 return theData
!= dtUnknown
;
1852 uint64_t Adaptation::Icap::SizedEstimate::size() const
1855 return static_cast<uint64_t>(theData
);
1858 Adaptation::Icap::VirginBodyAct::VirginBodyAct(): theStart(0), theState(stUndecided
)
1861 void Adaptation::Icap::VirginBodyAct::plan()
1864 Must(!theStart
); // not started
1865 theState
= stActive
;
1868 void Adaptation::Icap::VirginBodyAct::disable()
1870 theState
= stDisabled
;
1873 void Adaptation::Icap::VirginBodyAct::progress(size_t size
)
1876 #if SIZEOF_SIZE_T > 4
1877 /* always true for smaller size_t's */
1878 Must(static_cast<int64_t>(size
) >= 0);
1880 theStart
+= static_cast<int64_t>(size
);
1883 uint64_t Adaptation::Icap::VirginBodyAct::offset() const
1886 return static_cast<uint64_t>(theStart
);
1889 Adaptation::Icap::Preview::Preview(): theWritten(0), theAd(0), theState(stDisabled
)
1892 void Adaptation::Icap::Preview::enable(size_t anAd
)
1894 // TODO: check for anAd not exceeding preview size limit
1897 theState
= stWriting
;
1900 bool Adaptation::Icap::Preview::enabled() const
1902 return theState
!= stDisabled
;
1905 size_t Adaptation::Icap::Preview::ad() const
1911 bool Adaptation::Icap::Preview::done() const
1914 return theState
>= stIeof
;
1917 bool Adaptation::Icap::Preview::ieof() const
1920 return theState
== stIeof
;
1923 size_t Adaptation::Icap::Preview::debt() const
1926 return done() ? 0 : (theAd
- theWritten
);
1929 void Adaptation::Icap::Preview::wrote(size_t size
, bool wroteEof
)
1935 Must(theWritten
<= theAd
);
1938 theState
= stIeof
; // written size is irrelevant
1939 else if (theWritten
>= theAd
)
1943 bool Adaptation::Icap::ModXact::fillVirginHttpHeader(MemBuf
&mb
) const
1945 if (virgin
.header
== NULL
)
1948 virgin
.header
->firstLineBuf(mb
);
1953 void Adaptation::Icap::ModXact::detailError(int errDetail
)
1955 HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
1956 // if no adapted request, update virgin (and inherit its properties later)
1957 // TODO: make this and HttpRequest::detailError constant, like adaptHistory
1959 request
= const_cast<HttpRequest
*>(&virginRequest());
1962 request
->detailError(ERR_ICAP_FAILURE
, errDetail
);
1965 void Adaptation::Icap::ModXact::clearError()
1967 HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
1968 // if no adapted request, update virgin (and inherit its properties later)
1970 request
= const_cast<HttpRequest
*>(&virginRequest());
1973 request
->clearError();
1976 /* Adaptation::Icap::ModXactLauncher */
1978 Adaptation::Icap::ModXactLauncher::ModXactLauncher(HttpMsg
*virginHeader
, HttpRequest
*virginCause
, AccessLogEntry::Pointer
&alp
, Adaptation::ServicePointer aService
):
1979 AsyncJob("Adaptation::Icap::ModXactLauncher"),
1980 Adaptation::Icap::Launcher("Adaptation::Icap::ModXactLauncher", aService
),
1983 virgin
.setHeader(virginHeader
);
1984 virgin
.setCause(virginCause
);
1985 updateHistory(true);
1988 Adaptation::Icap::Xaction
*Adaptation::Icap::ModXactLauncher::createXaction()
1990 Adaptation::Icap::ServiceRep::Pointer s
=
1991 dynamic_cast<Adaptation::Icap::ServiceRep
*>(theService
.getRaw());
1993 return new Adaptation::Icap::ModXact(virgin
.header
, virgin
.cause
, al
, s
);
1996 void Adaptation::Icap::ModXactLauncher::swanSong()
1998 debugs(93, 5, HERE
<< "swan sings");
1999 updateHistory(false);
2000 Adaptation::Icap::Launcher::swanSong();
2003 void Adaptation::Icap::ModXactLauncher::updateHistory(bool doStart
)
2005 HttpRequest
*r
= virgin
.cause
?
2006 virgin
.cause
: dynamic_cast<HttpRequest
*>(virgin
.header
);
2008 // r should never be NULL but we play safe; TODO: add Should()
2010 Adaptation::Icap::History::Pointer h
= r
->icapHistory();
2013 h
->start("ICAPModXactLauncher");
2015 h
->stop("ICAPModXactLauncher");