2 * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 93 ICAP (RFC 3507) Client */
12 #include "AccessLogEntry.h"
13 #include "adaptation/Answer.h"
14 #include "adaptation/History.h"
15 #include "adaptation/icap/Client.h"
16 #include "adaptation/icap/Config.h"
17 #include "adaptation/icap/History.h"
18 #include "adaptation/icap/Launcher.h"
19 #include "adaptation/icap/ModXact.h"
20 #include "adaptation/icap/ServiceRep.h"
21 #include "adaptation/Initiator.h"
22 #include "auth/UserRequest.h"
23 #include "base/TextException.h"
26 #include "comm/Connection.h"
27 #include "error/Detail.h"
28 #include "error/ExceptionErrorDetail.h"
29 #include "http/ContentLengthInterpreter.h"
30 #include "HttpHeaderTools.h"
31 #include "HttpReply.h"
32 #include "MasterXaction.h"
33 #include "parser/Tokenizer.h"
34 #include "sbuf/Stream.h"
35 #include "SquidTime.h"
37 // flow and terminology:
38 // HTTP| --> receive --> encode --> write --> |network
39 // end | <-- send <-- parse <-- read <-- |end
41 // TODO: replace gotEncapsulated() with something faster; we call it often
43 CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap
, ModXact
);
44 CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap
, ModXactLauncher
);
46 static const size_t TheBackupLimit
= BodyPipe::MaxCapacity
;
48 const SBuf
Adaptation::Icap::ChunkExtensionValueParser::UseOriginalBodyName("use-original-body");
50 Adaptation::Icap::ModXact::State::State()
52 memset(this, 0, sizeof(*this));
55 Adaptation::Icap::ModXact::ModXact(Http::Message
*virginHeader
,
56 HttpRequest
*virginCause
, AccessLogEntry::Pointer
&alp
, Adaptation::Icap::ServiceRep::Pointer
&aService
):
57 AsyncJob("Adaptation::Icap::ModXact"),
58 Adaptation::Icap::Xaction("Adaptation::Icap::ModXact", aService
),
61 canStartBypass(false), // too early
62 protectGroupBypass(true),
63 replyHttpHeaderSize(-1),
64 replyHttpBodySize(-1),
66 trailerParser(nullptr),
71 virgin
.setHeader(virginHeader
); // sets virgin.body_pipe if needed
72 virgin
.setCause(virginCause
); // may be NULL
74 // adapted header and body are initialized when we parse them
76 // writing and reading ends are handled by Adaptation::Icap::Xaction
79 // nothing to do because we are using temporary buffers
81 // parsing; TODO: do not set until we parse, see ICAPOptXact
82 icapReply
= new HttpReply
;
83 icapReply
->protoPrefix
= "ICAP/"; // TODO: make an IcapReply class?
85 debugs(93,7, HERE
<< "initialized." << status());
88 // initiator wants us to start
89 void Adaptation::Icap::ModXact::start()
91 Adaptation::Icap::Xaction::start();
93 // reserve an adaptation history slot (attempts are known at this time)
94 Adaptation::History::Pointer ah
= virginRequest().adaptLogHistory();
96 adaptHistoryId
= ah
->recordXactStart(service().cfg().key
, icap_tr_start
, attempts
> 1);
98 estimateVirginBody(); // before virgin disappears!
100 canStartBypass
= service().cfg().bypass
;
102 // it is an ICAP violation to send request to a service w/o known OPTIONS
103 // and the service may is too busy for us: honor Max-Connections and such
104 if (service().up() && service().availableForNew())
110 void Adaptation::Icap::ModXact::waitForService()
113 Must(!state
.serviceWaiting
);
115 if (!service().up()) {
116 AsyncCall::Pointer call
= JobCallback(93,5,
117 ConnWaiterDialer
, this, Adaptation::Icap::ModXact::noteServiceReady
);
119 service().callWhenReady(call
);
120 comment
= "to be up";
122 //The service is unavailable because of max-connection or other reason
124 if (service().cfg().onOverload
!= srvWait
) {
125 // The service is overloaded, but waiting to be available prohibited by
126 // user configuration (onOverload is set to "block" or "bypass")
127 if (service().cfg().onOverload
== srvBlock
)
128 disableBypass("not available", true);
129 else //if (service().cfg().onOverload == srvBypass)
130 canStartBypass
= true;
133 disableRepeats("ICAP service is not available");
135 debugs(93, 7, HERE
<< "will not wait for the service to be available" <<
138 throw TexcHere("ICAP service is not available");
141 AsyncCall::Pointer call
= JobCallback(93,5,
142 ConnWaiterDialer
, this, Adaptation::Icap::ModXact::noteServiceAvailable
);
143 service().callWhenAvailable(call
, state
.waitedForService
);
144 comment
= "to be available";
147 debugs(93, 7, HERE
<< "will wait for the service " << comment
<< status());
148 state
.serviceWaiting
= true; // after callWhenReady() which may throw
149 state
.waitedForService
= true;
152 void Adaptation::Icap::ModXact::noteServiceReady()
154 Must(state
.serviceWaiting
);
155 state
.serviceWaiting
= false;
157 if (!service().up()) {
159 disableRepeats("ICAP service is unusable");
160 throw TexcHere("ICAP service is unusable");
163 if (service().availableForOld())
169 void Adaptation::Icap::ModXact::noteServiceAvailable()
171 Must(state
.serviceWaiting
);
172 state
.serviceWaiting
= false;
174 if (service().up() && service().availableForOld())
180 void Adaptation::Icap::ModXact::startWriting()
182 state
.writing
= State::writingConnect
;
184 decideOnPreview(); // must be decided before we decideOnRetries
190 void Adaptation::Icap::ModXact::startShoveling()
192 Must(state
.writing
== State::writingConnect
);
194 startReading(); // wait for early errors from the ICAP server
199 makeRequestHeaders(requestBuf
);
200 debugs(93, 9, HERE
<< "will write" << status() << ":\n" <<
201 (requestBuf
.terminate(), requestBuf
.content()));
204 state
.writing
= State::writingHeaders
;
205 icap_tio_start
= current_time
;
206 scheduleWrite(requestBuf
);
209 void Adaptation::Icap::ModXact::handleCommWrote(size_t sz
)
211 debugs(93, 5, HERE
<< "Wrote " << sz
<< " bytes");
213 if (state
.writing
== State::writingHeaders
)
214 handleCommWroteHeaders();
216 handleCommWroteBody();
219 void Adaptation::Icap::ModXact::handleCommWroteHeaders()
221 Must(state
.writing
== State::writingHeaders
);
223 // determine next step
224 if (preview
.enabled()) {
226 decideWritingAfterPreview("zero-size");
228 state
.writing
= State::writingPreview
;
229 } else if (virginBody
.expected()) {
230 state
.writing
= State::writingPrime
;
239 void Adaptation::Icap::ModXact::writeMore()
241 debugs(93, 5, HERE
<< "checking whether to write more" << status());
243 if (writer
!= NULL
) // already writing something
246 switch (state
.writing
) {
248 case State::writingInit
: // waiting for service OPTIONS
249 Must(state
.serviceWaiting
);
252 case State::writingConnect
: // waiting for the connection to establish
253 case State::writingHeaders
: // waiting for the headers to be written
254 case State::writingPaused
: // waiting for the ICAP server response
255 case State::writingReallyDone
: // nothing more to write
258 case State::writingAlmostDone
: // was waiting for the last write
262 case State::writingPreview
:
266 case State::writingPrime
:
271 throw TexcHere("Adaptation::Icap::ModXact in bad writing state");
275 void Adaptation::Icap::ModXact::writePreviewBody()
277 debugs(93, 8, HERE
<< "will write Preview body from " <<
278 virgin
.body_pipe
<< status());
279 Must(state
.writing
== State::writingPreview
);
280 Must(virgin
.body_pipe
!= NULL
);
282 const size_t sizeMax
= (size_t)virgin
.body_pipe
->buf().contentSize();
283 const size_t size
= min(preview
.debt(), sizeMax
);
284 writeSomeBody("preview body", size
);
286 // change state once preview is written
289 decideWritingAfterPreview("body");
292 /// determine state.writing after we wrote the entire preview
293 void Adaptation::Icap::ModXact::decideWritingAfterPreview(const char *kind
)
295 if (preview
.ieof()) // nothing more to write
297 else if (state
.parsing
== State::psIcapHeader
) // did not get a reply yet
298 state
.writing
= State::writingPaused
; // wait for the ICAP server reply
300 stopWriting(true); // ICAP server reply implies no post-preview writing
302 debugs(93, 6, HERE
<< "decided on writing after " << kind
<< " preview" <<
306 void Adaptation::Icap::ModXact::writePrimeBody()
308 Must(state
.writing
== State::writingPrime
);
309 Must(virginBodyWriting
.active());
311 const size_t size
= (size_t)virgin
.body_pipe
->buf().contentSize();
312 writeSomeBody("prime virgin body", size
);
314 if (virginBodyEndReached(virginBodyWriting
)) {
315 debugs(93, 5, HERE
<< "wrote entire body");
320 void Adaptation::Icap::ModXact::writeSomeBody(const char *label
, size_t size
)
322 Must(!writer
&& state
.writing
< state
.writingAlmostDone
);
323 Must(virgin
.body_pipe
!= NULL
);
324 debugs(93, 8, HERE
<< "will write up to " << size
<< " bytes of " <<
327 MemBuf writeBuf
; // TODO: suggest a min size based on size and lastChunk
329 writeBuf
.init(); // note: we assume that last-chunk will fit
331 const size_t writableSize
= virginContentSize(virginBodyWriting
);
332 const size_t chunkSize
= min(writableSize
, size
);
335 debugs(93, 7, HERE
<< "will write " << chunkSize
<<
336 "-byte chunk of " << label
);
338 openChunk(writeBuf
, chunkSize
, false);
339 writeBuf
.append(virginContentData(virginBodyWriting
), chunkSize
);
340 closeChunk(writeBuf
);
342 virginBodyWriting
.progress(chunkSize
);
345 debugs(93, 7, HERE
<< "has no writable " << label
<< " content");
348 const bool wroteEof
= virginBodyEndReached(virginBodyWriting
);
349 bool lastChunk
= wroteEof
;
350 if (state
.writing
== State::writingPreview
) {
351 preview
.wrote(chunkSize
, wroteEof
); // even if wrote nothing
352 lastChunk
= lastChunk
|| preview
.done();
356 debugs(93, 8, HERE
<< "will write last-chunk of " << label
);
357 addLastRequestChunk(writeBuf
);
360 debugs(93, 7, HERE
<< "will write " << writeBuf
.contentSize()
361 << " raw bytes of " << label
);
363 if (writeBuf
.hasContent()) {
364 scheduleWrite(writeBuf
); // comm will free the chunk
370 void Adaptation::Icap::ModXact::addLastRequestChunk(MemBuf
&buf
)
372 const bool ieof
= state
.writing
== State::writingPreview
&& preview
.ieof();
373 openChunk(buf
, 0, ieof
);
377 void Adaptation::Icap::ModXact::openChunk(MemBuf
&buf
, size_t chunkSize
, bool ieof
)
379 buf
.appendf((ieof
? "%x; ieof\r\n" : "%x\r\n"), (int) chunkSize
);
382 void Adaptation::Icap::ModXact::closeChunk(MemBuf
&buf
)
384 buf
.append(ICAP::crlf
, 2); // chunk-terminating CRLF
387 const HttpRequest
&Adaptation::Icap::ModXact::virginRequest() const
389 const HttpRequest
*request
= virgin
.cause
?
390 virgin
.cause
: dynamic_cast<const HttpRequest
*>(virgin
.header
);
395 // did the activity reached the end of the virgin body?
396 bool Adaptation::Icap::ModXact::virginBodyEndReached(const Adaptation::Icap::VirginBodyAct
&act
) const
399 !act
.active() || // did all (assuming it was originally planned)
400 !virgin
.body_pipe
->expectMoreAfter(act
.offset()); // will not have more
403 // the size of buffered virgin body data available for the specified activity
404 // if this size is zero, we may be done or may be waiting for more data
405 size_t Adaptation::Icap::ModXact::virginContentSize(const Adaptation::Icap::VirginBodyAct
&act
) const
408 // asbolute start of unprocessed data
409 const uint64_t dataStart
= act
.offset();
410 // absolute end of buffered data
411 const uint64_t dataEnd
= virginConsumed
+ virgin
.body_pipe
->buf().contentSize();
412 Must(virginConsumed
<= dataStart
&& dataStart
<= dataEnd
);
413 return static_cast<size_t>(dataEnd
- dataStart
);
416 // pointer to buffered virgin body data available for the specified activity
417 const char *Adaptation::Icap::ModXact::virginContentData(const Adaptation::Icap::VirginBodyAct
&act
) const
420 const uint64_t dataStart
= act
.offset();
421 Must(virginConsumed
<= dataStart
);
422 return virgin
.body_pipe
->buf().content() + static_cast<size_t>(dataStart
-virginConsumed
);
425 void Adaptation::Icap::ModXact::virginConsume()
427 debugs(93, 9, HERE
<< "consumption guards: " << !virgin
.body_pipe
<< isRetriable
<<
428 isRepeatable
<< canStartBypass
<< protectGroupBypass
);
430 if (!virgin
.body_pipe
)
431 return; // nothing to consume
434 return; // do not consume if we may have to retry later
436 BodyPipe
&bp
= *virgin
.body_pipe
;
437 const bool wantToPostpone
= isRepeatable
|| canStartBypass
|| protectGroupBypass
;
439 // Why > 2? HttpState does not use the last bytes in the buffer
440 // because delayAwareRead() is arguably broken. See
441 // HttpStateData::maybeReadVirginBody for more details.
442 if (wantToPostpone
&& bp
.buf().spaceSize() > 2) {
443 // Postponing may increase memory footprint and slow the HTTP side
444 // down. Not postponing may increase the number of ICAP errors
445 // if the ICAP service fails. We may also use "potential" space to
446 // postpone more aggressively. Should the trade-off be configurable?
447 debugs(93, 8, HERE
<< "postponing consumption from " << bp
.status());
451 const size_t have
= static_cast<size_t>(bp
.buf().contentSize());
452 const uint64_t end
= virginConsumed
+ have
;
453 uint64_t offset
= end
;
455 debugs(93, 9, HERE
<< "max virgin consumption offset=" << offset
<<
456 " acts " << virginBodyWriting
.active() << virginBodySending
.active() <<
457 " consumed=" << virginConsumed
<<
458 " from " << virgin
.body_pipe
->status());
460 if (virginBodyWriting
.active())
461 offset
= min(virginBodyWriting
.offset(), offset
);
463 if (virginBodySending
.active())
464 offset
= min(virginBodySending
.offset(), offset
);
466 Must(virginConsumed
<= offset
&& offset
<= end
);
468 if (const size_t size
= static_cast<size_t>(offset
- virginConsumed
)) {
469 debugs(93, 8, HERE
<< "consuming " << size
<< " out of " << have
<<
470 " virgin body bytes");
472 virginConsumed
+= size
;
473 Must(!isRetriable
); // or we should not be consuming
474 disableRepeats("consumed content");
475 disableBypass("consumed content", true);
479 void Adaptation::Icap::ModXact::handleCommWroteBody()
484 // Called when we do not expect to call comm_write anymore.
485 // We may have a pending write though.
486 // If stopping nicely, we will just wait for that pending write, if any.
487 void Adaptation::Icap::ModXact::stopWriting(bool nicely
)
489 if (state
.writing
== State::writingReallyDone
)
492 if (writer
!= NULL
) {
494 debugs(93, 7, HERE
<< "will wait for the last write" << status());
495 state
.writing
= State::writingAlmostDone
; // may already be set
499 debugs(93, 3, HERE
<< "will NOT wait for the last write" << status());
501 // Comm does not have an interface to clear the writer callback nicely,
502 // but without clearing the writer we cannot recycle the connection.
503 // We prevent connection reuse and hope that we can handle a callback
504 // call at any time, usually in the middle of the destruction sequence!
505 // Somebody should add comm_remove_write_handler() to comm API.
506 reuseConnection
= false;
507 ignoreLastWrite
= true;
510 debugs(93, 7, HERE
<< "will no longer write" << status());
511 if (virginBodyWriting
.active()) {
512 virginBodyWriting
.disable();
515 state
.writing
= State::writingReallyDone
;
519 void Adaptation::Icap::ModXact::stopBackup()
521 if (!virginBodySending
.active())
524 debugs(93, 7, HERE
<< "will no longer backup" << status());
525 virginBodySending
.disable();
529 bool Adaptation::Icap::ModXact::doneAll() const
531 return Adaptation::Icap::Xaction::doneAll() && !state
.serviceWaiting
&&
533 doneReading() && state
.doneWriting();
536 void Adaptation::Icap::ModXact::startReading()
538 Must(haveConnection());
540 Must(!adapted
.header
);
541 Must(!adapted
.body_pipe
);
543 // we use the same buffer for headers and body and then consume headers
547 void Adaptation::Icap::ModXact::readMore()
549 if (reader
!= NULL
|| doneReading()) {
550 debugs(93,3,HERE
<< "returning from readMore because reader or doneReading()");
554 // do not fill readBuf if we have no space to store the result
555 if (adapted
.body_pipe
!= NULL
&&
556 !adapted
.body_pipe
->buf().hasPotentialSpace()) {
557 debugs(93,3,HERE
<< "not reading because ICAP reply pipe is full");
561 if (readBuf
.length() < SQUID_TCP_SO_RCVBUF
)
564 debugs(93,3,HERE
<< "cannot read with a full buffer");
567 // comm module read a portion of the ICAP response for us
568 void Adaptation::Icap::ModXact::handleCommRead(size_t)
570 Must(!state
.doneParsing());
571 icap_tio_finish
= current_time
;
576 void Adaptation::Icap::ModXact::echoMore()
578 Must(state
.sending
== State::sendingVirgin
);
579 Must(adapted
.body_pipe
!= NULL
);
580 Must(virginBodySending
.active());
582 const size_t sizeMax
= virginContentSize(virginBodySending
);
583 debugs(93,5, HERE
<< "will echo up to " << sizeMax
<< " bytes from " <<
584 virgin
.body_pipe
->status());
585 debugs(93,5, HERE
<< "will echo up to " << sizeMax
<< " bytes to " <<
586 adapted
.body_pipe
->status());
589 const size_t size
= adapted
.body_pipe
->putMoreData(virginContentData(virginBodySending
), sizeMax
);
590 debugs(93,5, HERE
<< "echoed " << size
<< " out of " << sizeMax
<<
592 virginBodySending
.progress(size
);
593 disableRepeats("echoed content");
594 disableBypass("echoed content", true);
598 if (virginBodyEndReached(virginBodySending
)) {
599 debugs(93, 5, HERE
<< "echoed all" << status());
602 debugs(93, 5, HERE
<< "has " <<
603 virgin
.body_pipe
->buf().contentSize() << " bytes " <<
604 "and expects more to echo" << status());
605 // TODO: timeout if virgin or adapted pipes are broken
609 bool Adaptation::Icap::ModXact::doneSending() const
611 return state
.sending
== State::sendingDone
;
614 // stop (or do not start) sending adapted message body
615 void Adaptation::Icap::ModXact::stopSending(bool nicely
)
617 debugs(93, 7, HERE
<< "Enter stop sending ");
620 debugs(93, 7, HERE
<< "Proceed with stop sending ");
622 if (state
.sending
!= State::sendingUndecided
) {
623 debugs(93, 7, HERE
<< "will no longer send" << status());
624 if (adapted
.body_pipe
!= NULL
) {
625 virginBodySending
.disable();
626 // we may leave debts if we were echoing and the virgin
627 // body_pipe got exhausted before we echoed all planned bytes
628 const bool leftDebts
= adapted
.body_pipe
->needsMoreData();
629 stopProducingFor(adapted
.body_pipe
, nicely
&& !leftDebts
);
632 debugs(93, 7, HERE
<< "will not start sending" << status());
633 Must(!adapted
.body_pipe
);
636 state
.sending
= State::sendingDone
;
640 // should be called after certain state.writing or state.sending changes
641 void Adaptation::Icap::ModXact::checkConsuming()
643 // quit if we already stopped or are still using the pipe
644 if (!virgin
.body_pipe
|| !state
.doneConsumingVirgin())
647 debugs(93, 7, HERE
<< "will stop consuming" << status());
648 stopConsumingFrom(virgin
.body_pipe
);
651 void Adaptation::Icap::ModXact::parseMore()
653 debugs(93, 5, "have " << readBuf
.length() << " bytes to parse" << status());
654 debugs(93, 5, "\n" << readBuf
);
656 if (state
.parsingHeaders())
659 if (state
.parsing
== State::psBody
)
662 if (state
.parsing
== State::psIcapTrailer
)
666 void Adaptation::Icap::ModXact::callException(const std::exception
&e
)
668 if (!canStartBypass
|| isRetriable
) {
670 if (const TextException
*te
= dynamic_cast<const TextException
*>(&e
))
671 detailError(new ExceptionErrorDetail(te
->id()));
673 detailError(new ExceptionErrorDetail(Here().id()));
675 Adaptation::Icap::Xaction::callException(e
);
680 debugs(93, 3, HERE
<< "bypassing " << inCall
<< " exception: " <<
681 e
.what() << ' ' << status());
683 } catch (const TextException
&bypassTe
) {
684 detailError(new ExceptionErrorDetail(bypassTe
.id()));
685 Adaptation::Icap::Xaction::callException(bypassTe
);
686 } catch (const std::exception
&bypassE
) {
687 detailError(new ExceptionErrorDetail(Here().id()));
688 Adaptation::Icap::Xaction::callException(bypassE
);
692 void Adaptation::Icap::ModXact::bypassFailure()
694 disableBypass("already started to bypass", false);
696 Must(!isRetriable
); // or we should not be bypassing
697 // TODO: should the same be enforced for isRepeatable? Check icap_repeat??
703 // end all activities associated with the ICAP server
707 stopWriting(true); // or should we force it?
708 if (haveConnection()) {
709 reuseConnection
= false; // be conservative
710 cancelRead(); // may not work; and we cannot stop connecting either
712 debugs(93, 7, HERE
<< "Warning: bypass failed to stop I/O" << status());
715 service().noteFailure(); // we are bypassing, but this is still a failure
718 void Adaptation::Icap::ModXact::disableBypass(const char *reason
, bool includingGroupBypass
)
720 if (canStartBypass
) {
721 debugs(93,7, HERE
<< "will never start bypass because " << reason
);
722 canStartBypass
= false;
724 if (protectGroupBypass
&& includingGroupBypass
) {
725 debugs(93,7, HERE
<< "not protecting group bypass because " << reason
);
726 protectGroupBypass
= false;
730 // note that allocation for echoing is done in handle204NoContent()
731 void Adaptation::Icap::ModXact::maybeAllocateHttpMsg()
733 if (adapted
.header
) // already allocated
736 if (gotEncapsulated("res-hdr")) {
737 adapted
.setHeader(new HttpReply
);
738 setOutcome(service().cfg().method
== ICAP::methodReqmod
?
739 xoSatisfied
: xoModified
);
740 } else if (gotEncapsulated("req-hdr")) {
741 adapted
.setHeader(new HttpRequest(virginRequest().masterXaction
));
742 setOutcome(xoModified
);
744 throw TexcHere("Neither res-hdr nor req-hdr in maybeAllocateHttpMsg()");
747 void Adaptation::Icap::ModXact::parseHeaders()
749 Must(state
.parsingHeaders());
751 if (state
.parsing
== State::psIcapHeader
) {
752 debugs(93, 5, HERE
<< "parse ICAP headers");
756 if (state
.parsing
== State::psHttpHeader
) {
757 debugs(93, 5, HERE
<< "parse HTTP headers");
761 if (state
.parsingHeaders()) { // need more data
769 // called after parsing all headers or when bypassing an exception
770 void Adaptation::Icap::ModXact::startSending()
772 disableRepeats("sent headers");
773 disableBypass("sent headers", true);
774 sendAnswer(Answer::Forward(adapted
.header
));
776 if (state
.sending
== State::sendingVirgin
)
779 // If we are not using the virgin HTTP object update the
780 // Http::Message::sources flag.
781 // The state.sending may set to State::sendingVirgin in the case
782 // of 206 responses too, where we do not want to update Http::Message::sources
783 // flag. However even for 206 responses the state.sending is
784 // not set yet to sendingVirgin. This is done in later step
785 // after the parseBody method called.
790 void Adaptation::Icap::ModXact::parseIcapHead()
792 Must(state
.sending
== State::sendingUndecided
);
794 if (!parseHead(icapReply
.getRaw()))
797 if (expectIcapTrailers()) {
798 Must(!trailerParser
);
799 trailerParser
= new TrailerParser
;
802 static SBuf
close("close", 5);
803 if (httpHeaderHasConnDir(&icapReply
->header
, close
)) {
804 debugs(93, 5, HERE
<< "found connection close");
805 reuseConnection
= false;
808 switch (icapReply
->sline
.status()) {
810 case Http::scContinue
:
815 case Http::scCreated
: // Symantec Scan Engine 5.0 and later when modifying HTTP msg
817 if (!validate200Ok()) {
818 throw TexcHere("Invalid ICAP Response");
825 case Http::scNoContent
:
826 handle204NoContent();
829 case Http::scPartialContent
:
830 handle206PartialContent();
834 debugs(93, 5, "ICAP status " << icapReply
->sline
.status());
835 handleUnknownScode();
839 const HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
841 request
= &virginRequest();
843 // update the cross-transactional database if needed (all status codes!)
844 if (const char *xxName
= Adaptation::Config::masterx_shared_name
) {
845 Adaptation::History::Pointer ah
= request
->adaptHistory(true);
846 if (ah
!= NULL
) { // TODO: reorder checks to avoid creating history
847 const String val
= icapReply
->header
.getByName(xxName
);
848 if (val
.size() > 0) // XXX: HttpHeader lacks empty value detection
849 ah
->updateXxRecord(xxName
, val
);
853 // update the adaptation plan if needed (all status codes!)
854 if (service().cfg().routing
) {
856 if (icapReply
->header
.getList(Http::HdrType::X_NEXT_SERVICES
, &services
)) {
857 Adaptation::History::Pointer ah
= request
->adaptHistory(true);
859 ah
->updateNextServices(services
);
861 } // TODO: else warn (occasionally!) if we got Http::HdrType::X_NEXT_SERVICES
863 // We need to store received ICAP headers for <icapLastHeader logformat option.
864 // If we already have stored headers from previous ICAP transaction related to this
865 // request, old headers will be replaced with the new one.
867 Adaptation::History::Pointer ah
= request
->adaptLogHistory();
869 ah
->recordMeta(&icapReply
->header
);
871 // handle100Continue() manages state.writing on its own.
872 // Non-100 status means the server needs no postPreview data from us.
873 if (state
.writing
== State::writingPaused
)
877 /// Parses ICAP trailers and stops parsing, if all trailer data
878 /// have been received.
879 void Adaptation::Icap::ModXact::parseIcapTrailer() {
881 if (parsePart(trailerParser
, "trailer")) {
882 for (const auto &e
: trailerParser
->trailer
.entries
)
883 debugs(93, 5, "ICAP trailer: " << e
->name
<< ": " << e
->value
);
888 bool Adaptation::Icap::ModXact::validate200Ok()
890 if (service().cfg().method
== ICAP::methodRespmod
)
891 return gotEncapsulated("res-hdr");
893 return service().cfg().method
== ICAP::methodReqmod
&&
897 void Adaptation::Icap::ModXact::handle100Continue()
899 Must(state
.writing
== State::writingPaused
);
900 // server must not respond before the end of preview: we may send ieof
901 Must(preview
.enabled() && preview
.done() && !preview
.ieof());
903 // 100 "Continue" cancels our Preview commitment,
904 // but not commitment to handle 204 or 206 outside Preview
905 if (!state
.allowedPostview204
&& !state
.allowedPostview206
)
908 state
.parsing
= State::psIcapHeader
; // eventually
911 state
.writing
= State::writingPrime
;
916 void Adaptation::Icap::ModXact::handle200Ok()
918 state
.parsing
= State::psHttpHeader
;
919 state
.sending
= State::sendingAdapted
;
924 void Adaptation::Icap::ModXact::handle204NoContent()
930 void Adaptation::Icap::ModXact::handle206PartialContent()
932 if (state
.writing
== State::writingPaused
) {
933 Must(preview
.enabled());
934 Must(state
.allowedPreview206
);
935 debugs(93, 7, HERE
<< "206 inside preview");
937 Must(state
.writing
> State::writingPaused
);
938 Must(state
.allowedPostview206
);
939 debugs(93, 7, HERE
<< "206 outside preview");
941 state
.parsing
= State::psHttpHeader
;
942 state
.sending
= State::sendingAdapted
;
943 state
.readyForUob
= true;
947 // Called when we receive a 204 No Content response and
948 // when we are trying to bypass a service failure.
949 // We actually start sending (echoig or not) in startSending.
950 void Adaptation::Icap::ModXact::prepEchoing()
952 disableRepeats("preparing to echo content");
953 disableBypass("preparing to echo content", true);
956 // We want to clone the HTTP message, but we do not want
957 // to copy some non-HTTP state parts that Http::Message kids carry in them.
958 // Thus, we cannot use a smart pointer, copy constructor, or equivalent.
959 // Instead, we simply write the HTTP message and "clone" it by parsing.
960 // TODO: use Http::Message::clone()!
962 Http::Message
*oldHead
= virgin
.header
;
963 debugs(93, 7, HERE
<< "cloning virgin message " << oldHead
);
967 // write the virgin message into a memory buffer
969 packHead(httpBuf
, oldHead
);
971 // allocate the adapted message and copy metainfo
972 Must(!adapted
.header
);
974 Http::MessagePointer newHead
;
975 if (const HttpRequest
*r
= dynamic_cast<const HttpRequest
*>(oldHead
)) {
976 newHead
= new HttpRequest(r
->masterXaction
);
977 } else if (dynamic_cast<const HttpReply
*>(oldHead
)) {
978 newHead
= new HttpReply
;
982 newHead
->inheritProperties(oldHead
);
984 adapted
.setHeader(newHead
.getRaw());
987 // parse the buffer back
988 Http::StatusCode error
= Http::scNone
;
990 httpBuf
.terminate(); // Http::Message::parse requires nil-terminated buffer
991 Must(adapted
.header
->parse(httpBuf
.content(), httpBuf
.contentSize(), true, &error
));
992 Must(adapted
.header
->hdr_sz
== httpBuf
.contentSize()); // no leftovers
996 debugs(93, 7, HERE
<< "cloned virgin message " << oldHead
<< " to " <<
999 // setup adapted body pipe if needed
1000 if (oldHead
->body_pipe
!= NULL
) {
1001 debugs(93, 7, HERE
<< "will echo virgin body from " <<
1002 oldHead
->body_pipe
);
1003 if (!virginBodySending
.active())
1004 virginBodySending
.plan(); // will throw if not possible
1005 state
.sending
= State::sendingVirgin
;
1008 // TODO: optimize: is it possible to just use the oldHead pipe and
1009 // remove ICAP from the loop? This echoing is probably a common case!
1010 makeAdaptedBodyPipe("echoed virgin response");
1011 if (oldHead
->body_pipe
->bodySizeKnown())
1012 adapted
.body_pipe
->setBodySize(oldHead
->body_pipe
->bodySize());
1013 debugs(93, 7, HERE
<< "will echo virgin body to " <<
1016 debugs(93, 7, HERE
<< "no virgin body to echo");
1021 /// Called when we received use-original-body chunk extension in 206 response.
1022 /// We actually start sending (echoing or not) in startSending().
1023 void Adaptation::Icap::ModXact::prepPartialBodyEchoing(uint64_t pos
)
1025 Must(virginBodySending
.active());
1026 Must(virgin
.header
->body_pipe
!= NULL
);
1028 setOutcome(xoPartEcho
);
1030 debugs(93, 7, HERE
<< "will echo virgin body suffix from " <<
1031 virgin
.header
->body_pipe
<< " offset " << pos
);
1033 // check that use-original-body=N does not point beyond buffered data
1034 const uint64_t virginDataEnd
= virginConsumed
+
1035 virgin
.body_pipe
->buf().contentSize();
1036 Must(pos
<= virginDataEnd
);
1037 virginBodySending
.progress(static_cast<size_t>(pos
));
1039 state
.sending
= State::sendingVirgin
;
1042 if (virgin
.header
->body_pipe
->bodySizeKnown())
1043 adapted
.body_pipe
->expectProductionEndAfter(virgin
.header
->body_pipe
->bodySize() - pos
);
1045 debugs(93, 7, HERE
<< "will echo virgin body suffix to " <<
1048 // Start echoing data
1052 void Adaptation::Icap::ModXact::handleUnknownScode()
1056 // TODO: mark connection as "bad"
1058 // Terminate the transaction; we do not know how to handle this response.
1059 throw TexcHere("Unsupported ICAP status code");
1062 void Adaptation::Icap::ModXact::parseHttpHead()
1064 if (expectHttpHeader()) {
1065 replyHttpHeaderSize
= 0;
1066 maybeAllocateHttpMsg();
1068 if (!parseHead(adapted
.header
))
1069 return; // need more header data
1072 replyHttpHeaderSize
= adapted
.header
->hdr_sz
;
1074 if (dynamic_cast<HttpRequest
*>(adapted
.header
)) {
1075 const HttpRequest
*oldR
= dynamic_cast<const HttpRequest
*>(virgin
.header
);
1077 // TODO: the adapted request did not really originate from the
1078 // client; give proxy admin an option to prevent copying of
1079 // sensitive client information here. See the following thread:
1080 // http://www.squid-cache.org/mail-archive/squid-dev/200703/0040.html
1083 // Maybe adapted.header==NULL if HttpReply and have Http 0.9 ....
1085 adapted
.header
->inheritProperties(virgin
.header
);
1088 decideOnParsingBody();
1091 template<class Part
>
1092 bool Adaptation::Icap::ModXact::parsePart(Part
*part
, const char *description
)
1095 debugs(93, 5, "have " << readBuf
.length() << ' ' << description
<< " bytes to parse; state: " << state
.parsing
);
1096 Http::StatusCode error
= Http::scNone
;
1097 // XXX: performance regression. c_str() data copies
1098 // XXX: Http::Message::parse requires a terminated string buffer
1099 const char *tmpBuf
= readBuf
.c_str();
1100 const bool parsed
= part
->parse(tmpBuf
, readBuf
.length(), commEof
, &error
);
1101 debugs(93, (!parsed
&& error
) ? 2 : 5, description
<< " parsing result: " << parsed
<< " detail: " << error
);
1102 Must(parsed
|| !error
);
1104 readBuf
.consume(part
->hdr_sz
);
1108 // parses both HTTP and ICAP headers
1110 Adaptation::Icap::ModXact::parseHead(Http::Message
*head
)
1112 if (!parsePart(head
, "head")) {
1119 bool Adaptation::Icap::ModXact::expectHttpHeader() const
1121 return gotEncapsulated("res-hdr") || gotEncapsulated("req-hdr");
1124 bool Adaptation::Icap::ModXact::expectHttpBody() const
1126 return gotEncapsulated("res-body") || gotEncapsulated("req-body");
1129 bool Adaptation::Icap::ModXact::expectIcapTrailers() const
1132 const bool promisesToSendTrailer
= icapReply
->header
.getByIdIfPresent(Http::HdrType::TRAILER
, &trailers
);
1133 const bool supportsTrailers
= icapReply
->header
.hasListMember(Http::HdrType::ALLOW
, "trailers", ',');
1134 // ICAP Trailer specs require us to reject transactions having either Trailer
1135 // header or Allow:trailers
1136 Must((promisesToSendTrailer
== supportsTrailers
) || (!promisesToSendTrailer
&& supportsTrailers
));
1137 if (promisesToSendTrailer
&& !trailers
.size())
1138 debugs(93, DBG_IMPORTANT
, "ERROR: ICAP Trailer response header field must not be empty (salvaged)");
1139 return promisesToSendTrailer
;
1142 void Adaptation::Icap::ModXact::decideOnParsingBody()
1144 if (expectHttpBody()) {
1145 debugs(93, 5, HERE
<< "expecting a body");
1146 state
.parsing
= State::psBody
;
1147 replyHttpBodySize
= 0;
1148 bodyParser
= new Http1::TeChunkedParser
;
1149 bodyParser
->parseExtensionValuesWith(&extensionParser
);
1150 makeAdaptedBodyPipe("adapted response from the ICAP server");
1151 Must(state
.sending
== State::sendingAdapted
);
1153 debugs(93, 5, HERE
<< "not expecting a body");
1155 state
.parsing
= State::psIcapTrailer
;
1162 void Adaptation::Icap::ModXact::parseBody()
1164 Must(state
.parsing
== State::psBody
);
1167 debugs(93, 5, "have " << readBuf
.length() << " body bytes to parse");
1169 // the parser will throw on errors
1170 BodyPipeCheckout
bpc(*adapted
.body_pipe
);
1171 bodyParser
->setPayloadBuffer(&bpc
.buf
);
1172 const bool parsed
= bodyParser
->parse(readBuf
);
1173 readBuf
= bodyParser
->remaining(); // sync buffers after parse
1176 debugs(93, 5, "have " << readBuf
.length() << " body bytes after parsed all: " << parsed
);
1177 replyHttpBodySize
+= adapted
.body_pipe
->buf().contentSize();
1179 // TODO: expose BodyPipe::putSize() to make this check simpler and clearer
1180 // TODO: do we really need this if we disable when sending headers?
1181 if (adapted
.body_pipe
->buf().contentSize() > 0) { // parsed something sometime
1182 disableRepeats("sent adapted content");
1183 disableBypass("sent adapted content", true);
1187 if (state
.readyForUob
&& extensionParser
.sawUseOriginalBody())
1188 prepPartialBodyEchoing(extensionParser
.useOriginalBody());
1190 stopSending(true); // the parser succeeds only if all parsed data fits
1192 state
.parsing
= State::psIcapTrailer
;
1198 debugs(93,3,HERE
<< this << " needsMoreData = " << bodyParser
->needsMoreData());
1200 if (bodyParser
->needsMoreData()) {
1201 debugs(93,3,HERE
<< this);
1202 Must(mayReadMore());
1206 if (bodyParser
->needsMoreSpace()) {
1207 Must(!doneSending()); // can hope for more space
1208 Must(adapted
.body_pipe
->buf().contentSize() > 0); // paranoid
1209 // TODO: there should be a timeout in case the sink is broken
1210 // or cannot consume partial content (while we need more space)
1214 void Adaptation::Icap::ModXact::stopParsing(const bool checkUnparsedData
)
1216 if (state
.parsing
== State::psDone
)
1219 if (checkUnparsedData
)
1220 Must(readBuf
.isEmpty());
1222 debugs(93, 7, "will no longer parse" << status());
1225 bodyParser
= nullptr;
1227 delete trailerParser
;
1228 trailerParser
= nullptr;
1230 state
.parsing
= State::psDone
;
1233 // HTTP side added virgin body data
1234 void Adaptation::Icap::ModXact::noteMoreBodyDataAvailable(BodyPipe::Pointer
)
1238 if (state
.sending
== State::sendingVirgin
)
1242 // HTTP side sent us all virgin info
1243 void Adaptation::Icap::ModXact::noteBodyProductionEnded(BodyPipe::Pointer
)
1245 Must(virgin
.body_pipe
->productionEnded());
1247 // push writer and sender in case we were waiting for the last-chunk
1250 if (state
.sending
== State::sendingVirgin
)
1254 // body producer aborted, but the initiator may still want to know
1255 // the answer, even though the HTTP message has been truncated
1256 void Adaptation::Icap::ModXact::noteBodyProducerAborted(BodyPipe::Pointer
)
1258 Must(virgin
.body_pipe
->productionEnded());
1260 // push writer and sender in case we were waiting for the last-chunk
1263 if (state
.sending
== State::sendingVirgin
)
1267 // adapted body consumer wants more adapted data and
1268 // possibly freed some buffer space
1269 void Adaptation::Icap::ModXact::noteMoreBodySpaceAvailable(BodyPipe::Pointer
)
1271 if (state
.sending
== State::sendingVirgin
)
1273 else if (state
.sending
== State::sendingAdapted
)
1276 Must(state
.sending
== State::sendingUndecided
);
1279 // adapted body consumer aborted
1280 void Adaptation::Icap::ModXact::noteBodyConsumerAborted(BodyPipe::Pointer
)
1282 static const auto d
= MakeNamedErrorDetail("ICAP_XACT_BODY_CONSUMER_ABORT");
1284 mustStop("adapted body consumer aborted");
1287 Adaptation::Icap::ModXact::~ModXact()
1290 delete trailerParser
;
1294 void Adaptation::Icap::ModXact::swanSong()
1296 debugs(93, 5, HERE
<< "swan sings" << status());
1301 if (theInitiator
.set()) { // we have not sent the answer to the initiator
1302 static const auto d
= MakeNamedErrorDetail("ICAP_XACT_OTHER");
1306 // update adaptation history if start was called and we reserved a slot
1307 Adaptation::History::Pointer ah
= virginRequest().adaptLogHistory();
1308 if (ah
!= NULL
&& adaptHistoryId
>= 0)
1309 ah
->recordXactFinish(adaptHistoryId
);
1311 Adaptation::Icap::Xaction::swanSong();
1314 void prepareLogWithRequestDetails(HttpRequest
*, AccessLogEntry::Pointer
&);
1316 void Adaptation::Icap::ModXact::finalizeLogInfo()
1318 HttpRequest
*adapted_request_
= nullptr;
1319 HttpReply
*adapted_reply_
= nullptr;
1320 HttpRequest
*virgin_request_
= const_cast<HttpRequest
*>(&virginRequest());
1321 if (!(adapted_request_
= dynamic_cast<HttpRequest
*>(adapted
.header
))) {
1322 // if the request was not adapted, use virgin request to simplify
1323 // the code further below
1324 adapted_request_
= virgin_request_
;
1325 adapted_reply_
= dynamic_cast<HttpReply
*>(adapted
.header
);
1328 Adaptation::Icap::History::Pointer h
= virgin_request_
->icapHistory();
1329 Must(h
!= NULL
); // ICAPXaction::maybeLog calls only if there is a log
1330 al
.icp
.opcode
= ICP_INVALID
;
1331 al
.url
= h
->log_uri
.termedBuf();
1332 const Adaptation::Icap::ServiceRep
&s
= service();
1333 al
.icap
.reqMethod
= s
.cfg().method
;
1335 al
.cache
.caddr
= virgin_request_
->client_addr
;
1337 al
.request
= virgin_request_
;
1338 HTTPMSGLOCK(al
.request
);
1339 al
.adapted_request
= adapted_request_
;
1340 HTTPMSGLOCK(al
.adapted_request
);
1342 // XXX: This reply (and other ALE members!) may have been needed earlier.
1343 al
.reply
= adapted_reply_
;
1345 if (h
->rfc931
.size())
1346 al
.cache
.rfc931
= h
->rfc931
.termedBuf();
1349 if (h
->ssluser
.size())
1350 al
.cache
.ssluser
= h
->ssluser
.termedBuf();
1352 al
.cache
.code
= h
->logType
;
1354 const Http::Message
*virgin_msg
= dynamic_cast<HttpReply
*>(virgin
.header
);
1356 virgin_msg
= virgin_request_
;
1357 assert(virgin_msg
!= virgin
.cause
);
1358 al
.http
.clientRequestSz
.header
= virgin_msg
->hdr_sz
;
1359 if (virgin_msg
->body_pipe
!= NULL
)
1360 al
.http
.clientRequestSz
.payloadData
= virgin_msg
->body_pipe
->producedSize();
1362 // leave al.icap.bodyBytesRead negative if no body
1363 if (replyHttpHeaderSize
>= 0 || replyHttpBodySize
>= 0) {
1364 const int64_t zero
= 0; // to make max() argument types the same
1365 const uint64_t headerSize
= max(zero
, replyHttpHeaderSize
);
1366 const uint64_t bodySize
= max(zero
, replyHttpBodySize
);
1367 al
.icap
.bodyBytesRead
= headerSize
+ bodySize
;
1368 al
.http
.clientReplySz
.header
= headerSize
;
1369 al
.http
.clientReplySz
.payloadData
= bodySize
;
1372 if (adapted_reply_
) {
1373 al
.http
.code
= adapted_reply_
->sline
.status();
1374 al
.http
.content_type
= adapted_reply_
->content_type
.termedBuf();
1375 if (replyHttpBodySize
>= 0)
1376 al
.cache
.highOffset
= replyHttpBodySize
;
1377 //don't set al.cache.objectSize because it hasn't exist yet
1379 prepareLogWithRequestDetails(adapted_request_
, alep
);
1380 Xaction::finalizeLogInfo();
1383 void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf
&buf
)
1385 char ntoabuf
[MAX_IPSTRLEN
];
1387 * XXX These should use HttpHdr interfaces instead of Printfs
1389 const Adaptation::ServiceConfig
&s
= service().cfg();
1390 buf
.appendf("%s " SQUIDSTRINGPH
" ICAP/1.0\r\n", s
.methodStr(), SQUIDSTRINGPRINT(s
.uri
));
1391 buf
.appendf("Host: " SQUIDSTRINGPH
":%d\r\n", SQUIDSTRINGPRINT(s
.host
), s
.port
);
1392 buf
.appendf("Date: %s\r\n", mkrfc1123(squid_curtime
));
1394 if (!TheConfig
.reuse_connections
)
1395 buf
.appendf("Connection: close\r\n");
1397 const HttpRequest
*request
= &virginRequest();
1399 // we must forward "Proxy-Authenticate" and "Proxy-Authorization"
1401 if (virgin
.header
->header
.has(Http::HdrType::PROXY_AUTHENTICATE
)) {
1402 String vh
=virgin
.header
->header
.getById(Http::HdrType::PROXY_AUTHENTICATE
);
1403 buf
.appendf("Proxy-Authenticate: " SQUIDSTRINGPH
"\r\n",SQUIDSTRINGPRINT(vh
));
1406 if (virgin
.header
->header
.has(Http::HdrType::PROXY_AUTHORIZATION
)) {
1407 String vh
=virgin
.header
->header
.getById(Http::HdrType::PROXY_AUTHORIZATION
);
1408 buf
.appendf("Proxy-Authorization: " SQUIDSTRINGPH
"\r\n", SQUIDSTRINGPRINT(vh
));
1409 } else if (request
->extacl_user
.size() > 0 && request
->extacl_passwd
.size() > 0) {
1410 struct base64_encode_ctx ctx
;
1411 base64_encode_init(&ctx
);
1412 char base64buf
[base64_encode_len(MAX_LOGIN_SZ
)];
1413 size_t resultLen
= base64_encode_update(&ctx
, base64buf
, request
->extacl_user
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_user
.rawBuf()));
1414 resultLen
+= base64_encode_update(&ctx
, base64buf
+resultLen
, 1, reinterpret_cast<const uint8_t*>(":"));
1415 resultLen
+= base64_encode_update(&ctx
, base64buf
+resultLen
, request
->extacl_passwd
.size(), reinterpret_cast<const uint8_t*>(request
->extacl_passwd
.rawBuf()));
1416 resultLen
+= base64_encode_final(&ctx
, base64buf
+resultLen
);
1417 buf
.appendf("Proxy-Authorization: Basic %.*s\r\n", (int)resultLen
, base64buf
);
1420 // share the cross-transactional database records if needed
1421 if (Adaptation::Config::masterx_shared_name
) {
1422 Adaptation::History::Pointer ah
= request
->adaptHistory(false);
1425 if (ah
->getXxRecord(name
, value
)) {
1426 buf
.appendf(SQUIDSTRINGPH
": " SQUIDSTRINGPH
"\r\n", SQUIDSTRINGPRINT(name
), SQUIDSTRINGPRINT(value
));
1431 buf
.append("Encapsulated: ", 14);
1437 // build HTTP request header, if any
1438 ICAP::Method m
= s
.method
;
1440 // to simplify, we could assume that request is always available
1443 if (ICAP::methodRespmod
== m
)
1444 encapsulateHead(buf
, "req-hdr", httpBuf
, request
);
1445 else if (ICAP::methodReqmod
== m
)
1446 encapsulateHead(buf
, "req-hdr", httpBuf
, virgin
.header
);
1449 if (ICAP::methodRespmod
== m
)
1450 if (const Http::Message
*prime
= virgin
.header
)
1451 encapsulateHead(buf
, "res-hdr", httpBuf
, prime
);
1453 if (!virginBody
.expected())
1454 buf
.appendf("null-body=%d", (int) httpBuf
.contentSize());
1455 else if (ICAP::methodReqmod
== m
)
1456 buf
.appendf("req-body=%d", (int) httpBuf
.contentSize());
1458 buf
.appendf("res-body=%d", (int) httpBuf
.contentSize());
1460 buf
.append(ICAP::crlf
, 2); // terminate Encapsulated line
1462 if (preview
.enabled()) {
1463 buf
.appendf("Preview: %d\r\n", (int)preview
.ad());
1464 if (!virginBody
.expected()) // there is no body to preview
1465 finishNullOrEmptyBodyPreview(httpBuf
);
1468 makeAllowHeader(buf
);
1470 if (TheConfig
.send_client_ip
&& request
) {
1471 Ip::Address client_addr
;
1472 #if FOLLOW_X_FORWARDED_FOR
1473 if (TheConfig
.use_indirect_client
) {
1474 client_addr
= request
->indirect_client_addr
;
1477 client_addr
= request
->client_addr
;
1478 if (!client_addr
.isAnyAddr() && !client_addr
.isNoAddr())
1479 buf
.appendf("X-Client-IP: %s\r\n", client_addr
.toStr(ntoabuf
,MAX_IPSTRLEN
));
1482 if (TheConfig
.send_username
&& request
)
1483 makeUsernameHeader(request
, buf
);
1485 // Adaptation::Config::metaHeaders
1486 for (auto h
: Adaptation::Config::metaHeaders
) {
1487 HttpRequest
*r
= virgin
.cause
?
1488 virgin
.cause
: dynamic_cast<HttpRequest
*>(virgin
.header
);
1491 HttpReply
*reply
= dynamic_cast<HttpReply
*>(virgin
.header
);
1494 if (h
->match(r
, reply
, alMaster
, matched
)) {
1495 buf
.append(h
->key().rawContent(), h
->key().length());
1496 buf
.append(": ", 2);
1497 buf
.append(matched
.rawContent(), matched
.length());
1498 buf
.append("\r\n", 2);
1499 Adaptation::History::Pointer ah
= request
->adaptHistory(false);
1501 if (ah
->metaHeaders
== NULL
)
1502 ah
->metaHeaders
= new NotePairs
;
1503 if (!ah
->metaHeaders
->hasPair(h
->key(), matched
))
1504 ah
->metaHeaders
->add(h
->key(), matched
);
1509 // fprintf(stderr, "%s\n", buf.content());
1511 buf
.append(ICAP::crlf
, 2); // terminate ICAP header
1513 // fill icapRequest for logging
1514 Must(icapRequest
->parseCharBuf(buf
.content(), buf
.contentSize()));
1516 // start ICAP request body with encapsulated HTTP headers
1517 buf
.append(httpBuf
.content(), httpBuf
.contentSize());
1522 // decides which Allow values to write and updates the request buffer
1523 void Adaptation::Icap::ModXact::makeAllowHeader(MemBuf
&buf
)
1525 const bool allow204in
= preview
.enabled(); // TODO: add shouldAllow204in()
1526 const bool allow204out
= state
.allowedPostview204
= shouldAllow204();
1527 const bool allow206in
= state
.allowedPreview206
= shouldAllow206in();
1528 const bool allow206out
= state
.allowedPostview206
= shouldAllow206out();
1529 const bool allowTrailers
= true; // TODO: make configurable
1531 debugs(93, 9, "Allows: " << allow204in
<< allow204out
<<
1532 allow206in
<< allow206out
<< allowTrailers
);
1534 const bool allow204
= allow204in
|| allow204out
;
1535 const bool allow206
= allow206in
|| allow206out
;
1537 if ((allow204
|| allow206
) && virginBody
.expected())
1538 virginBodySending
.plan(); // if there is a virgin body, plan to send it
1540 // writing Preview:... means we will honor 204 inside preview
1541 // writing Allow/204 means we will honor 204 outside preview
1542 // writing Allow:206 means we will honor 206 inside preview
1543 // writing Allow:204,206 means we will honor 206 outside preview
1544 if (allow204
|| allow206
|| allowTrailers
) {
1545 buf
.appendf("Allow: ");
1547 buf
.appendf("204, ");
1549 buf
.appendf("206, ");
1551 buf
.appendf("trailers");
1552 buf
.appendf("\r\n");
1556 void Adaptation::Icap::ModXact::makeUsernameHeader(const HttpRequest
*request
, MemBuf
&buf
)
1559 struct base64_encode_ctx ctx
;
1560 base64_encode_init(&ctx
);
1562 const char *value
= NULL
;
1563 if (request
->auth_user_request
!= NULL
) {
1564 value
= request
->auth_user_request
->username();
1565 } else if (request
->extacl_user
.size() > 0) {
1566 value
= request
->extacl_user
.termedBuf();
1570 if (TheConfig
.client_username_encode
) {
1571 char base64buf
[base64_encode_len(MAX_LOGIN_SZ
)];
1572 size_t resultLen
= base64_encode_update(&ctx
, base64buf
, strlen(value
), reinterpret_cast<const uint8_t*>(value
));
1573 resultLen
+= base64_encode_final(&ctx
, base64buf
+resultLen
);
1574 buf
.appendf("%s: %.*s\r\n", TheConfig
.client_username_header
, (int)resultLen
, base64buf
);
1576 buf
.appendf("%s: %s\r\n", TheConfig
.client_username_header
, value
);
1585 Adaptation::Icap::ModXact::encapsulateHead(MemBuf
&icapBuf
, const char *section
, MemBuf
&httpBuf
, const Http::Message
*head
)
1587 // update ICAP header
1588 icapBuf
.appendf("%s=%d, ", section
, (int) httpBuf
.contentSize());
1591 Http::MessagePointer headClone
;
1593 if (const HttpRequest
* old_request
= dynamic_cast<const HttpRequest
*>(head
)) {
1594 HttpRequest::Pointer
new_request(new HttpRequest(old_request
->masterXaction
));
1595 // copy the request-line details
1596 new_request
->method
= old_request
->method
;
1597 new_request
->url
= old_request
->url
;
1598 new_request
->http_ver
= old_request
->http_ver
;
1599 headClone
= new_request
.getRaw();
1600 } else if (const HttpReply
*old_reply
= dynamic_cast<const HttpReply
*>(head
)) {
1601 HttpReply::Pointer
new_reply(new HttpReply
);
1602 new_reply
->sline
= old_reply
->sline
;
1603 headClone
= new_reply
.getRaw();
1606 headClone
->inheritProperties(head
);
1608 HttpHeaderPos pos
= HttpHeaderInitPos
;
1609 while (HttpHeaderEntry
* p_head_entry
= head
->header
.getEntry(&pos
))
1610 headClone
->header
.addEntry(p_head_entry
->clone());
1614 // remove all hop-by-hop headers from the clone
1615 headClone
->header
.delById(Http::HdrType::PROXY_AUTHENTICATE
);
1616 headClone
->header
.removeHopByHopEntries();
1618 // TODO: modify HttpHeader::removeHopByHopEntries to accept a list of
1619 // excluded hop-by-hop headers
1620 if (head
->header
.has(Http::HdrType::UPGRADE
)) {
1621 const auto upgrade
= head
->header
.getList(Http::HdrType::UPGRADE
);
1622 headClone
->header
.putStr(Http::HdrType::UPGRADE
, upgrade
.termedBuf());
1625 // pack polished HTTP header
1626 packHead(httpBuf
, headClone
.getRaw());
1628 // headClone unlocks and, hence, deletes the message we packed
1632 Adaptation::Icap::ModXact::packHead(MemBuf
&httpBuf
, const Http::Message
*head
)
1634 head
->packInto(&httpBuf
, true);
1637 // decides whether to offer a preview and calculates its size
1638 void Adaptation::Icap::ModXact::decideOnPreview()
1640 if (!TheConfig
.preview_enable
) {
1641 debugs(93, 5, HERE
<< "preview disabled by squid.conf");
1645 const SBuf
urlPath(virginRequest().url
.path());
1647 if (!service().wantsPreview(urlPath
, wantedSize
)) {
1648 debugs(93, 5, "should not offer preview for " << urlPath
);
1652 // we decided to do preview, now compute its size
1654 // cannot preview more than we can backup
1655 size_t ad
= min(wantedSize
, TheBackupLimit
);
1657 if (!virginBody
.expected())
1659 else if (virginBody
.knownSize())
1660 ad
= min(static_cast<uint64_t>(ad
), virginBody
.size()); // not more than we have
1662 debugs(93, 5, HERE
<< "should offer " << ad
<< "-byte preview " <<
1663 "(service wanted " << wantedSize
<< ")");
1666 Must(preview
.enabled());
1669 // decides whether to allow 204 responses
1670 bool Adaptation::Icap::ModXact::shouldAllow204()
1672 if (!service().allows204())
1675 return canBackupEverything();
1678 // decides whether to allow 206 responses in some mode
1679 bool Adaptation::Icap::ModXact::shouldAllow206any()
1681 return TheConfig
.allow206_enable
&& service().allows206() &&
1682 virginBody
.expected(); // no need for 206 without a body
1685 // decides whether to allow 206 responses in preview mode
1686 bool Adaptation::Icap::ModXact::shouldAllow206in()
1688 return shouldAllow206any() && preview
.enabled();
1691 // decides whether to allow 206 responses outside of preview
1692 bool Adaptation::Icap::ModXact::shouldAllow206out()
1694 return shouldAllow206any() && canBackupEverything();
1697 // used by shouldAllow204 and decideOnRetries
1698 bool Adaptation::Icap::ModXact::canBackupEverything() const
1700 if (!virginBody
.expected())
1701 return true; // no body means no problems with backup
1703 // if there is a body, check whether we can backup it all
1705 if (!virginBody
.knownSize())
1708 // or should we have a different backup limit?
1709 // note that '<' allows for 0-termination of the "full" backup buffer
1710 return virginBody
.size() < TheBackupLimit
;
1713 // Decide whether this transaction can be retried if pconn fails
1714 // Must be called after decideOnPreview and before openConnection()
1715 void Adaptation::Icap::ModXact::decideOnRetries()
1718 return; // no, already decided
1720 if (preview
.enabled())
1721 return; // yes, because preview provides enough guarantees
1723 if (canBackupEverything())
1724 return; // yes, because we can back everything up
1726 disableRetries(); // no, because we cannot back everything up
1729 // Normally, the body-writing code handles preview body. It can deal with
1730 // bodies of unexpected size, including those that turn out to be empty.
1731 // However, that code assumes that the body was expected and body control
1732 // structures were initialized. This is not the case when there is no body
1733 // or the body is known to be empty, because the virgin message will lack a
1734 // body_pipe. So we handle preview of null-body and zero-size bodies here.
1735 void Adaptation::Icap::ModXact::finishNullOrEmptyBodyPreview(MemBuf
&)
1737 Must(!virginBodyWriting
.active()); // one reason we handle it here
1738 Must(!virgin
.body_pipe
); // another reason we handle it here
1739 Must(!preview
.ad());
1741 // do not add last-chunk because our Encapsulated header says null-body
1742 // addLastRequestChunk(buf);
1743 preview
.wrote(0, true);
1745 Must(preview
.done());
1746 Must(preview
.ieof());
1749 void Adaptation::Icap::ModXact::fillPendingStatus(MemBuf
&buf
) const
1751 Adaptation::Icap::Xaction::fillPendingStatus(buf
);
1753 if (state
.serviceWaiting
)
1756 if (virgin
.body_pipe
!= NULL
)
1759 if (haveConnection() && !doneReading())
1762 if (!state
.doneWriting() && state
.writing
!= State::writingInit
)
1763 buf
.appendf("w(%d)", state
.writing
);
1765 if (preview
.enabled()) {
1766 if (!preview
.done())
1767 buf
.appendf("P(%d)", (int) preview
.debt());
1770 if (virginBodySending
.active())
1773 if (!state
.doneParsing() && state
.parsing
!= State::psIcapHeader
)
1774 buf
.appendf("p(%d)", state
.parsing
);
1776 if (!doneSending() && state
.sending
!= State::sendingUndecided
)
1777 buf
.appendf("S(%d)", state
.sending
);
1779 if (state
.readyForUob
)
1785 if (protectGroupBypass
)
1789 void Adaptation::Icap::ModXact::fillDoneStatus(MemBuf
&buf
) const
1791 Adaptation::Icap::Xaction::fillDoneStatus(buf
);
1793 if (!virgin
.body_pipe
)
1796 if (state
.doneWriting())
1799 if (preview
.enabled()) {
1801 buf
.appendf("P%s", preview
.ieof() ? "(ieof)" : "");
1807 if (state
.doneParsing())
1814 bool Adaptation::Icap::ModXact::gotEncapsulated(const char *section
) const
1816 return !icapReply
->header
.getByNameListMember("Encapsulated",
1817 section
, ',').isEmpty();
1820 // calculate whether there is a virgin HTTP body and
1821 // whether its expected size is known
1822 // TODO: rename because we do not just estimate
1823 void Adaptation::Icap::ModXact::estimateVirginBody()
1825 // note: lack of size info may disable previews and 204s
1827 Http::Message
*msg
= virgin
.header
;
1830 HttpRequestMethod method
;
1833 method
= virgin
.cause
->method
;
1834 else if (HttpRequest
*req
= dynamic_cast<HttpRequest
*>(msg
))
1835 method
= req
->method
;
1837 method
= Http::METHOD_NONE
;
1840 // expectingBody returns true for zero-sized bodies, but we will not
1841 // get a pipe for that body, so we treat the message as bodyless
1842 if (method
!= Http::METHOD_NONE
&& msg
->expectingBody(method
, size
) && size
) {
1843 debugs(93, 6, HERE
<< "expects virgin body from " <<
1844 virgin
.body_pipe
<< "; size: " << size
);
1846 virginBody
.expect(size
);
1847 virginBodyWriting
.plan();
1849 // sign up as a body consumer
1850 Must(msg
->body_pipe
!= NULL
);
1851 Must(msg
->body_pipe
== virgin
.body_pipe
);
1852 Must(virgin
.body_pipe
->setConsumerIfNotLate(this));
1854 // make sure TheBackupLimit is in-sync with the buffer size
1855 Must(TheBackupLimit
<= static_cast<size_t>(msg
->body_pipe
->buf().max_capacity
));
1857 debugs(93, 6, HERE
<< "does not expect virgin body");
1858 Must(msg
->body_pipe
== NULL
);
1863 void Adaptation::Icap::ModXact::makeAdaptedBodyPipe(const char *what
)
1865 Must(!adapted
.body_pipe
);
1866 Must(!adapted
.header
->body_pipe
);
1867 adapted
.header
->body_pipe
= new BodyPipe(this);
1868 adapted
.body_pipe
= adapted
.header
->body_pipe
;
1869 debugs(93, 7, HERE
<< "will supply " << what
<< " via " <<
1870 adapted
.body_pipe
<< " pipe");
1873 // TODO: Move SizedEstimate and Preview elsewhere
1875 Adaptation::Icap::SizedEstimate::SizedEstimate()
1876 : theData(dtUnexpected
)
1879 void Adaptation::Icap::SizedEstimate::expect(int64_t aSize
)
1881 theData
= (aSize
>= 0) ? aSize
: (int64_t)dtUnknown
;
1884 bool Adaptation::Icap::SizedEstimate::expected() const
1886 return theData
!= dtUnexpected
;
1889 bool Adaptation::Icap::SizedEstimate::knownSize() const
1892 return theData
!= dtUnknown
;
1895 uint64_t Adaptation::Icap::SizedEstimate::size() const
1898 return static_cast<uint64_t>(theData
);
1901 Adaptation::Icap::VirginBodyAct::VirginBodyAct(): theStart(0), theState(stUndecided
)
1904 void Adaptation::Icap::VirginBodyAct::plan()
1907 Must(!theStart
); // not started
1908 theState
= stActive
;
1911 void Adaptation::Icap::VirginBodyAct::disable()
1913 theState
= stDisabled
;
1916 void Adaptation::Icap::VirginBodyAct::progress(size_t size
)
1919 #if SIZEOF_SIZE_T > 4
1920 /* always true for smaller size_t's */
1921 Must(static_cast<int64_t>(size
) >= 0);
1923 theStart
+= static_cast<int64_t>(size
);
1926 uint64_t Adaptation::Icap::VirginBodyAct::offset() const
1929 return static_cast<uint64_t>(theStart
);
1932 Adaptation::Icap::Preview::Preview(): theWritten(0), theAd(0), theState(stDisabled
)
1935 void Adaptation::Icap::Preview::enable(size_t anAd
)
1937 // TODO: check for anAd not exceeding preview size limit
1940 theState
= stWriting
;
1943 bool Adaptation::Icap::Preview::enabled() const
1945 return theState
!= stDisabled
;
1948 size_t Adaptation::Icap::Preview::ad() const
1954 bool Adaptation::Icap::Preview::done() const
1957 return theState
>= stIeof
;
1960 bool Adaptation::Icap::Preview::ieof() const
1963 return theState
== stIeof
;
1966 size_t Adaptation::Icap::Preview::debt() const
1969 return done() ? 0 : (theAd
- theWritten
);
1972 void Adaptation::Icap::Preview::wrote(size_t size
, bool wroteEof
)
1978 Must(theWritten
<= theAd
);
1981 theState
= stIeof
; // written size is irrelevant
1982 else if (theWritten
>= theAd
)
1986 bool Adaptation::Icap::ModXact::fillVirginHttpHeader(MemBuf
&mb
) const
1988 if (virgin
.header
== NULL
)
1991 virgin
.header
->firstLineBuf(mb
);
1996 void Adaptation::Icap::ModXact::detailError(const ErrorDetail::Pointer
&errDetail
)
1998 HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
1999 // if no adapted request, update virgin (and inherit its properties later)
2000 // TODO: make this and HttpRequest::detailError constant, like adaptHistory
2002 request
= const_cast<HttpRequest
*>(&virginRequest());
2005 request
->detailError(ERR_ICAP_FAILURE
, errDetail
);
2008 void Adaptation::Icap::ModXact::clearError()
2010 HttpRequest
*request
= dynamic_cast<HttpRequest
*>(adapted
.header
);
2011 // if no adapted request, update virgin (and inherit its properties later)
2013 request
= const_cast<HttpRequest
*>(&virginRequest());
2016 request
->clearError();
2019 void Adaptation::Icap::ModXact::updateSources()
2021 Must(adapted
.header
);
2022 adapted
.header
->sources
|= (service().cfg().connectionEncryption
? Http::Message::srcIcaps
: Http::Message::srcIcap
);
2025 /* Adaptation::Icap::ModXactLauncher */
2027 Adaptation::Icap::ModXactLauncher::ModXactLauncher(Http::Message
*virginHeader
, HttpRequest
*virginCause
, AccessLogEntry::Pointer
&alp
, Adaptation::ServicePointer aService
):
2028 AsyncJob("Adaptation::Icap::ModXactLauncher"),
2029 Adaptation::Icap::Launcher("Adaptation::Icap::ModXactLauncher", aService
),
2032 virgin
.setHeader(virginHeader
);
2033 virgin
.setCause(virginCause
);
2034 updateHistory(true);
2037 Adaptation::Icap::Xaction
*Adaptation::Icap::ModXactLauncher::createXaction()
2039 Adaptation::Icap::ServiceRep::Pointer s
=
2040 dynamic_cast<Adaptation::Icap::ServiceRep
*>(theService
.getRaw());
2042 return new Adaptation::Icap::ModXact(virgin
.header
, virgin
.cause
, al
, s
);
2045 void Adaptation::Icap::ModXactLauncher::swanSong()
2047 debugs(93, 5, HERE
<< "swan sings");
2048 updateHistory(false);
2049 Adaptation::Icap::Launcher::swanSong();
2052 void Adaptation::Icap::ModXactLauncher::updateHistory(bool doStart
)
2054 HttpRequest
*r
= virgin
.cause
?
2055 virgin
.cause
: dynamic_cast<HttpRequest
*>(virgin
.header
);
2057 // r should never be NULL but we play safe; TODO: add Should()
2059 Adaptation::Icap::History::Pointer h
= r
->icapHistory();
2062 h
->start("ICAPModXactLauncher");
2064 h
->stop("ICAPModXactLauncher");
2069 bool Adaptation::Icap::TrailerParser::parse(const char *buf
, int len
, int atEnd
, Http::StatusCode
*error
) {
2070 Http::ContentLengthInterpreter clen
;
2071 // RFC 7230 section 4.1.2: MUST NOT generate a trailer that contains
2072 // a field necessary for message framing (e.g., Transfer-Encoding and Content-Length)
2073 clen
.applyTrailerRules();
2074 const int parsed
= trailer
.parse(buf
, len
, atEnd
, hdr_sz
, clen
);
2076 *error
= Http::scInvalidHeader
; // TODO: should we add a new Http::scInvalidTrailer?
2081 Adaptation::Icap::ChunkExtensionValueParser::parse(Tokenizer
&tok
, const SBuf
&extName
)
2083 if (extName
== UseOriginalBodyName
) {
2084 useOriginalBody_
= tok
.udec64("use-original-body");
2085 assert(useOriginalBody_
>= 0);
2087 Ignore(tok
, extName
);