return bodySize() - thePutSize; // bodySize() asserts that size is known
}
+void BodyPipe::expectProductionEndAfter(uint64_t size)
+{
+ const uint64_t expectedSize = thePutSize + size;
+ if (bodySizeKnown())
+ Must(bodySize() == expectedSize);
+ else
+ theBodySize = expectedSize;
+}
+
void
BodyPipe::clearProducer(bool atEof)
{
bool needsMoreData() const { return bodySizeKnown() && unproducedSize() > 0; }
uint64_t unproducedSize() const; // size of still unproduced data
bool stillProducing(const Producer *producer) const { return theProducer == producer; }
+ void expectProductionEndAfter(uint64_t extraSize); ///< sets or checks body size
// called by consumers
bool setConsumerIfNotLate(Consumer *aConsumer);
theChunkSize = theLeftBodySize = 0;
doNeedMoreData = false;
theIn = theOut = NULL;
+ useOriginBody = -1;
}
bool ChunkedCodingParser::parse(MemBuf *rawData, MemBuf *parsedContent)
return;
}
+ // to allow chunk extensions in any chunk, remove this size check
+ if (size == 0) // is this the last-chunk?
+ parseChunkExtension(p, theIn->content() + crlfBeg);
+
theIn->consume(crlfEnd);
theChunkSize = theLeftBodySize = size;
debugs(94,7, "found chunk: " << theChunkSize);
return false;
}
+// chunk-extension= *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+void ChunkedCodingParser::parseChunkExtension(const char *startExt, const char *endExt)
+{
+ // chunk-extension starts at startExt and ends with LF at endEx
+ for (const char *p = startExt; p < endExt;) {
+
+ while (*p == ' ' || *p == '\t') ++p; // skip spaces before ';'
+
+ if (*p++ != ';') // each ext name=value pair is preceded with ';'
+ return;
+
+ while (*p == ' ' || *p == '\t') ++p; // skip spaces before name
+
+ if (p >= endExt)
+ return; // malformed extension: ';' without ext name=value pair
+
+ const int extSize = endExt - p;
+ // TODO: we need debugData() stream manipulator to dump data
+ debugs(94,7, "Found chunk extension; size=" << extSize);
+
+ // TODO: support implied *LWS around '='
+ if (extSize > 18 && strncmp(p, "use-original-body=", 18) == 0) {
+ (void)StringToInt64(p+18, useOriginBody, &p, 10);
+ debugs(94, 3, HERE << "use-original-body=" << useOriginBody);
+ return; // remove to support more than just use-original-body
+ } else {
+ debugs(94, 5, HERE << "skipping unknown chunk extension");
+ // TODO: support quoted-string chunk-ext-val
+ while (p < endExt && *p != ';') ++p; // skip until the next ';'
+ }
+ }
+}
void parseTrailerHeader();
void parseMessageEnd();
+ void parseChunkExtension(const char *, const char * );
bool findCrlf(size_t &crlfBeg, size_t &crlfEnd);
private:
uint64_t theChunkSize;
uint64_t theLeftBodySize;
bool doNeedMoreData;
+
+public:
+ int64_t useOriginBody;
};
#endif /* SQUID_CHUNKEDCODINGPARSER_H */
Adaptation::Icap::Config Adaptation::Icap::TheConfig;
Adaptation::Icap::Config::Config(): preview_enable(0), preview_size(0),
+ allow206_enable(0),
connect_timeout_raw(0), io_timeout_raw(0), reuse_connections(0),
client_username_header(NULL), client_username_encode(0), repeat(NULL)
{
int default_options_ttl;
int preview_enable;
int preview_size;
+ int allow206_enable;
time_t connect_timeout_raw;
time_t io_timeout_raw;
int reuse_connections;
const XactOutcome xoError = "ICAP_ERR_OTHER";
const XactOutcome xoOpt = "ICAP_OPT";
const XactOutcome xoEcho = "ICAP_ECHO";
+const XactOutcome xoPartEcho = "ICAP_PART_ECHO";
const XactOutcome xoModified = "ICAP_MOD";
const XactOutcome xoSatisfied = "ICAP_SAT";
extern const XactOutcome xoError; ///< all kinds of transaction errors
extern const XactOutcome xoOpt; ///< OPTION transaction
extern const XactOutcome xoEcho; ///< preserved virgin message (ICAP 204)
+extern const XactOutcome xoPartEcho; ///< preserved virgin msg part (ICAP 206)
extern const XactOutcome xoModified; ///< replaced virgin msg with adapted
extern const XactOutcome xoSatisfied; ///< request satisfaction
handle204NoContent();
break;
+ case 206:
+ handle206PartialContent();
+ break;
+
default:
debugs(93, 5, HERE << "ICAP status " << icapReply->sline.status);
handleUnknownScode();
// server must not respond before the end of preview: we may send ieof
Must(preview.enabled() && preview.done() && !preview.ieof());
- // 100 "Continue" cancels our preview commitment, not 204s outside preview
- if (!state.allowedPostview204)
+ // 100 "Continue" cancels our Preview commitment,
+ // but not commitment to handle 204 or 206 outside Preview
+ if (!state.allowedPostview204 && !state.allowedPostview206)
stopBackup();
state.parsing = State::psIcapHeader; // eventually
prepEchoing();
}
+void Adaptation::Icap::ModXact::handle206PartialContent()
+{
+ if (state.writing == State::writingPaused) {
+ Must(preview.enabled());
+ Must(state.allowedPreview206);
+ debugs(93, 7, HERE << "206 inside preview");
+ } else {
+ Must(state.writing > State::writingPaused);
+ Must(state.allowedPostview206);
+ debugs(93, 7, HERE << "206 outside preview");
+ }
+ state.parsing = State::psHttpHeader;
+ state.sending = State::sendingAdapted;
+ state.readyForUob = true;
+ checkConsuming();
+}
+
// Called when we receive a 204 No Content response and
// when we are trying to bypass a service failure.
// We actually start sending (echoig or not) in startSending.
}
}
+/// Called when we received use-original-body chunk extension in 206 response.
+/// We actually start sending (echoing or not) in startSending().
+void Adaptation::Icap::ModXact::prepPartialBodyEchoing(uint64_t pos)
+{
+ Must(virginBodySending.active());
+ Must(virgin.header->body_pipe != NULL);
+
+ setOutcome(xoPartEcho);
+
+ debugs(93, 7, HERE << "will echo virgin body suffix from " <<
+ virgin.header->body_pipe << " offset " << pos );
+
+ // check that use-original-body=N does not point beyond buffered data
+ const uint64_t virginDataEnd = virginConsumed +
+ virgin.body_pipe->buf().contentSize();
+ Must(pos <= virginDataEnd);
+ virginBodySending.progress(static_cast<size_t>(pos));
+
+ state.sending = State::sendingVirgin;
+ checkConsuming();
+
+ if (virgin.header->body_pipe->bodySizeKnown())
+ adapted.body_pipe->expectProductionEndAfter(virgin.header->body_pipe->bodySize() - pos);
+
+ debugs(93, 7, HERE << "will echo virgin body suffix to " <<
+ adapted.body_pipe);
+
+ // Start echoing data
+ echoMore();
+}
+
void Adaptation::Icap::ModXact::handleUnknownScode()
{
stopParsing();
}
if (parsed) {
+ if (state.readyForUob && bodyParser->useOriginBody >= 0) {
+ prepPartialBodyEchoing(
+ static_cast<uint64_t>(bodyParser->useOriginBody));
+ stopParsing();
+ return;
+ }
+
stopParsing();
stopSending(true); // the parser succeeds only if all parsed data fits
return;
if (preview.enabled()) {
buf.Printf("Preview: %d\r\n", (int)preview.ad());
- if (virginBody.expected()) // there is a body to preview
- virginBodySending.plan();
- else
+ if (!virginBody.expected()) // there is no body to preview
finishNullOrEmptyBodyPreview(httpBuf);
}
- if (shouldAllow204()) {
- debugs(93,5, HERE << "will allow 204s outside of preview");
- state.allowedPostview204 = true;
- buf.Printf("Allow: 204\r\n");
- if (virginBody.expected()) // there is a body to echo
- virginBodySending.plan();
- }
+ makeAllowHeader(buf);
if (TheConfig.send_client_ip && request) {
Ip::Address client_addr;
httpBuf.clean();
}
+// decides which Allow values to write and updates the request buffer
+void Adaptation::Icap::ModXact::makeAllowHeader(MemBuf &buf)
+{
+ const bool allow204in = preview.enabled(); // TODO: add shouldAllow204in()
+ const bool allow204out = state.allowedPostview204 = shouldAllow204();
+ const bool allow206in = state.allowedPreview206 = shouldAllow206in();
+ const bool allow206out = state.allowedPostview206 = shouldAllow206out();
+
+ debugs(93,9, HERE << "Allows: " << allow204in << allow204out <<
+ allow206in << allow206out);
+
+ const bool allow204 = allow204in || allow204out;
+ const bool allow206 = allow206in || allow206out;
+
+ if (!allow204 && !allow206)
+ return; // nothing to do
+
+ if (virginBody.expected()) // if there is a virgin body, plan to send it
+ virginBodySending.plan();
+
+ // writing Preview:... means we will honor 204 inside preview
+ // writing Allow/204 means we will honor 204 outside preview
+ // writing Allow:206 means we will honor 206 inside preview
+ // writing Allow:204,206 means we will honor 206 outside preview
+ const char *allowHeader = NULL;
+ if (allow204out && allow206)
+ allowHeader = "Allow: 204, 206\r\n";
+ else if (allow204out)
+ allowHeader = "Allow: 204\r\n";
+ else if (allow206)
+ allowHeader = "Allow: 206\r\n";
+
+ if (allowHeader) { // may be nil if only allow204in is true
+ buf.append(allowHeader, strlen(allowHeader));
+ debugs(93,5, HERE << "Will write " << allowHeader);
+ }
+}
+
void Adaptation::Icap::ModXact::makeUsernameHeader(const HttpRequest *request, MemBuf &buf)
{
if (request->auth_user_request != NULL) {
return canBackupEverything();
}
+// decides whether to allow 206 responses in some mode
+bool Adaptation::Icap::ModXact::shouldAllow206any()
+{
+ return TheConfig.allow206_enable && service().allows206() &&
+ virginBody.expected(); // no need for 206 without a body
+}
+
+// decides whether to allow 206 responses in preview mode
+bool Adaptation::Icap::ModXact::shouldAllow206in()
+{
+ return shouldAllow206any() && preview.enabled();
+}
+
+// decides whether to allow 206 responses outside of preview
+bool Adaptation::Icap::ModXact::shouldAllow206out()
+{
+ return shouldAllow206any() && canBackupEverything();
+}
+
// used by shouldAllow204 and decideOnRetries
bool Adaptation::Icap::ModXact::canBackupEverything() const
{
if (!doneSending() && state.sending != State::sendingUndecided)
buf.Printf("S(%d)", state.sending);
+ if (state.readyForUob)
+ buf.append("6", 1);
+
if (canStartBypass)
buf.append("Y", 1);
bool virginBodyEndReached(const VirginBodyAct &act) const;
void makeRequestHeaders(MemBuf &buf);
+ void makeAllowHeader(MemBuf &buf);
void makeUsernameHeader(const HttpRequest *request, MemBuf &buf);
void addLastRequestChunk(MemBuf &buf);
void openChunk(MemBuf &buf, size_t chunkSize, bool ieof);
void decideOnPreview();
void decideOnRetries();
bool shouldAllow204();
+ bool shouldAllow206any();
+ bool shouldAllow206in();
+ bool shouldAllow206out();
bool canBackupEverything() const;
void prepBackup(size_t expectedSize);
bool validate200Ok();
void handle200Ok();
void handle204NoContent();
+ void handle206PartialContent();
void handleUnknownScode();
void bypassFailure();
void disableBypass(const char *reason, bool includeGroupBypass);
void prepEchoing();
+ void prepPartialBodyEchoing(uint64_t pos);
void echoMore();
virtual bool doneAll() const;
bool serviceWaiting; // waiting for ICAP service options
bool allowedPostview204; // mmust handle 204 No Content outside preview
+ bool allowedPostview206; // must handle 206 Partial Content outside preview
+ bool allowedPreview206; // must handle 206 Partial Content inside preview
+ bool readyForUob; ///< got a 206 response and expect a use-origin-body
// will not write anything [else] to the ICAP server connection
bool doneWriting() const { return writing == writingReallyDone; }
// will not use virgin.body_pipe
bool doneConsumingVirgin() const {
return writing >= writingAlmostDone
- && (sending == sendingAdapted || sending == sendingDone);
+ && ((sending == sendingAdapted && !readyForUob) ||
+ sending == sendingDone);
}
// parsed entire ICAP response from the ICAP server
#include "adaptation/icap/OptXact.h"
#include "adaptation/icap/Options.h"
+#include "adaptation/icap/Config.h"
#include "base/TextException.h"
#include "SquidTime.h"
#include "HttpRequest.h"
buf.Printf("OPTIONS " SQUIDSTRINGPH " ICAP/1.0\r\n", SQUIDSTRINGPRINT(uri));
const String host = s.cfg().host;
buf.Printf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(host), s.cfg().port);
+ if (TheConfig.allow206_enable)
+ buf.Printf("Allow: 206\r\n");
buf.append(ICAP::crlf, 2);
// XXX: HttpRequest cannot fully parse ICAP Request-Line
Adaptation::Icap::Options::Options(): error("unconfigured"),
max_connections(-1), allow204(false),
+ allow206(false),
preview(-1), theTTL(-1)
{
theTransfers.preview.name = "Transfer-Preview";
if (h->hasListMember(HDR_ALLOW, "204", ','))
allow204 = true;
+ if (h->hasListMember(HDR_ALLOW, "206", ','))
+ allow206 = true;
+
cfgIntHeader(h, "Preview", preview);
cfgTransferList(h, theTransfers.preview);
String serviceId;
int max_connections;
bool allow204;
+ bool allow206;
int preview;
protected:
return true; // in the future, we may have ACLs to prevent 204s
}
+bool Adaptation::Icap::ServiceRep::allows206() const
+{
+ Must(hasOptions());
+ if (theOptions->allow206)
+ return true; // in the future, we may have ACLs to prevent 206s
+ return false;
+}
+
static
void ServiceRep_noteTimeToUpdate(void *data)
bool wantsUrl(const String &urlPath) const;
bool wantsPreview(const String &urlPath, size_t &wantedSize) const;
bool allows204() const;
+ bool allows206() const;
void noteFailure(); // called by transactions to report service failure
basis by OPTIONS requests.
DOC_END
+NAME: icap_206_enable
+TYPE: onoff
+IFDEF: ICAP_CLIENT
+COMMENT: on|off
+LOC: Adaptation::Icap::TheConfig.allow206_enable
+DEFAULT: on
+DOC_START
+ 206 (Partial Content) responses is an ICAP extension that allows the
+ ICAP agents to optionally combine adapted and original HTTP message
+ content. The decision to combine is postponed until the end of the
+ ICAP response. Squid supports Partial Content extension by default.
+
+ Activation of the Partial Content extension is negotiated with each
+ ICAP service during OPTIONS exchange. Most ICAP servers should handle
+ negotation correctly even if they do not support the extension, but
+ some might fail. To disable Partial Content support for all ICAP
+ services and to avoid any negotiation, set this option to "off".
+
+ Example:
+ icap_206_enable off
+DOC_END
+
NAME: icap_default_options_ttl
TYPE: int
IFDEF: ICAP_CLIENT