From: Christos Tsantilas <chtsanti@users.sourceforge.net>
Date: Mon, 14 Jun 2010 20:01:59 +0000 (+0300)
Subject: Author: Tsantilas Christos <chtsanti@users.sourceforge.net> , Alex Rousskov <rousskov... 
X-Git-Tag: SQUID_3_2_0_1~140
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=83c51da9947dad7253ee1c449ac1c0f388aaf688;p=thirdparty%2Fsquid.git

Author: Tsantilas Christos <chtsanti@users.sourceforge.net> , Alex Rousskov <rousskov@measurement-factory.com>
Support ICAP 206 Partial Content extension

The ICAP 206 Partial Content extension allows the ICAP agents to optionally
combine adapted and original HTTP message content.
For more information about ICAP Partial Content extension look at the
draft from the ICAP forum:

http://www.icap-forum.org/documents/specification/draft-icap-ext-partial-content-07.txt

The new configuration option "icap_206_enable" added to enable/disable
the 206 feature.

This is a Measurement Factory project.
---

diff --git a/src/BodyPipe.cc b/src/BodyPipe.cc
index 57732ac088..7717c2bf11 100644
--- a/src/BodyPipe.cc
+++ b/src/BodyPipe.cc
@@ -180,6 +180,15 @@ uint64_t BodyPipe::unproducedSize() const
     return bodySize() - thePutSize; // bodySize() asserts that size is known
 }
 
+void BodyPipe::expectProductionEndAfter(uint64_t size)
+{
+    const uint64_t expectedSize = thePutSize + size;
+    if (bodySizeKnown())
+        Must(bodySize() == expectedSize);
+    else
+        theBodySize = expectedSize;
+}
+
 void
 BodyPipe::clearProducer(bool atEof)
 {
diff --git a/src/BodyPipe.h b/src/BodyPipe.h
index ee5378b594..e8ccb52611 100644
--- a/src/BodyPipe.h
+++ b/src/BodyPipe.h
@@ -101,6 +101,7 @@ public:
     bool needsMoreData() const { return bodySizeKnown() && unproducedSize() > 0; }
     uint64_t unproducedSize() const; // size of still unproduced data
     bool stillProducing(const Producer *producer) const { return theProducer == producer; }
+    void expectProductionEndAfter(uint64_t extraSize); ///< sets or checks body size
 
     // called by consumers
     bool setConsumerIfNotLate(Consumer *aConsumer);
diff --git a/src/ChunkedCodingParser.cc b/src/ChunkedCodingParser.cc
index bbbf891387..66e80ce134 100644
--- a/src/ChunkedCodingParser.cc
+++ b/src/ChunkedCodingParser.cc
@@ -21,6 +21,7 @@ void ChunkedCodingParser::reset()
     theChunkSize = theLeftBodySize = 0;
     doNeedMoreData = false;
     theIn = theOut = NULL;
+    useOriginBody = -1;
 }
 
 bool ChunkedCodingParser::parse(MemBuf *rawData, MemBuf *parsedContent)
@@ -74,6 +75,10 @@ void ChunkedCodingParser::parseChunkBeg()
                 return;
             }
 
+            // to allow chunk extensions in any chunk, remove this size check
+            if (size == 0) // is this the last-chunk?
+                parseChunkExtension(p, theIn->content() + crlfBeg);
+
             theIn->consume(crlfEnd);
             theChunkSize = theLeftBodySize = size;
             debugs(94,7, "found chunk: " << theChunkSize);
@@ -228,3 +233,35 @@ bool ChunkedCodingParser::findCrlf(size_t &crlfBeg, size_t &crlfEnd)
     return false;
 }
 
+// chunk-extension= *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+void ChunkedCodingParser::parseChunkExtension(const char *startExt, const char *endExt)
+{
+    // chunk-extension starts at startExt and ends with LF at endEx
+    for (const char *p = startExt; p < endExt;) {
+
+        while (*p == ' ' || *p == '\t') ++p; // skip spaces before ';'
+        
+        if (*p++ != ';') // each ext name=value pair is preceded with ';'
+            return;
+        
+        while (*p == ' ' || *p == '\t') ++p; // skip spaces before name
+        
+        if (p >= endExt)
+            return; // malformed extension: ';' without ext name=value pair
+
+        const int extSize = endExt - p;
+        // TODO: we need debugData() stream manipulator to dump data
+        debugs(94,7, "Found chunk extension; size=" << extSize);
+
+        // TODO: support implied *LWS around '='
+        if (extSize > 18 && strncmp(p, "use-original-body=", 18) == 0) {
+            (void)StringToInt64(p+18, useOriginBody, &p, 10);
+            debugs(94, 3, HERE << "use-original-body=" << useOriginBody);
+            return; // remove to support more than just use-original-body
+        } else {
+            debugs(94, 5, HERE << "skipping unknown chunk extension");
+            // TODO: support quoted-string chunk-ext-val
+            while (p < endExt && *p != ';') ++p; // skip until the next ';'
+        }
+    }
+}
diff --git a/src/ChunkedCodingParser.h b/src/ChunkedCodingParser.h
index db6785a5a7..5b34c75998 100644
--- a/src/ChunkedCodingParser.h
+++ b/src/ChunkedCodingParser.h
@@ -80,6 +80,7 @@ private:
     void parseTrailerHeader();
     void parseMessageEnd();
 
+    void parseChunkExtension(const char *, const char * );
     bool findCrlf(size_t &crlfBeg, size_t &crlfEnd);
 
 private:
@@ -96,6 +97,9 @@ private:
     uint64_t theChunkSize;
     uint64_t theLeftBodySize;
     bool doNeedMoreData;
+
+public:
+    int64_t useOriginBody;
 };
 
 #endif /* SQUID_CHUNKEDCODINGPARSER_H */
diff --git a/src/adaptation/icap/Config.cc b/src/adaptation/icap/Config.cc
index 06abc33356..126a11f777 100644
--- a/src/adaptation/icap/Config.cc
+++ b/src/adaptation/icap/Config.cc
@@ -46,6 +46,7 @@
 Adaptation::Icap::Config Adaptation::Icap::TheConfig;
 
 Adaptation::Icap::Config::Config(): preview_enable(0), preview_size(0),
+        allow206_enable(0),
         connect_timeout_raw(0), io_timeout_raw(0), reuse_connections(0),
         client_username_header(NULL), client_username_encode(0), repeat(NULL)
 {
diff --git a/src/adaptation/icap/Config.h b/src/adaptation/icap/Config.h
index eb8be9fdde..ffe5892f27 100644
--- a/src/adaptation/icap/Config.h
+++ b/src/adaptation/icap/Config.h
@@ -57,6 +57,7 @@ public:
     int default_options_ttl;
     int preview_enable;
     int preview_size;
+    int allow206_enable;
     time_t connect_timeout_raw;
     time_t io_timeout_raw;
     int reuse_connections;
diff --git a/src/adaptation/icap/Elements.cc b/src/adaptation/icap/Elements.cc
index 35cba8c40a..ade7591ac1 100644
--- a/src/adaptation/icap/Elements.cc
+++ b/src/adaptation/icap/Elements.cc
@@ -11,6 +11,7 @@ const XactOutcome xoUnknown = "ICAP_ERR_UNKNOWN";
 const XactOutcome xoError = "ICAP_ERR_OTHER";
 const XactOutcome xoOpt = "ICAP_OPT";
 const XactOutcome xoEcho = "ICAP_ECHO";
+const XactOutcome xoPartEcho = "ICAP_PART_ECHO";
 const XactOutcome xoModified = "ICAP_MOD";
 const XactOutcome xoSatisfied = "ICAP_SAT";
 
diff --git a/src/adaptation/icap/Elements.h b/src/adaptation/icap/Elements.h
index ab3b1c8f87..7dad0e4a48 100644
--- a/src/adaptation/icap/Elements.h
+++ b/src/adaptation/icap/Elements.h
@@ -67,6 +67,7 @@ extern const XactOutcome xoUnknown; ///< initial value: outcome was not set
 extern const XactOutcome xoError; ///< all kinds of transaction errors
 extern const XactOutcome xoOpt; ///< OPTION transaction
 extern const XactOutcome xoEcho; ///< preserved virgin message (ICAP 204)
+extern const XactOutcome xoPartEcho; ///< preserved virgin msg part (ICAP 206)
 extern const XactOutcome xoModified; ///< replaced virgin msg with adapted
 extern const XactOutcome xoSatisfied; ///< request satisfaction
 
diff --git a/src/adaptation/icap/ModXact.cc b/src/adaptation/icap/ModXact.cc
index fcdbc2fa9c..1fcccd70b4 100644
--- a/src/adaptation/icap/ModXact.cc
+++ b/src/adaptation/icap/ModXact.cc
@@ -726,6 +726,10 @@ void Adaptation::Icap::ModXact::parseIcapHead()
         handle204NoContent();
         break;
 
+    case 206:
+        handle206PartialContent();
+        break;
+
     default:
         debugs(93, 5, HERE << "ICAP status " << icapReply->sline.status);
         handleUnknownScode();
@@ -797,8 +801,9 @@ void Adaptation::Icap::ModXact::handle100Continue()
     // server must not respond before the end of preview: we may send ieof
     Must(preview.enabled() && preview.done() && !preview.ieof());
 
-    // 100 "Continue" cancels our preview commitment, not 204s outside preview
-    if (!state.allowedPostview204)
+    // 100 "Continue" cancels our Preview commitment,
+    // but not commitment to handle 204 or 206 outside Preview
+    if (!state.allowedPostview204 && !state.allowedPostview206)
         stopBackup();
 
     state.parsing = State::psIcapHeader; // eventually
@@ -823,6 +828,23 @@ void Adaptation::Icap::ModXact::handle204NoContent()
     prepEchoing();
 }
 
+void Adaptation::Icap::ModXact::handle206PartialContent()
+{
+    if (state.writing == State::writingPaused) {
+        Must(preview.enabled());
+        Must(state.allowedPreview206);
+        debugs(93, 7, HERE << "206 inside preview");
+    } else {
+        Must(state.writing > State::writingPaused);
+        Must(state.allowedPostview206);
+        debugs(93, 7, HERE << "206 outside preview");
+    }
+    state.parsing = State::psHttpHeader;
+    state.sending = State::sendingAdapted;
+    state.readyForUob = true;
+    checkConsuming();
+}
+
 // Called when we receive a 204 No Content response and
 // when we are trying to bypass a service failure.
 // We actually start sending (echoig or not) in startSending.
@@ -898,6 +920,37 @@ void Adaptation::Icap::ModXact::prepEchoing()
     }
 }
 
+/// Called when we received use-original-body chunk extension in 206 response.
+/// We actually start sending (echoing or not) in startSending().
+void Adaptation::Icap::ModXact::prepPartialBodyEchoing(uint64_t pos)
+{
+    Must(virginBodySending.active());
+    Must(virgin.header->body_pipe != NULL);
+
+    setOutcome(xoPartEcho);
+
+    debugs(93, 7, HERE << "will echo virgin body suffix from " <<
+           virgin.header->body_pipe << " offset " << pos );
+
+    // check that use-original-body=N does not point beyond buffered data
+    const uint64_t virginDataEnd = virginConsumed +
+        virgin.body_pipe->buf().contentSize();
+    Must(pos <= virginDataEnd);
+    virginBodySending.progress(static_cast<size_t>(pos));
+
+    state.sending = State::sendingVirgin;
+    checkConsuming();
+
+    if (virgin.header->body_pipe->bodySizeKnown())
+        adapted.body_pipe->expectProductionEndAfter(virgin.header->body_pipe->bodySize() - pos);
+
+    debugs(93, 7, HERE << "will echo virgin body suffix to " <<
+           adapted.body_pipe);
+
+    // Start echoing data
+    echoMore();
+}
+
 void Adaptation::Icap::ModXact::handleUnknownScode()
 {
     stopParsing();
@@ -997,6 +1050,13 @@ void Adaptation::Icap::ModXact::parseBody()
     }
 
     if (parsed) {
+        if (state.readyForUob && bodyParser->useOriginBody >= 0) {
+            prepPartialBodyEchoing(
+                static_cast<uint64_t>(bodyParser->useOriginBody));
+            stopParsing();
+            return;
+        }
+
         stopParsing();
         stopSending(true); // the parser succeeds only if all parsed data fits
         return;
@@ -1235,19 +1295,11 @@ void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf)
 
     if (preview.enabled()) {
         buf.Printf("Preview: %d\r\n", (int)preview.ad());
-        if (virginBody.expected()) // there is a body to preview
-            virginBodySending.plan();
-        else
+        if (!virginBody.expected()) // there is no body to preview
             finishNullOrEmptyBodyPreview(httpBuf);
     }
 
-    if (shouldAllow204()) {
-        debugs(93,5, HERE << "will allow 204s outside of preview");
-        state.allowedPostview204 = true;
-        buf.Printf("Allow: 204\r\n");
-        if (virginBody.expected()) // there is a body to echo
-            virginBodySending.plan();
-    }
+    makeAllowHeader(buf);
 
     if (TheConfig.send_client_ip && request) {
         Ip::Address client_addr;
@@ -1277,6 +1329,44 @@ void Adaptation::Icap::ModXact::makeRequestHeaders(MemBuf &buf)
     httpBuf.clean();
 }
 
+// decides which Allow values to write and updates the request buffer
+void Adaptation::Icap::ModXact::makeAllowHeader(MemBuf &buf)
+{
+    const bool allow204in = preview.enabled(); // TODO: add shouldAllow204in()
+    const bool allow204out = state.allowedPostview204 = shouldAllow204();
+    const bool allow206in = state.allowedPreview206 = shouldAllow206in();
+    const bool allow206out = state.allowedPostview206 = shouldAllow206out();
+
+    debugs(93,9, HERE << "Allows: " << allow204in << allow204out <<
+        allow206in << allow206out);
+
+    const bool allow204 = allow204in || allow204out;
+    const bool allow206 = allow206in || allow206out;
+
+    if (!allow204 && !allow206)
+        return; // nothing to do
+
+    if (virginBody.expected()) // if there is a virgin body, plan to send it
+        virginBodySending.plan();
+
+    // writing Preview:...   means we will honor 204 inside preview
+    // writing Allow/204     means we will honor 204 outside preview
+    // writing Allow:206     means we will honor 206 inside preview
+    // writing Allow:204,206 means we will honor 206 outside preview
+    const char *allowHeader = NULL;
+    if (allow204out && allow206)
+        allowHeader = "Allow: 204, 206\r\n";
+    else if (allow204out)
+        allowHeader = "Allow: 204\r\n";
+    else if (allow206)
+        allowHeader = "Allow: 206\r\n";
+    
+    if (allowHeader) { // may be nil if only allow204in is true
+        buf.append(allowHeader, strlen(allowHeader));
+        debugs(93,5, HERE << "Will write " << allowHeader);
+    }
+}
+
 void Adaptation::Icap::ModXact::makeUsernameHeader(const HttpRequest *request, MemBuf &buf)
 {
     if (request->auth_user_request != NULL) {
@@ -1379,6 +1469,25 @@ bool Adaptation::Icap::ModXact::shouldAllow204()
     return canBackupEverything();
 }
 
+// decides whether to allow 206 responses in some mode
+bool Adaptation::Icap::ModXact::shouldAllow206any()
+{
+    return TheConfig.allow206_enable && service().allows206() &&
+        virginBody.expected(); // no need for 206 without a body
+}
+
+// decides whether to allow 206 responses in preview mode
+bool Adaptation::Icap::ModXact::shouldAllow206in()
+{
+    return shouldAllow206any() && preview.enabled();
+}
+
+// decides whether to allow 206 responses outside of preview
+bool Adaptation::Icap::ModXact::shouldAllow206out()
+{
+    return shouldAllow206any() && canBackupEverything();
+}
+
 // used by shouldAllow204 and decideOnRetries
 bool Adaptation::Icap::ModXact::canBackupEverything() const
 {
@@ -1461,6 +1570,9 @@ void Adaptation::Icap::ModXact::fillPendingStatus(MemBuf &buf) const
     if (!doneSending() && state.sending != State::sendingUndecided)
         buf.Printf("S(%d)", state.sending);
 
+    if (state.readyForUob)
+        buf.append("6", 1);
+
     if (canStartBypass)
         buf.append("Y", 1);
 
diff --git a/src/adaptation/icap/ModXact.h b/src/adaptation/icap/ModXact.h
index a837fcb242..7ee51a0f70 100644
--- a/src/adaptation/icap/ModXact.h
+++ b/src/adaptation/icap/ModXact.h
@@ -195,6 +195,7 @@ private:
     bool virginBodyEndReached(const VirginBodyAct &act) const;
 
     void makeRequestHeaders(MemBuf &buf);
+    void makeAllowHeader(MemBuf &buf);
     void makeUsernameHeader(const HttpRequest *request, MemBuf &buf);
     void addLastRequestChunk(MemBuf &buf);
     void openChunk(MemBuf &buf, size_t chunkSize, bool ieof);
@@ -205,6 +206,9 @@ private:
     void decideOnPreview();
     void decideOnRetries();
     bool shouldAllow204();
+    bool shouldAllow206any();
+    bool shouldAllow206in();
+    bool shouldAllow206out();
     bool canBackupEverything() const;
 
     void prepBackup(size_t expectedSize);
@@ -225,6 +229,7 @@ private:
     bool validate200Ok();
     void handle200Ok();
     void handle204NoContent();
+    void handle206PartialContent();
     void handleUnknownScode();
 
     void bypassFailure();
@@ -233,6 +238,7 @@ private:
     void disableBypass(const char *reason, bool includeGroupBypass);
 
     void prepEchoing();
+    void prepPartialBodyEchoing(uint64_t pos);
     void echoMore();
 
     virtual bool doneAll() const;
@@ -281,6 +287,9 @@ private:
 
         bool serviceWaiting; // waiting for ICAP service options
         bool allowedPostview204; // mmust handle 204 No Content outside preview
+        bool allowedPostview206; // must handle 206 Partial Content outside preview
+        bool allowedPreview206; // must handle 206 Partial Content inside preview
+        bool readyForUob; ///< got a 206 response and expect a use-origin-body
 
         // will not write anything [else] to the ICAP server connection
         bool doneWriting() const { return writing == writingReallyDone; }
@@ -288,7 +297,8 @@ private:
         // will not use virgin.body_pipe
         bool doneConsumingVirgin() const {
             return writing >= writingAlmostDone
-                   && (sending == sendingAdapted || sending == sendingDone);
+                && ((sending == sendingAdapted && !readyForUob) ||
+                    sending == sendingDone);
         }
 
         // parsed entire ICAP response from the ICAP server
diff --git a/src/adaptation/icap/OptXact.cc b/src/adaptation/icap/OptXact.cc
index 17990d9602..dfd9a768b9 100644
--- a/src/adaptation/icap/OptXact.cc
+++ b/src/adaptation/icap/OptXact.cc
@@ -8,6 +8,7 @@
 
 #include "adaptation/icap/OptXact.h"
 #include "adaptation/icap/Options.h"
+#include "adaptation/icap/Config.h"
 #include "base/TextException.h"
 #include "SquidTime.h"
 #include "HttpRequest.h"
@@ -49,6 +50,8 @@ void Adaptation::Icap::OptXact::makeRequest(MemBuf &buf)
     buf.Printf("OPTIONS " SQUIDSTRINGPH " ICAP/1.0\r\n", SQUIDSTRINGPRINT(uri));
     const String host = s.cfg().host;
     buf.Printf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(host), s.cfg().port);
+    if (TheConfig.allow206_enable)
+        buf.Printf("Allow: 206\r\n");
     buf.append(ICAP::crlf, 2);
 
     // XXX: HttpRequest cannot fully parse ICAP Request-Line
diff --git a/src/adaptation/icap/Options.cc b/src/adaptation/icap/Options.cc
index da84d28c62..4cf37cb337 100644
--- a/src/adaptation/icap/Options.cc
+++ b/src/adaptation/icap/Options.cc
@@ -8,6 +8,7 @@
 
 Adaptation::Icap::Options::Options(): error("unconfigured"),
         max_connections(-1), allow204(false),
+        allow206(false),
         preview(-1), theTTL(-1)
 {
     theTransfers.preview.name = "Transfer-Preview";
@@ -104,6 +105,9 @@ void Adaptation::Icap::Options::configure(const HttpReply *reply)
     if (h->hasListMember(HDR_ALLOW, "204", ','))
         allow204 = true;
 
+    if (h->hasListMember(HDR_ALLOW, "206", ','))
+        allow206 = true;
+
     cfgIntHeader(h, "Preview", preview);
 
     cfgTransferList(h, theTransfers.preview);
diff --git a/src/adaptation/icap/Options.h b/src/adaptation/icap/Options.h
index f17e2bf8a6..aeb8621a5d 100644
--- a/src/adaptation/icap/Options.h
+++ b/src/adaptation/icap/Options.h
@@ -81,6 +81,7 @@ public:
     String serviceId;
     int max_connections;
     bool allow204;
+    bool allow206;
     int preview;
 
 protected:
diff --git a/src/adaptation/icap/ServiceRep.cc b/src/adaptation/icap/ServiceRep.cc
index ff33f96e65..cba32c45df 100644
--- a/src/adaptation/icap/ServiceRep.cc
+++ b/src/adaptation/icap/ServiceRep.cc
@@ -125,6 +125,14 @@ bool Adaptation::Icap::ServiceRep::allows204() const
     return true; // in the future, we may have ACLs to prevent 204s
 }
 
+bool Adaptation::Icap::ServiceRep::allows206() const
+{
+    Must(hasOptions());
+    if (theOptions->allow206)
+        return true; // in the future, we may have ACLs to prevent 206s
+    return false;
+}
+
 
 static
 void ServiceRep_noteTimeToUpdate(void *data)
diff --git a/src/adaptation/icap/ServiceRep.h b/src/adaptation/icap/ServiceRep.h
index 4bbd38a657..45b32b7fe5 100644
--- a/src/adaptation/icap/ServiceRep.h
+++ b/src/adaptation/icap/ServiceRep.h
@@ -103,6 +103,7 @@ public:
     bool wantsUrl(const String &urlPath) const;
     bool wantsPreview(const String &urlPath, size_t &wantedSize) const;
     bool allows204() const;
+    bool allows206() const;
 
     void noteFailure(); // called by transactions to report service failure
 
diff --git a/src/cf.data.pre b/src/cf.data.pre
index b13b1c01bc..ee4246d5d8 100644
--- a/src/cf.data.pre
+++ b/src/cf.data.pre
@@ -5964,6 +5964,28 @@ DOC_START
 	basis by OPTIONS requests.
 DOC_END
 
+NAME: icap_206_enable
+TYPE: onoff
+IFDEF: ICAP_CLIENT
+COMMENT: on|off
+LOC: Adaptation::Icap::TheConfig.allow206_enable
+DEFAULT: on
+DOC_START
+	206 (Partial Content) responses is an ICAP extension that allows the
+	ICAP agents to optionally combine adapted and original HTTP message
+	content. The decision to combine is postponed until the end of the
+	ICAP response. Squid supports Partial Content extension by default.
+
+	Activation of the Partial Content extension is negotiated with each
+	ICAP service during OPTIONS exchange. Most ICAP servers should handle
+	negotation correctly even if they do not support the extension, but
+	some might fail. To disable Partial Content support for all ICAP
+	services and to avoid any negotiation, set this option to "off".
+
+	Example:
+	    icap_206_enable off
+DOC_END
+
 NAME: icap_default_options_ttl
 TYPE: int
 IFDEF: ICAP_CLIENT