Author: Christos Tsantilas <chtsanti@users.sourceforge.net>

Bug #1148: Add support for chunked transfer decoding of HTTP responses

This small patch uses ICAP's client chunks decoder to decode http chunked
responses.

diff --git a/src/http.cc b/src/http.cc
index 33c4a8d416fef3858ba667fc3b87fa08742926bb..1477d5fccd133c4acfb9732f8135a5e69949111a 100644 (file)
--- a/src/http.cc
+++ b/src/http.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.cc,v 1.542 2007/12/14 23:11:47 amosjeffries Exp $
+ * $Id: http.cc,v 1.543 2007/12/26 23:39:55 hno Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
@@ -56,6 +56,16 @@
 #include "DelayPools.h"
 #endif
 #include "SquidTime.h"
+#include "TextException.h"
+
+#define SQUID_ENTER_THROWING_CODE() try {
+#define SQUID_EXIT_THROWING_CODE(status) \
+       status = true; \
+    } \
+    catch (const TextException &e) { \
+       debugs (11, 1, "Exception error:" << e.message); \
+       status = false; \
+    }  
 
 CBDATA_CLASS_INIT(HttpStateData);
 
@@ -68,7 +78,7 @@ static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeader
         HttpHeader * hdr_out, int we_do_ranges, http_state_flags);
 
 HttpStateData::HttpStateData(FwdState *theFwdState) : ServerStateData(theFwdState),
-        header_bytes_read(0), reply_bytes_read(0)
+                      header_bytes_read(0), reply_bytes_read(0), httpChunkDecoder(NULL)
 {
     debugs(11,5,HERE << "HttpStateData " << this << " created");
     ignoreCacheControl = false;
@@ -119,7 +129,6 @@ HttpStateData::HttpStateData(FwdState *theFwdState) : ServerStateData(theFwdStat
         entry->setNoDelay(_peer->options.no_delay);
 
 #endif
-
     }
 
     /*
@@ -139,6 +148,9 @@ HttpStateData::~HttpStateData()
 
     delete readBuf;
 
+    if(httpChunkDecoder)
+       delete httpChunkDecoder;
+
     HTTPMSGUNLOCK(orig_request);
 
     debugs(11,5, HERE << "HttpStateData " << this << " destroyed; FD " << fd);
@@ -735,6 +747,12 @@ HttpStateData::processReplyHeader()
         readBuf->consume(header_bytes_read);
     }
 
+    flags.chunked = 0;
+    if (newrep->header.hasListMember(HDR_TRANSFER_ENCODING, "chunked", ',')) {
+        flags.chunked = 1;
+        httpChunkDecoder = new ChunkedCodingParser;
+    }
+
     HttpReply *vrep = setVirginReply(newrep);
     flags.headers_parsed = 1;
 
@@ -896,6 +914,13 @@ HttpStateData::persistentConnStatus() const
     if (!flags.headers_parsed)
         return INCOMPLETE_MSG;
 
+    /* In chunked responce we do not know the content length but we are absolutelly 
+     * sure about the end of response, so we are calling the statusIfComplete to
+     * decide if we can be persistant 
+     */
+    if (eof && flags.chunked)
+       return statusIfComplete();
+
     if (eof) // already reached EOF
         return COMPLETE_NONPERSISTENT_MSG;
 
@@ -1098,10 +1123,31 @@ HttpStateData::writeReplyBody()
 {
     const char *data = readBuf->content();
     int len = readBuf->contentSize();
-
     addVirginReplyBody(data, len);
     readBuf->consume(len);
+}
 
+bool
+HttpStateData::decodeAndWriteReplyBody()
+{
+    const char *data = NULL;
+    int len;
+    bool status = false;
+    assert(flags.chunked);
+    assert(httpChunkDecoder);
+    SQUID_ENTER_THROWING_CODE();
+    MemBuf decodedData;
+    decodedData.init();
+    const bool done = httpChunkDecoder->parse(readBuf,&decodedData);
+    len = decodedData.contentSize();
+    data=decodedData.content();
+    addVirginReplyBody(data, len);
+    if (done) {
+       eof = 1;
+       flags.do_next_read = 0;
+    }
+    SQUID_EXIT_THROWING_CODE(status);
+    return status;
 }
 
 /*
@@ -1134,7 +1180,15 @@ HttpStateData::processReplyBody()
      * That means header content has been removed from readBuf and
      * it contains only body data.
      */
-    writeReplyBody();
+    if(flags.chunked){
+       if(!decodeAndWriteReplyBody()){
+           flags.do_next_read = 0;
+           serverComplete();
+           return;
+       }
+    }
+    else
+       writeReplyBody();
 
     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
         /*

diff --git a/src/http.h b/src/http.h
index 3544d530cc6ebd4b42c20757d5ca9255fc202b21..ffec6722a3226aafbb0188e8f16f28aea0410b3d 100644 (file)
--- a/src/http.h
+++ b/src/http.h
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.h,v 1.32 2007/08/09 23:30:53 rousskov Exp $
+ * $Id: http.h,v 1.33 2007/12/26 23:39:55 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -38,6 +38,7 @@
 #include "comm.h"
 #include "forward.h"
 #include "Server.h"
+#include "ChunkedCodingParser.h"
 
 class HttpStateData : public ServerStateData
 {
@@ -103,6 +104,7 @@ private:
     virtual void handleRequestBodyProducerAborted();
 
     void writeReplyBody();
+    bool decodeAndWriteReplyBody();
     void doneSendingRequestBody();
     void requestBodyHandler(MemBuf &);
     virtual void sentRequestBody(int fd, size_t size, comm_err_t errflag);
@@ -113,6 +115,7 @@ private:
                                  http_state_flags flags);
     static bool decideIfWeDoRanges (HttpRequest * orig_request);
 
+    ChunkedCodingParser *httpChunkDecoder;
 private:
     CBDATA_CLASS2(HttpStateData);
 };

diff --git a/src/structs.h b/src/structs.h
index 625fbd69de9e34e922e6c74992688093731af2a7..272c3f0c328edfc8f3c15462e7536de03668b2f2 100644 (file)
--- a/src/structs.h
+++ b/src/structs.h
@@ -1,6 +1,6 @@
 
 /*
- * $Id: structs.h,v 1.569 2007/12/14 23:11:48 amosjeffries Exp $
+ * $Id: structs.h,v 1.570 2007/12/26 23:39:55 hno Exp $
  *
  *
  * SQUID Web Proxy Cache          http://www.squid-cache.org/
@@ -929,6 +929,8 @@ unsigned int do_next_read:
 
 unsigned int consume_body_data:
     1;
+
+unsigned int chunked:1;
 };
 
 struct _ipcache_addrs