Author: Alex Rousskov <rousskov@measurement-factory.com>

HTTP/1.1: Send chunked responses if body size is unknown.

Apply HTTP chunked transfer encoding to the response body sent to client
if all of the following conditions are met:

* client claims HTTP version 1.1 or later support
* response does not have a Content-Length header already
* response does not use multipart/byteranges encoding
* connection is persistent

If we decide to send chunked reply, chunked_reply flag is set. Chunked
encoding is done in ClientSocketContext::packChunk(). The last-chunk
is sent only when clientReplyContext complete flag is set.

This change helps keep client-side connections persistent.

diff --git a/include/squid_types.h b/include/squid_types.h
index 0b4a60529e8ffb5a3eff8df756e2572d434574b2..d4634489485f3fd7fc75427d934f1d231a90371a 100644 (file)
--- a/include/squid_types.h
+++ b/include/squid_types.h
@@ -106,4 +106,14 @@
 #endif
 #endif
 
+#ifndef PRIX64
+#ifdef _SQUID_MSWIN_           /* Windows native port using MSVCRT */
+#define PRIX64 "I64X"
+#elif SIZEOF_INT64_T > SIZEOF_LONG
+#define PRIX64 "llX"
+#else
+#define PRIX64 "lX"
+#endif
+#endif
+
 #endif /* SQUID_TYPES_H */

diff --git a/src/client_side.cc b/src/client_side.cc
index 18dd88fd86a9136cdb7394575943d57d185df39e..64caf9eaa1eb72db23cf90994d7e49b45113692e 100644 (file)
--- a/src/client_side.cc
+++ b/src/client_side.cc
@@ -855,7 +855,7 @@ ClientSocketContext::sendBody(HttpReply * rep, StoreIOBuffer bodyData)
 {
     assert(rep == NULL);
 
-    if (!multipartRangeRequest()) {
+    if (!multipartRangeRequest() && !http->request->flags.chunked_reply) {
         size_t length = lengthToSend(bodyData.range());
         noteSentBodyBytes (length);
         AsyncCall::Pointer call = commCbCall(33, 5, "clientWriteBodyComplete",
@@ -866,7 +866,10 @@ ClientSocketContext::sendBody(HttpReply * rep, StoreIOBuffer bodyData)
 
     MemBuf mb;
     mb.init();
-    packRange(bodyData, &mb);
+    if (multipartRangeRequest())
+        packRange(bodyData, &mb);
+    else
+        packChunk(bodyData, mb);
 
     if (mb.contentSize()) {
         /* write */
@@ -877,6 +880,22 @@ ClientSocketContext::sendBody(HttpReply * rep, StoreIOBuffer bodyData)
         writeComplete(fd(), NULL, 0, COMM_OK);
 }
 
+/**
+ * Packs bodyData into mb using chunked encoding. Packs the last-chunk
+ * if bodyData is empty.
+ */
+void
+ClientSocketContext::packChunk(const StoreIOBuffer &bodyData, MemBuf &mb)
+{
+    const uint64_t length =
+        static_cast<uint64_t>(lengthToSend(bodyData.range()));
+    noteSentBodyBytes(length);
+
+    mb.Printf("%"PRIX64"\r\n", length);
+    mb.append(bodyData.data, length);
+    mb.Printf("\r\n");
+}
+
 /** put terminating boundary for multiparts */
 static void
 clientPackTermBound(String boundary, MemBuf * mb)
@@ -1243,13 +1262,15 @@ ClientSocketContext::sendStartOfMessage(HttpReply * rep, StoreIOBuffer bodyData)
 #endif
 
     if (bodyData.data && bodyData.length) {
-        if (!multipartRangeRequest()) {
+        if (multipartRangeRequest())
+            packRange(bodyData, mb);
+        else if (http->request->flags.chunked_reply) {
+            packChunk(bodyData, *mb);
+        } else {
             size_t length = lengthToSend(bodyData.range());
             noteSentBodyBytes (length);
 
             mb->append(bodyData.data, length);
-        } else {
-            packRange(bodyData, mb);
         }
     }
 
@@ -1298,7 +1319,11 @@ clientSocketRecipient(clientStreamNode * node, ClientHttpRequest * http,
         return;
     }
 
-    if (responseFinishedOrFailed(rep, receivedData)) {
+    // After sending Transfer-Encoding: chunked (at least), always send
+    // the last-chunk if there was no error, ignoring responseFinishedOrFailed.
+    const bool mustSendLastChunk = http->request->flags.chunked_reply &&
+        !http->request->flags.stream_error && !context->startOfOutput();
+    if (responseFinishedOrFailed(rep, receivedData) && !mustSendLastChunk) {
         context->writeComplete(fd, NULL, 0, COMM_OK);
         PROF_stop(clientSocketRecipient);
         return;

diff --git a/src/client_side.h b/src/client_side.h
index 0ec5ce5d5f158166d039f3f5476644154b41a5c0..acb13c353bddd32c9aa587a446ac2d516b18473f 100644 (file)
--- a/src/client_side.h
+++ b/src/client_side.h
@@ -117,6 +117,7 @@ public:
 private:
     CBDATA_CLASS(ClientSocketContext);
     void prepareReply(HttpReply * rep);
+    void packChunk(const StoreIOBuffer &bodyData, MemBuf &mb);
     void packRange(StoreIOBuffer const &, MemBuf * mb);
     void deRegisterWithConn();
     void doClose();

diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc
index f3b01c7f036c7e268d22dd51286e6b52c27926e0..9569810c319880478cf25b03b725a63324a7a77a 100644 (file)
--- a/src/client_side_reply.cc
+++ b/src/client_side_reply.cc
@@ -978,6 +978,11 @@ clientReplyContext::checkTransferDone()
     if (http->flags.done_copying)
         return 1;
 
+    if (http->request->flags.chunked_reply && !flags.complete) {
+        // last-chunk was not sent
+        return 0;
+    }
+
     /*
      * Handle STORE_OK objects.
      * objectLen(entry) will be set proprely.
@@ -1124,7 +1129,9 @@ clientReplyContext::replyStatus()
         debugs(88, 5, "clientReplyStatus: transfer is DONE");
         /* Ok we're finished, but how? */
 
-        if (http->storeEntry()->getReply()->bodySize(http->request->method) < 0) {
+        const int64_t expectedBodySize =
+            http->storeEntry()->getReply()->bodySize(http->request->method);
+        if (!http->request->flags.proxy_keepalive && expectedBodySize < 0) {
             debugs(88, 5, "clientReplyStatus: closing, content_length < 0");
             return STREAM_FAILED;
         }
@@ -1134,7 +1141,7 @@ clientReplyContext::replyStatus()
             return STREAM_FAILED;
         }
 
-        if (!http->gotEnough()) {
+        if (expectedBodySize >= 0 && !http->gotEnough()) {
             debugs(88, 5, "clientReplyStatus: client didn't get all it expected");
             return STREAM_UNPLANNED_COMPLETE;
         }
@@ -1370,6 +1377,9 @@ clientReplyContext::buildReplyHeader()
 
 #endif
 
+    const bool maySendChunkedReply = !request->multipartRangeRequest() &&
+        (request->http_ver >= HttpVersion(1, 1));
+
     /* Check whether we should send keep-alive */
     if (!Config.onoff.error_pconns && reply->sline.status >= 400 && !request->flags.must_keepalive) {
         debugs(33, 3, "clientBuildReplyHeader: Error, don't keep-alive");
@@ -1383,7 +1393,7 @@ clientReplyContext::buildReplyHeader()
     } else if (request->flags.connection_auth && !reply->keep_alive) {
         debugs(33, 2, "clientBuildReplyHeader: Connection oriented auth but server side non-persistent");
         request->flags.proxy_keepalive = 0;
-    } else if (reply->bodySize(request->method) < 0) {
+    } else if (reply->bodySize(request->method) < 0 && !maySendChunkedReply) {
         debugs(88, 3, "clientBuildReplyHeader: can't keep-alive, unknown body size" );
         request->flags.proxy_keepalive = 0;
     } else if (fdUsageHigh()&& !request->flags.must_keepalive) {
@@ -1391,6 +1401,14 @@ clientReplyContext::buildReplyHeader()
         request->flags.proxy_keepalive = 0;
     }
 
+    // Decide if we send chunked reply
+    if (maySendChunkedReply &&
+        request->flags.proxy_keepalive &&
+        reply->bodySize(request->method) < 0) {
+        debugs(88, 3, "clientBuildReplyHeader: chunked reply");
+        request->flags.chunked_reply = 1;
+        hdr->putStr(HDR_TRANSFER_ENCODING, "chunked");
+    }
 
     /* Append VIA */
     if (Config.onoff.via) {
@@ -1725,6 +1743,7 @@ clientReplyContext::sendStreamError(StoreIOBuffer const &result)
     debugs(88, 5, "clientReplyContext::sendStreamError: A stream error has occured, marking as complete and sending no data.");
     StoreIOBuffer localTempBuffer;
     flags.complete = 1;
+    http->request->flags.stream_error = 1;
     localTempBuffer.flags.error = result.flags.error;
     clientStreamCallback((clientStreamNode*)http->client_stream.head->data, http, NULL,
                          localTempBuffer);

diff --git a/src/structs.h b/src/structs.h
index 493f822e050a4074a8af9c7bbde6da56513f7359..928037e8634aea3c20d5be52a88e48a840ac672c 100644 (file)
--- a/src/structs.h
+++ b/src/structs.h
@@ -1010,7 +1010,7 @@ struct _iostats {
 
 
 struct request_flags {
-    request_flags(): range(0),nocache(0),ims(0),auth(0),cachable(0),hierarchical(0),loopdetect(0),proxy_keepalive(0),proxying(0),refresh(0),redirected(0),need_validation(0),accelerated(0),ignore_cc(0),intercepted(0),spoof_client_ip(0),internal(0),internalclient(0),must_keepalive(0),destinationIPLookedUp_(0) {
+    request_flags(): range(0),nocache(0),ims(0),auth(0),cachable(0),hierarchical(0),loopdetect(0),proxy_keepalive(0),proxying(0),refresh(0),redirected(0),need_validation(0),accelerated(0),ignore_cc(0),intercepted(0),spoof_client_ip(0),internal(0),internalclient(0),must_keepalive(0),chunked_reply(0),stream_error(0),destinationIPLookedUp_(0) {
 #if HTTP_VIOLATIONS
         nocache_hack = 0;
 #endif
@@ -1048,6 +1048,8 @@ unsigned int proxying:
     unsigned int pinned:1;      /* Request sent on a pinned connection */
     unsigned int auth_sent:1;   /* Authentication forwarded */
     unsigned int no_direct:1;  /* Deny direct forwarding unless overriden by always_direct. Used in accelerator mode */
+    unsigned int chunked_reply:1; /**< Reply with chunked transfer encoding */
+    unsigned int stream_error:1; /**< Whether stream error has occured */
 
     // When adding new flags, please update cloneAdaptationImmune() as needed.