]> git.ipfire.org Git - thirdparty/squid.git/commitdiff
Convert Http::Stream::reqbuf to a MemBlob, making it configurable at runtime.
authorNathan Hoad <nathan@getoffmalawn.com>
Fri, 24 Jun 2016 05:29:44 +0000 (17:29 +1200)
committerAmos Jeffries <squid3@treenet.co.nz>
Fri, 24 Jun 2016 05:29:44 +0000 (17:29 +1200)
This also makes many other auxilary changes:

 * Increases the size of Http::Stream::requestBuffer to match that of
   read_ahead_gap. Previously this was a 4kb fixed size buffer. As a result,
   the overhead for a single client connection has been increased by 12 KB in
   the default configuration, but Squid will no longer artifically slow down
   client responses in this situation by fragmenting the read(2)/write(2)
   calls.

 * Improves the performance of large uncacheable replies. This was achieved by
   increasing the buffer size to 16 KB as mentioned above, but it is worth
   mentioning separately. Specifically, for a server, client and proxy all
   running on my local machine, this patch increases throughput on a 5 GB file
   from ~110 MB/s to ~340 MB/s.

 * Documents the influence that read_ahead_gap had on the size of read(2) calls
   for HTTP, and now the size of write(2) calls.

 * Prevent read_ahead_gap from being set to 0. Previously this would result in
   hung requests.

This work is submitted on behalf of Bloomberg L.P.

src/cache_cf.cc
src/cf.data.pre
src/client_side.cc
src/http/Stream.cc
src/http/Stream.h
src/servers/FtpServer.cc

index 2e23180161f5b8aa6e374b56ec7e17e262eb59dd..67573c1cb06e1c6a64d26091eeb3c0c6ac7705f0 100644 (file)
@@ -946,6 +946,10 @@ configDoConfigure(void)
         }
     }
 #endif
+
+    if (Config.readAheadGap <= 0) {
+        fatalf("read_ahead_gap must be greater than 0 bytes");
+    }
 }
 
 /** Parse a line containing an obsolete directive.
index 26fe3786b0733d13518b15e6533d86d007b6516d..a00b85447bfe173b8fb194a924a9a2eb67b447b5 100644 (file)
@@ -5650,6 +5650,17 @@ DEFAULT: 16 KB
 DOC_START
        The amount of data the cache will buffer ahead of what has been
        sent to the client when retrieving an object from another server.
+       
+       This also influences the maximum network read(2)/write(2) sizes in some
+       circumstances. Reducing the size of this buffer will decrease
+       per-connection memory usage at the cost of more read(2)/write(2) calls.
+       Conversely, increasing the size of this buffer will decrease the number of
+       read(2)/write(2) calls at the cost of memory usage, potentially improving
+       performance.
+
+       Squid does not slow does the response delivery to the client in order to
+       fill the buffer.
+
 DOC_END
 
 NAME: negative_ttl
index 74ca44c8559dff6ef53265397ea99de50721b47c..511fa241413829763be363c8f9963a245a731619 100644 (file)
@@ -1012,12 +1012,9 @@ ConnStateData::abortRequestParsing(const char *const uri)
     http->uri = xstrdup(uri);
     setLogUri (http, uri);
     auto *context = new Http::Stream(clientConnection, http);
-    StoreIOBuffer tempBuffer;
-    tempBuffer.data = context->reqbuf;
-    tempBuffer.length = HTTP_REQBUF_SZ;
     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
                      clientReplyStatus, new clientReplyContext(http), clientSocketRecipient,
-                     clientSocketDetach, context, tempBuffer);
+                     clientSocketDetach, context, context->getClientStreamBuffer());
     return context;
 }
 
@@ -1359,15 +1356,11 @@ parseHttpRequest(ConnStateData *csd, const Http1::RequestParserPointer &hp)
     http->req_sz = hp->messageHeaderSize();
     Http::Stream *result = new Http::Stream(csd->clientConnection, http);
 
-    StoreIOBuffer tempBuffer;
-    tempBuffer.data = result->reqbuf;
-    tempBuffer.length = HTTP_REQBUF_SZ;
-
     ClientStreamData newServer = new clientReplyContext(http);
     ClientStreamData newClient = result;
     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
                      clientReplyStatus, newServer, clientSocketRecipient,
-                     clientSocketDetach, newClient, tempBuffer);
+                     clientSocketDetach, newClient, result->getClientStreamBuffer());
 
     /* set url */
     debugs(33,5, "Prepare absolute URL from " <<
index ed6088f4e7673ae98b476d8527f21928557d6e8d..f2b9211d420d1e227256e87092cff7f96c30d552 100644 (file)
@@ -11,6 +11,7 @@
 #include "http/Stream.h"
 #include "HttpHdrContRange.h"
 #include "HttpHeaderTools.h"
+#include "SquidConfig.h"
 #include "Store.h"
 #include "TimeOrTag.h"
 
@@ -20,10 +21,10 @@ Http::Stream::Stream(const Comm::ConnectionPointer &aConn, ClientHttpRequest *aR
     reply(nullptr),
     writtenToSocket(0),
     mayUseConnection_(false),
-    connRegistered_(false)
+    connRegistered_(false),
+    requestBuffer(nullptr)
 {
     assert(http != nullptr);
-    memset(reqbuf, '\0', sizeof (reqbuf));
     flags.deferred = 0;
     flags.parsed_ok = 0;
     deferredparams.node = nullptr;
@@ -109,12 +110,10 @@ Http::Stream::pullData()
     debugs(33, 5, reply << " written " << http->out.size << " into " << clientConnection);
 
     /* More data will be coming from the stream. */
-    StoreIOBuffer readBuffer;
+    StoreIOBuffer readBuffer = getClientStreamBuffer();
     /* XXX: Next requested byte in the range sequence */
     /* XXX: length = getmaximumrangelenfgth */
     readBuffer.offset = getNextRangeOffset();
-    readBuffer.length = HTTP_REQBUF_SZ;
-    readBuffer.data = reqbuf;
     /* we may note we have reached the end of the wanted ranges */
     clientStreamRead(getTail(), http, readBuffer);
 }
@@ -568,6 +567,18 @@ Http::Stream::deferRecipientForLater(clientStreamNode *node, HttpReply *rep, Sto
     deferredparams.queuedBuffer = receivedData;
 }
 
+StoreIOBuffer
+Http::Stream::getClientStreamBuffer()
+{
+    if (!requestBuffer) {
+        requestBuffer = new MemBlob(Config.readAheadGap);
+    }
+    StoreIOBuffer tempBuffer;
+    tempBuffer.data = requestBuffer->mem;
+    tempBuffer.length = requestBuffer->spaceSize();
+    return tempBuffer;
+}
+
 void
 Http::Stream::prepareReply(HttpReply *rep)
 {
index 946ff6337b531cd8d4ad2633426457c132fe08cf..21bfdc2cd3d494462dfb98bf91e9188b51f8a81a 100644 (file)
@@ -120,12 +120,13 @@ public:
 
     void deferRecipientForLater(clientStreamNode *, HttpReply *, StoreIOBuffer receivedData);
 
+    StoreIOBuffer getClientStreamBuffer();
+
 public: // HTTP/1.x state data
 
     Comm::ConnectionPointer clientConnection; ///< details about the client connection socket
     ClientHttpRequest *http;    /* we pretend to own that Job */
     HttpReply *reply;
-    char reqbuf[HTTP_REQBUF_SZ];
     struct {
         unsigned deferred:1; ///< This is a pipelined request waiting for the current object to complete
         unsigned parsed_ok:1; ///< Was this parsed correctly?
@@ -158,6 +159,8 @@ private:
 
     bool mayUseConnection_; /* This request may use the connection. Don't read anymore requests for now */
     bool connRegistered_;
+
+    MemBlob::Pointer requestBuffer;
 };
 
 } // namespace Http
index 2e6e32cba1d4a7511285eab84656de58c291435e..c2bc795a59817c603eedf837ef22d8e3bfdf44dc 100644 (file)
@@ -759,15 +759,11 @@ Ftp::Server::parseOneRequest()
     Http::Stream *const result =
         new Http::Stream(clientConnection, http);
 
-    StoreIOBuffer tempBuffer;
-    tempBuffer.data = result->reqbuf;
-    tempBuffer.length = HTTP_REQBUF_SZ;
-
     ClientStreamData newServer = new clientReplyContext(http);
     ClientStreamData newClient = result;
     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
                      clientReplyStatus, newServer, clientSocketRecipient,
-                     clientSocketDetach, newClient, tempBuffer);
+                     clientSocketDetach, newClient, result->getClientStreamBuffer());
 
     result->flags.parsed_ok = 1;
     return result;