Better helper-to-Squid buffer size management.

The minimum buffer size is reduced from 8KB to 4KB after a squid-dev
discussion to prevent wasting of "several hundred KB of unused permanent
memory on some installations".

We now increase the buffer if we cannot parse the helper response message.

The maximum buffer size is now 32KB. This should be enough for all known
helper responses.

We now warn if the read buffer reaches its capacity and kill the offending
helper explicitly. An increase in maximum buffer capacity to 32KB should make
such events rare.

Motivation: ssl_crtd helper may produce responses exceeding 9907 bytes in size
(and possibly much larger if multiple chained certificates need to be returned
to Squid). The old helper.cc code would fill the read buffer completely,
schedule a read for zero bytes, receive zero bytes, declare an EOF condition,
and close the stream (which kills ssl_crtd).  Due to insufficient information
logged, the observable symptoms were pretty much the same as if ssl_crtd
closed the stream first, indicating a ssl_crtd bug.

diff --git a/src/helper.cc b/src/helper.cc
index b9ef32b772244bc6681326c8c38c4d2ea34ded4c..9165908fcae420bd93ea3da788f7f7e845ca46ae 100644 (file)
--- a/src/helper.cc
+++ b/src/helper.cc
@@ -43,10 +43,18 @@
 
 #define HELPER_MAX_ARGS 64
 
-/* size of helper read buffer (maximum?). no reason given for this size */
-/* though it has been seen to be too short for some requests */
-/* it is dynamic, so increasng should not have side effects */
-#define BUF_8KB        8192
+
+/** Initial Squid input buffer size. Helper responses may exceed this, and
+ * Squid will grow the input buffer as needed, up to ReadBufMaxSize.
+ */
+const size_t ReadBufMinSize(4*1024);
+
+/** Maximum safe size of a helper-to-Squid response message plus one.
+ * Squid will warn and close the stream if a helper sends a too-big response.
+ * ssl_crtd helper is known to produce responses of at least 10KB in size.
+ * Some undocumented helpers are known to produce responses exceeding 8KB.
+ */
+const size_t ReadBufMaxSize(32*1024);
 
 static IOCB helperHandleRead;
 static IOCB helperStatefulHandleRead;
@@ -150,7 +158,7 @@ helperOpenServers(helper * hlp)
         srv->addr = hlp->addr;
         srv->rfd = rfd;
         srv->wfd = wfd;
-        srv->rbuf = (char *)memAllocBuf(BUF_8KB, &srv->rbuf_sz);
+        srv->rbuf = (char *)memAllocBuf(ReadBufMinSize, &srv->rbuf_sz);
         srv->wqueue = new MemBuf;
         srv->roffset = 0;
         srv->requests = (helper_request **)xcalloc(hlp->concurrency ? hlp->concurrency : 1, sizeof(*srv->requests));
@@ -262,7 +270,7 @@ helperStatefulOpenServers(statefulhelper * hlp)
         srv->addr = hlp->addr;
         srv->rfd = rfd;
         srv->wfd = wfd;
-        srv->rbuf = (char *)memAllocBuf(BUF_8KB, &srv->rbuf_sz);
+        srv->rbuf = (char *)memAllocBuf(ReadBufMinSize, &srv->rbuf_sz);
         srv->roffset = 0;
         srv->parent = cbdataReference(hlp);
 
@@ -948,8 +956,30 @@ helperHandleRead(int fd, char *buf, size_t len, comm_err_t flag, int xerrno, voi
         helperReturnBuffer(i, srv, hlp, msg, t);
     }
 
-    if (srv->rfd != -1)
-        comm_read(fd, srv->rbuf + srv->roffset, srv->rbuf_sz - srv->roffset - 1, helperHandleRead, srv);
+    if (srv->rfd != -1) {
+        int spaceSize = srv->rbuf_sz - srv->roffset - 1;
+        assert(spaceSize >= 0);
+
+        // grow the input buffer if needed and possible
+        if (!spaceSize && srv->rbuf_sz + 4096 <= ReadBufMaxSize) {
+            srv->rbuf = (char *)memReallocBuf(srv->rbuf, srv->rbuf_sz + 4096, &srv->rbuf_sz);
+            debugs(84, 3, HERE << "Grew read buffer to " << srv->rbuf_sz);
+            spaceSize = srv->rbuf_sz - srv->roffset - 1;
+            assert(spaceSize >= 0);
+        }
+
+        // quit reading if there is no space left
+        if (!spaceSize) {
+            debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " <<
+                   "helper that overflowed " << srv->rbuf_sz << "-byte " <<
+                   "Squid input buffer: " << hlp->id_name << " #" <<
+                   (srv->index + 1));
+            srv->closePipesSafely();
+            return;
+        }
+
+        comm_read(fd, srv->rbuf + srv->roffset, spaceSize, helperHandleRead, srv);
+    }
 }
 
 static void
@@ -1025,9 +1055,30 @@ helperStatefulHandleRead(int fd, char *buf, size_t len, comm_err_t flag, int xer
             helperStatefulReleaseServer(srv);
     }
 
-    if (srv->rfd != -1)
-        comm_read(srv->rfd, srv->rbuf + srv->roffset, srv->rbuf_sz - srv->roffset - 1,
-                  helperStatefulHandleRead, srv);
+    if (srv->rfd != -1) {
+        int spaceSize = srv->rbuf_sz - srv->roffset - 1;
+        assert(spaceSize >= 0);
+
+        // grow the input buffer if needed and possible
+        if (!spaceSize && srv->rbuf_sz + 4096 <= ReadBufMaxSize) {
+            srv->rbuf = (char *)memReallocBuf(srv->rbuf, srv->rbuf_sz + 4096, &srv->rbuf_sz);
+            debugs(84, 3, HERE << "Grew read buffer to " << srv->rbuf_sz);
+            spaceSize = srv->rbuf_sz - srv->roffset - 1;
+            assert(spaceSize >= 0);
+        }
+
+        // quit reading if there is no space left
+        if (!spaceSize) {
+            debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " <<
+                   "helper that overflowed " << srv->rbuf_sz << "-byte " <<
+                   "Squid input buffer: " << hlp->id_name << " #" <<
+                   (srv->index + 1));
+            srv->closePipesSafely();
+            return;
+        }
+
+        comm_read(srv->rfd, srv->rbuf + srv->roffset, spaceSize, helperStatefulHandleRead, srv);
+    }
 }
 
 static void