Optimization: Make read requests in [Rock] IpcIo bypass max-swap-rate limit.

Before the change, IpcIoFile::WaitBeforePop() delayed both swap ins
(hits) and swap outs (misses).  This is suboptimal because reads do
not usually accumulate unfinished I/O requests in OS buffers and,
hence, do not eventually require the OS to block all I/O.

Ideally, a disker should probably dequeue all pending disker requests,
satisfy reads ASAP, and then handle writes, but that is difficult for
several reasons.  The patch implements a simpler approach: peek the
next request to be popped, and if it is a swap in (i.e., read or hit),
then pop it without any delay.

When a read is popped, we still adjust the balance member and LastIo,
because we do want to maintain the configured average I/O rate. When a
write request comes in, it will be delayed [longer] if needed.

In the extreme case of a very long stream of read requests (no writes
at all), there will be essentially no I/O rate limit and that is what
we want.

diff --git a/src/DiskIO/IpcIo/IpcIoFile.cc b/src/DiskIO/IpcIo/IpcIoFile.cc
index e27a167ad10ff95d6f517d4931e97b7be3c55d14..63ca74198f6418f6f5b98645b0e5f0f934b51699 100644 (file)
--- a/src/DiskIO/IpcIo/IpcIoFile.cc
+++ b/src/DiskIO/IpcIo/IpcIoFile.cc
@@ -690,8 +690,10 @@ IpcIoFile::WaitBeforePop()
         return false;
 
     // is there an I/O request we could potentially delay?
-    if (!queue->popReady()) {
-        // unlike pop(), popReady() is not reliable and does not block reader
+    int processId;
+    IpcIoMsg ipcIo;
+    if (!queue->peek(processId, ipcIo)) {
+        // unlike pop(), peek() is not reliable and does not block reader
         // so we must proceed with pop() even if it is likely to fail
         return false;
     }
@@ -711,9 +713,10 @@ IpcIoFile::WaitBeforePop()
 
     debugs(47, 7, HERE << "rate limiting balance: " << balance << " after +" << credit << " -" << debit);
 
-    if (balance > maxImbalance) {
-        // if we accumulated too much time for future slow I/Os,
-        // then shed accumulated time to keep just half of the excess
+    if (ipcIo.command == IpcIo::cmdWrite && balance > maxImbalance) {
+        // if the next request is (likely) write and we accumulated
+        // too much time for future slow I/Os, then shed accumulated
+        // time to keep just half of the excess
         const int64_t toSpend = balance - maxImbalance/2;
 
         if (toSpend/1e3 > Timeout)

diff --git a/src/cf.data.pre b/src/cf.data.pre
index 5d1017db5d2f169836f42c81832995f773a33967..b682b09b2b9f05ad22f3060fb086e9673ae9fc80 100644 (file)
--- a/src/cf.data.pre
+++ b/src/cf.data.pre
@@ -2806,9 +2806,13 @@ DOC_START
        expected swap wait time.
 
        max-swap-rate=swaps/sec: Artificially limits disk access using
-       the specified I/O rate limit. Swap in and swap out requests that
+       the specified I/O rate limit. Swap out requests that
        would cause the average I/O rate to exceed the limit are
-       delayed. This is necessary on file systems that buffer "too
+       delayed. Individual swap in requests (i.e., hits or reads) are
+       not delayed, but they do contribute to measured swap rate and
+       since they are placed in the same FIFO queue as swap out
+       requests, they may wait longer if max-swap-rate is smaller.
+       This is necessary on file systems that buffer "too
        many" writes and then start blocking Squid and other processes
        while committing those writes to disk.  Usually used together
        with swap-timeout to avoid excessive delays and queue overflows

diff --git a/src/ipc/Queue.cc b/src/ipc/Queue.cc
index 4cba4bff26693e787e3040a25964e770c9539bac..6627aecb5d9ecd01fccae301995577ed1478fae7 100644 (file)
--- a/src/ipc/Queue.cc
+++ b/src/ipc/Queue.cc
@@ -255,21 +255,6 @@ Ipc::FewToFewBiQueue::clearReaderSignal(const int remoteProcessId)
     // theLastPopProcessId = remoteProcessId;
 }
 
-bool
-Ipc::FewToFewBiQueue::popReady() const
-{
-    // mimic FewToFewBiQueue::pop() but quit just before popping
-    int popProcessId = theLastPopProcessId; // preserve for future pop()
-    for (int i = 0; i < remoteGroupSize(); ++i) {
-        if (++popProcessId >= remoteGroupIdOffset() + remoteGroupSize())
-            popProcessId = remoteGroupIdOffset();
-        const OneToOneUniQueue &queue = oneToOneQueue(remoteGroup(), popProcessId, theLocalGroup, theLocalProcessId);
-        if (!queue.empty())
-            return true;
-    }
-    return false; // most likely, no process had anything to pop
-}
-
 Ipc::QueueReader::Balance &
 Ipc::FewToFewBiQueue::localBalance()
 {

diff --git a/src/ipc/Queue.h b/src/ipc/Queue.h
index b9d6e2b3a403c03cde8c9075d69f3b19363c118a..5f80b3aa2c0a17d92a22d322006890db5d58a2fe 100644 (file)
--- a/src/ipc/Queue.h
+++ b/src/ipc/Queue.h
@@ -205,8 +205,8 @@ public:
     /// us and the given remote process
     template<class Value> bool findOldest(const int remoteProcessId, Value &value) const;
 
-    /// returns true if pop() would have probably succeeded but does not pop()
-    bool popReady() const;
+    /// peeks at the item likely to be pop()ed next
+    template<class Value> bool peek(int &remoteProcessId, Value &value) const;
 
     /// returns local reader's balance
     QueueReader::Balance &localBalance();
@@ -387,6 +387,26 @@ FewToFewBiQueue::findOldest(const int remoteProcessId, Value &value) const
     return out.peek(value);
 }
 
+template <class Value>
+bool
+FewToFewBiQueue::peek(int &remoteProcessId, Value &value) const
+{
+    // mimic FewToFewBiQueue::pop() but quit just before popping
+    int popProcessId = theLastPopProcessId; // preserve for future pop()
+    for (int i = 0; i < remoteGroupSize(); ++i) {
+        if (++popProcessId >= remoteGroupIdOffset() + remoteGroupSize())
+            popProcessId = remoteGroupIdOffset();
+        const OneToOneUniQueue &queue =
+            oneToOneQueue(remoteGroup(), popProcessId,
+                          theLocalGroup, theLocalProcessId);
+        if (queue.peek(value)) {
+            remoteProcessId = popProcessId;
+            return true;
+        }
+    }
+    return false; // most likely, no process had anything to pop
+}
+
 } // namespace Ipc
 
 #endif // SQUID_IPC_QUEUE_H