Independent shared I/O page limit.

Shared memory pages are used for shared memory cache and IPC I/O module.
Before this change, the number of shared memory pages needed for IPC I/O
was calculated from the size of shared memory cache.  Moreover, shared
memory cache was required for IPC I/O.

The patch makes the limit for shared I/O pages independent from the
shared memory cache size and presence.  IPC I/O pages limit is calculated
from the number of workers and diskers; it does not depend on cache_dir
configuration.  This may change in the future if we learn how to compute
it (e.g., by multiplying max-swap-rate and swap-timeout if both are
available).

diff --git a/src/DiskIO/IpcIo/IpcIoFile.cc b/src/DiskIO/IpcIo/IpcIoFile.cc
index 94271e59865d32cbcb157122c82b84d955d99c72..3c8177069709b396ddbee2e3a731f28085e6958d 100644 (file)
--- a/src/DiskIO/IpcIo/IpcIoFile.cc
+++ b/src/DiskIO/IpcIo/IpcIoFile.cc
@@ -23,6 +23,11 @@ CBDATA_CLASS_INIT(IpcIoFile);
 
 /// shared memory segment path to use for IpcIoFile maps
 static const char *const ShmLabel = "io_file";
+/// a single worker-to-disker or disker-to-worker queue capacity; up
+/// to 2*QueueCapacity I/O requests queued between a single worker and
+/// a single disker
+// TODO: make configurable or compute from squid.conf settings if possible
+static const int QueueCapacity = 1024;
 
 const double IpcIoFile::Timeout = 7; // seconds;  XXX: ALL,9 may require more
 IpcIoFile::IpcIoFileList IpcIoFile::WaitingForOpen;
@@ -843,6 +848,30 @@ DiskerClose(const String &path)
 }
 
 
+/// reports our needs for shared memory pages to Ipc::Mem::Pages
+class IpcIoClaimMemoryNeedsRr: public RegisteredRunner
+{
+public:
+    /* RegisteredRunner API */
+    virtual void run(const RunnerRegistry &r);
+};
+
+RunnerRegistrationEntry(rrClaimMemoryNeeds, IpcIoClaimMemoryNeedsRr);
+
+
+void
+IpcIoClaimMemoryNeedsRr::run(const RunnerRegistry &)
+{
+    const int itemsCount = Ipc::FewToFewBiQueue::MaxItemsCount(
+        ::Config.workers, ::Config.cacheSwap.n_strands, QueueCapacity);
+    // the maximum number of shared I/O pages is approximately the
+    // number of queue slots, we add a fudge factor to that to account
+    // for corner cases where I/O pages are created before queue
+    // limits are checked or destroyed long after the I/O is dequeued
+    Ipc::Mem::NotePageNeed(Ipc::Mem::PageId::ioPage, itemsCount * 1.1);
+}
+
+
 /// initializes shared memory segments used by IpcIoFile
 class IpcIoRr: public Ipc::Mem::RegisteredRunner
 {
@@ -867,11 +896,10 @@ void IpcIoRr::create(const RunnerRegistry &)
         return;
 
     Must(!owner);
-    // XXX: make capacity configurable
     owner = Ipc::FewToFewBiQueue::Init(ShmLabel, Config.workers, 1,
                                        Config.cacheSwap.n_strands,
                                        1 + Config.workers, sizeof(IpcIoMsg),
-                                       1024);
+                                       QueueCapacity);
 }
 
 IpcIoRr::~IpcIoRr()

diff --git a/src/MemStore.cc b/src/MemStore.cc
index 02d4cabfaaa3d8a9a89d2ed5101846a60b7a2c3b..42d82b9724c8085ba209b59fe22b2d3863f2d6d6 100644 (file)
--- a/src/MemStore.cc
+++ b/src/MemStore.cc
@@ -356,6 +356,24 @@ MemStore::EntryLimit()
 }
 
 
+/// reports our needs for shared memory pages to Ipc::Mem::Pages
+class MemStoreClaimMemoryNeedsRr: public RegisteredRunner
+{
+public:
+    /* RegisteredRunner API */
+    virtual void run(const RunnerRegistry &r);
+};
+
+RunnerRegistrationEntry(rrClaimMemoryNeeds, MemStoreClaimMemoryNeedsRr);
+
+
+void
+MemStoreClaimMemoryNeedsRr::run(const RunnerRegistry &)
+{
+    Ipc::Mem::NotePageNeed(Ipc::Mem::PageId::cachePage, MemStore::EntryLimit());
+}
+
+
 /// initializes shared memory segments used by MemStore
 class MemStoreRr: public Ipc::Mem::RegisteredRunner
 {
@@ -402,8 +420,14 @@ void MemStoreRr::create(const RunnerRegistry &)
 
     Must(!owner);
     const int64_t entryLimit = MemStore::EntryLimit();
-    if (entryLimit <= 0)
+    if (entryLimit <= 0) {
+        if (Config.memMaxSize > 0) {
+            debugs(20, DBG_IMPORTANT, "WARNING: mem-cache size is too small ("
+                   << (Config.memMaxSize / 1024.0) << " KB), should be >= " <<
+                   (Ipc::Mem::PageSize() / 1024.0) << " KB");
+        }
         return; // no memory cache configured or a misconfiguration
+    }
     owner = MemStoreMap::Init(ShmLabel, entryLimit);
 }
 

diff --git a/src/base/RunnersRegistry.h b/src/base/RunnersRegistry.h
index 06e68e61ba943a72f79ce777deee24ab274e9870..2d5b84048d12fa1751c386283672f7157fd1bf8f 100644 (file)
--- a/src/base/RunnersRegistry.h
+++ b/src/base/RunnersRegistry.h
@@ -21,6 +21,11 @@
 
 /// well-known registries
 typedef enum {
+    /// managed by main.cc; activated after parsing squid.conf but
+    /// before rrAfterConfig, deactivated after rrAfterConfig but
+    /// before freeing configuration-related memory or exit()-ing
+    rrClaimMemoryNeeds,
+
     /// managed by main.cc; activated after parsing squid.conf and
     /// deactivated before freeing configuration-related memory or exit()-ing
     rrAfterConfig,

diff --git a/src/ipc/Queue.cc b/src/ipc/Queue.cc
index 76e266b68418b71c9ccf94478ab46abc409ddd19..4cba4bff26693e787e3040a25964e770c9539bac 100644 (file)
--- a/src/ipc/Queue.cc
+++ b/src/ipc/Queue.cc
@@ -149,6 +149,12 @@ Ipc::FewToFewBiQueue::FewToFewBiQueue(const String &id, const Group aLocalGroup,
     debugs(54, 7, HERE << "queue " << id << " reader: " << localReader.id);
 }
 
+int
+Ipc::FewToFewBiQueue::MaxItemsCount(const int groupASize, const int groupBSize, const int capacity)
+{
+    return capacity * groupASize * groupBSize * 2;
+}
+
 bool
 Ipc::FewToFewBiQueue::validProcessId(const Group group, const int processId) const
 {

diff --git a/src/ipc/Queue.h b/src/ipc/Queue.h
index 640342d9739a7087bb3b88fc482676c0b1f39d41..c562379d5f140241fd53f9ae444dc7ec2af4ecaa 100644 (file)
--- a/src/ipc/Queue.h
+++ b/src/ipc/Queue.h
@@ -186,6 +186,9 @@ public:
     enum Group { groupA = 0, groupB = 1 };
     FewToFewBiQueue(const String &id, const Group aLocalGroup, const int aLocalProcessId);
 
+    /// maximum number of items in the queue
+    static int MaxItemsCount(const int groupASize, const int groupBSize, const int capacity);
+
     Group localGroup() const { return theLocalGroup; }
     Group remoteGroup() const { return theLocalGroup == groupA ? groupB : groupA; }
 

diff --git a/src/ipc/mem/Pages.cc b/src/ipc/mem/Pages.cc
index 054938bf8cc842df7299e706782ca3ada067f6e0..c97487d2fe153a61ff53d7276488580845f34ed3 100644 (file)
--- a/src/ipc/mem/Pages.cc
+++ b/src/ipc/mem/Pages.cc
@@ -19,6 +19,7 @@
 // TODO: make pool id more unique so it does not conflict with other Squids?
 static const char *PagePoolId = "squid-page-pool";
 static Ipc::Mem::PagePool *ThePagePool = 0;
+static int TheLimits[Ipc::Mem::PageId::maxPurpose];
 
 // TODO: make configurable to avoid waste when mem-cached objects are small/big
 size_t
@@ -60,16 +61,17 @@ Ipc::Mem::PageLimit()
 size_t
 Ipc::Mem::PageLimit(const int purpose)
 {
-    switch (purpose) {
-    case PageId::cachePage:
-        return Config.memMaxSize > 0 ? Config.memMaxSize / PageSize() : 0;
-    case PageId::ioPage:
-        // XXX: this should be independent from memory cache pages
-        return PageLimit(PageId::cachePage)/2;
-    default:
-        Must(false);
-    }
-    return 0;
+    Must(0 <= purpose && purpose <= PageId::maxPurpose);
+    return TheLimits[purpose];
+}
+
+// note: adjust this if we start recording needs during reconfigure
+void
+Ipc::Mem::NotePageNeed(const int purpose, const int count)
+{
+    Must(0 <= purpose && purpose <= PageId::maxPurpose);
+    Must(count >= 0);
+    TheLimits[purpose] += count;
 }
 
 size_t
@@ -105,22 +107,8 @@ RunnerRegistrationEntry(rrAfterConfig, SharedMemPagesRr);
 void
 SharedMemPagesRr::run(const RunnerRegistry &r)
 {
-    if (!UsingSmp())
-        return;
-
-    // When cache_dirs start using shared memory pages, they would
-    // need to communicate their needs to us somehow.
-    if (Config.memMaxSize <= 0)
-        return;
-
-    if (Ipc::Mem::PageLimit() <= 0) {
-        if (IamMasterProcess()) {
-            debugs(54, DBG_IMPORTANT, "WARNING: mem-cache size is too small ("
-                   << (Config.memMaxSize / 1024.0) << " KB), should be >= " <<
-                   (Ipc::Mem::PageSize() / 1024.0) << " KB");
-        }
+    if (Ipc::Mem::PageLimit() <= 0)
         return;
-    }
 
     Ipc::Mem::RegisteredRunner::run(r);
 }

diff --git a/src/ipc/mem/Pages.h b/src/ipc/mem/Pages.h
index aeea1890e15b6a664ec2e9fe3d9d9a621d53899f..bb626d10e321d6f0507fcc7de6e19876f3059d41 100644 (file)
--- a/src/ipc/mem/Pages.h
+++ b/src/ipc/mem/Pages.h
@@ -51,6 +51,9 @@ inline size_t PagesAvailable(const int purpose) { return PageLimit(purpose) - Pa
 /// returns page size in bytes; all pages are assumed to be the same size
 size_t PageSize();
 
+/// claim the need for a number of pages for a given purpose
+void NotePageNeed(const int purpose, const int count);
+
 } // namespace Mem
 
 } // namespace Ipc

diff --git a/src/main.cc b/src/main.cc
index 546eb92bed685d7d856602b2f11c23c0315d5317..21e45a24ba478debd41637dc2d57acf151673720 100644 (file)
--- a/src/main.cc
+++ b/src/main.cc
@@ -1442,6 +1442,7 @@ SquidMain(int argc, char **argv)
 
     debugs(1,2, HERE << "Doing post-config initialization\n");
     leave_suid();
+    ActivateRegistered(rrClaimMemoryNeeds);
     ActivateRegistered(rrAfterConfig);
     enter_suid();
 
@@ -1811,6 +1812,7 @@ watch_child(char *argv[])
         if (!TheKids.someRunning() && !TheKids.shouldRestartSome()) {
             leave_suid();
             DeactivateRegistered(rrAfterConfig);
+            DeactivateRegistered(rrClaimMemoryNeeds);
             enter_suid();
 
             if (TheKids.someSignaled(SIGINT) || TheKids.someSignaled(SIGTERM)) {
@@ -1913,6 +1915,7 @@ SquidShutdown()
     StoreFileSystem::FreeAllFs();
     DiskIOModule::FreeAllModules();
     DeactivateRegistered(rrAfterConfig);
+    DeactivateRegistered(rrClaimMemoryNeeds);
 #if LEAK_CHECK_MODE && 0 /* doesn't work at the moment */
 
     configFreeMemory();