Add shared memory page reserve for Rock and check page availability before IO.

The patch adds a shared memory page reserve for Rock (any IPC IO in
general) that can not be used by the memory cache.  Also, Rock checks
if there are free pages available before trying to do IPC IO (both
read and write).  This allows to fail early for cache reads and go
MISS route.

diff --git a/src/MemStore.cc b/src/MemStore.cc
index 5496c089f0f54f1c0238ff8f779a341165f3adcc..a59e4a0b25f4112ea20ffef4948b6b2c243a03e4 100644 (file)
--- a/src/MemStore.cc
+++ b/src/MemStore.cc
@@ -290,7 +290,8 @@ bool
 MemStore::copyToShm(StoreEntry &e, MemStoreMap::Extras &extras)
 {
     Ipc::Mem::PageId page;
-    if (!Ipc::Mem::GetPage(page)) {
+    if (Ipc::Mem::CachePageLevel() > Ipc::Mem::CachePageLimit() ||
+        !Ipc::Mem::GetPage(page)) {
         debugs(20, 5, HERE << "No mem-cache page for " << e);
         return false; // GetPage is responsible for any cleanup on failures
     }

diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc
index ddb3c11465d6188db0617f58f07088d24f850481..9e74a296fecc03c7f88b878904ec464b9528832a 100644 (file)
--- a/src/fs/rock/RockSwapDir.cc
+++ b/src/fs/rock/RockSwapDir.cc
@@ -18,6 +18,7 @@
 #include "fs/rock/RockIoState.h"
 #include "fs/rock/RockIoRequests.h"
 #include "fs/rock/RockRebuild.h"
+#include "ipc/mem/Pages.h"
 
 const int64_t Rock::SwapDir::HeaderSize = 16*1024;
 
@@ -353,6 +354,11 @@ Rock::SwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load)
     if (!map)
         return false;
 
+    if (Ipc::Mem::IoPageLevel() > Ipc::Mem::IoPageLimit()) {
+        debugs(47, 5, HERE << "too few shared pages for IPC IO left");
+        return false;
+    }
+
     if (io->shedLoad())
         return false;
 
@@ -439,6 +445,11 @@ Rock::SwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB *cbFile, StoreIOS
         return NULL;
     }
 
+    if (Ipc::Mem::IoPageLevel() > Ipc::Mem::IoPageLimit()) {
+        debugs(47, 5, HERE << "too few shared pages for IPC IO left");
+        return NULL;
+    }
+
     // The are two ways an entry can get swap_filen: our get() locked it for
     // reading or our storeSwapOutStart() locked it for writing. Peeking at our
     // locked entry is safe, but no support for reading a filling entry.

diff --git a/src/ipc/mem/Pages.cc b/src/ipc/mem/Pages.cc
index 89b3ab3d8011570651e82445602cb023f9e90e3b..68e659494d9f7f870833722291d1cedac22ef679 100644 (file)
--- a/src/ipc/mem/Pages.cc
+++ b/src/ipc/mem/Pages.cc
@@ -47,20 +47,46 @@ Ipc::Mem::PagePointer(const PageId &page)
 }
 
 size_t
-Ipc::Mem::Limit()
+Ipc::Mem::PageLimit()
+{
+    return ThePagePool ? ThePagePool->capacity() : 0;
+}
+
+size_t
+Ipc::Mem::CachePageLimit()
 {
     // TODO: adjust cache_mem description to say that in SMP mode,
     // in-transit objects are not allocated using cache_mem. Eventually,
     // they should not use cache_mem even if shared memory is not used:
     // in-transit objects have nothing to do with caching.
-    return Config.memMaxSize;
+    return Config.memMaxSize > 0 ? Config.memMaxSize / PageSize() : 0;
+}
+
+size_t
+Ipc::Mem::IoPageLimit()
+{
+    // XXX: this should be independent from memory cache pages
+    return CachePageLimit();
+}
+
+size_t
+Ipc::Mem::PageLevel()
+{
+    return ThePagePool ? ThePagePool->capacity() - ThePagePool->size() : 0;
+}
+
+size_t
+Ipc::Mem::CachePageLevel()
+{
+    // TODO: make a separate counter for shared memory pages for memory cache
+    return PageLevel();
 }
 
 size_t
-Ipc::Mem::Level()
+Ipc::Mem::IoPageLevel()
 {
-    return ThePagePool ?
-        (ThePagePool->capacity() - ThePagePool->size()) * PageSize() : 0;
+    // TODO: make a separate counter for shared memory pages for IPC IO
+    return PageLevel();
 }
 
 /// initializes shared memory pages
@@ -86,13 +112,13 @@ void SharedMemPagesRr::run(const RunnerRegistry &)
 
     // When cache_dirs start using shared memory pages, they would
     // need to communicate their needs to us somehow.
-    if (!Ipc::Mem::Limit())
+    if (Config.memMaxSize <= 0)
         return;
 
-    if (Ipc::Mem::Limit() < Ipc::Mem::PageSize()) {
+    if (Ipc::Mem::CachePageLimit() <= 0) {
         if (IamMasterProcess()) {
             debugs(54, DBG_IMPORTANT, "WARNING: mem-cache size is too small ("
-                   << (Ipc::Mem::Limit() / 1024.0) << " KB), should be >= " <<
+                   << (Config.memMaxSize / 1024.0) << " KB), should be >= " <<
                    (Ipc::Mem::PageSize() / 1024.0) << " KB");
         }
         return;
@@ -100,7 +126,8 @@ void SharedMemPagesRr::run(const RunnerRegistry &)
 
     if (IamMasterProcess()) {
         Must(!owner);
-        const size_t capacity = Ipc::Mem::Limit() / Ipc::Mem::PageSize();
+        // reserve 10% for IPC IO
+        const size_t capacity = Ipc::Mem::CachePageLimit() * 1.1;
         owner = Ipc::Mem::PagePool::Init(PagePoolId, capacity, Ipc::Mem::PageSize());
     }
 

diff --git a/src/ipc/mem/Pages.h b/src/ipc/mem/Pages.h
index 9e54c4bae1a22a203b70fe7162ee465dbe9b74fa..e2615eef72379f0f8ac90becdd396f0142af93d6 100644 (file)
--- a/src/ipc/mem/Pages.h
+++ b/src/ipc/mem/Pages.h
@@ -26,14 +26,35 @@ char *PagePointer(const PageId &page);
 
 /* Limits and statistics */
 
-/// the total number of shared memory bytes that can be in use at any time
-size_t Limit();
+/// the total number of shared memory pages that can be in use at any time
+size_t PageLimit();
 
-/// approximate total number of shared memory bytes used now
-size_t Level();
+/// the total number of shared memory pages for memory cache that can be in
+/// use at any time
+size_t CachePageLimit();
 
-/// approximate total number of shared memory bytes we can allocate now
-inline size_t Available() { return Limit() - Level(); }
+/// the total number of shared memory pages for IPC IO that can be in
+/// use at any time
+size_t IoPageLimit();
+
+/// approximate total number of shared memory pages used now
+size_t PageLevel();
+
+/// approximate total number of shared memory pages for memory cache used now
+size_t CachePageLevel();
+
+/// approximate total number of shared memory pages for IPC IO used now
+size_t IoPageLevel();
+
+/// approximate total number of shared memory pages we can allocate now
+inline size_t PagesAvailable() { return PageLimit() - PageLevel(); }
+
+/// approximate total number of shared memory pages for memory cache we can
+/// allocate now
+inline size_t CachePagesAvailable() { return CachePageLimit() - CachePageLevel(); }
+
+/// approximate total number of shared memory pages for IPC IO we can allocate now
+inline size_t IoPagesAvailable() { return IoPageLimit() - IoPageLevel(); }
 
 /// returns page size in bytes; all pages are assumed to be the same size
 size_t PageSize();