From: Dmitry Kurochkin Date: Thu, 27 Oct 2011 21:57:26 +0000 (-0600) Subject: Independent shared I/O page limit. X-Git-Tag: BumpSslServerFirst.take01~68 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ea2cdeb6461a5d0969a6357ec4e945ee3b4f0b0d;p=thirdparty%2Fsquid.git Independent shared I/O page limit. Shared memory pages are used for shared memory cache and IPC I/O module. Before this change, the number of shared memory pages needed for IPC I/O was calculated from the size of shared memory cache. Moreover, shared memory cache was required for IPC I/O. The patch makes the limit for shared I/O pages independent from the shared memory cache size and presence. IPC I/O pages limit is calculated from the number of workers and diskers; it does not depend on cache_dir configuration. This may change in the future if we learn how to compute it (e.g., by multiplying max-swap-rate and swap-timeout if both are available). --- diff --git a/src/DiskIO/IpcIo/IpcIoFile.cc b/src/DiskIO/IpcIo/IpcIoFile.cc index 94271e5986..3c81770697 100644 --- a/src/DiskIO/IpcIo/IpcIoFile.cc +++ b/src/DiskIO/IpcIo/IpcIoFile.cc @@ -23,6 +23,11 @@ CBDATA_CLASS_INIT(IpcIoFile); /// shared memory segment path to use for IpcIoFile maps static const char *const ShmLabel = "io_file"; +/// a single worker-to-disker or disker-to-worker queue capacity; up +/// to 2*QueueCapacity I/O requests queued between a single worker and +/// a single disker +// TODO: make configurable or compute from squid.conf settings if possible +static const int QueueCapacity = 1024; const double IpcIoFile::Timeout = 7; // seconds; XXX: ALL,9 may require more IpcIoFile::IpcIoFileList IpcIoFile::WaitingForOpen; @@ -843,6 +848,30 @@ DiskerClose(const String &path) } +/// reports our needs for shared memory pages to Ipc::Mem::Pages +class IpcIoClaimMemoryNeedsRr: public RegisteredRunner +{ +public: + /* RegisteredRunner API */ + virtual void run(const RunnerRegistry &r); +}; + +RunnerRegistrationEntry(rrClaimMemoryNeeds, IpcIoClaimMemoryNeedsRr); + + +void +IpcIoClaimMemoryNeedsRr::run(const RunnerRegistry &) +{ + const int itemsCount = Ipc::FewToFewBiQueue::MaxItemsCount( + ::Config.workers, ::Config.cacheSwap.n_strands, QueueCapacity); + // the maximum number of shared I/O pages is approximately the + // number of queue slots, we add a fudge factor to that to account + // for corner cases where I/O pages are created before queue + // limits are checked or destroyed long after the I/O is dequeued + Ipc::Mem::NotePageNeed(Ipc::Mem::PageId::ioPage, itemsCount * 1.1); +} + + /// initializes shared memory segments used by IpcIoFile class IpcIoRr: public Ipc::Mem::RegisteredRunner { @@ -867,11 +896,10 @@ void IpcIoRr::create(const RunnerRegistry &) return; Must(!owner); - // XXX: make capacity configurable owner = Ipc::FewToFewBiQueue::Init(ShmLabel, Config.workers, 1, Config.cacheSwap.n_strands, 1 + Config.workers, sizeof(IpcIoMsg), - 1024); + QueueCapacity); } IpcIoRr::~IpcIoRr() diff --git a/src/MemStore.cc b/src/MemStore.cc index 02d4cabfaa..42d82b9724 100644 --- a/src/MemStore.cc +++ b/src/MemStore.cc @@ -356,6 +356,24 @@ MemStore::EntryLimit() } +/// reports our needs for shared memory pages to Ipc::Mem::Pages +class MemStoreClaimMemoryNeedsRr: public RegisteredRunner +{ +public: + /* RegisteredRunner API */ + virtual void run(const RunnerRegistry &r); +}; + +RunnerRegistrationEntry(rrClaimMemoryNeeds, MemStoreClaimMemoryNeedsRr); + + +void +MemStoreClaimMemoryNeedsRr::run(const RunnerRegistry &) +{ + Ipc::Mem::NotePageNeed(Ipc::Mem::PageId::cachePage, MemStore::EntryLimit()); +} + + /// initializes shared memory segments used by MemStore class MemStoreRr: public Ipc::Mem::RegisteredRunner { @@ -402,8 +420,14 @@ void MemStoreRr::create(const RunnerRegistry &) Must(!owner); const int64_t entryLimit = MemStore::EntryLimit(); - if (entryLimit <= 0) + if (entryLimit <= 0) { + if (Config.memMaxSize > 0) { + debugs(20, DBG_IMPORTANT, "WARNING: mem-cache size is too small (" + << (Config.memMaxSize / 1024.0) << " KB), should be >= " << + (Ipc::Mem::PageSize() / 1024.0) << " KB"); + } return; // no memory cache configured or a misconfiguration + } owner = MemStoreMap::Init(ShmLabel, entryLimit); } diff --git a/src/base/RunnersRegistry.h b/src/base/RunnersRegistry.h index 06e68e61ba..2d5b84048d 100644 --- a/src/base/RunnersRegistry.h +++ b/src/base/RunnersRegistry.h @@ -21,6 +21,11 @@ /// well-known registries typedef enum { + /// managed by main.cc; activated after parsing squid.conf but + /// before rrAfterConfig, deactivated after rrAfterConfig but + /// before freeing configuration-related memory or exit()-ing + rrClaimMemoryNeeds, + /// managed by main.cc; activated after parsing squid.conf and /// deactivated before freeing configuration-related memory or exit()-ing rrAfterConfig, diff --git a/src/ipc/Queue.cc b/src/ipc/Queue.cc index 76e266b684..4cba4bff26 100644 --- a/src/ipc/Queue.cc +++ b/src/ipc/Queue.cc @@ -149,6 +149,12 @@ Ipc::FewToFewBiQueue::FewToFewBiQueue(const String &id, const Group aLocalGroup, debugs(54, 7, HERE << "queue " << id << " reader: " << localReader.id); } +int +Ipc::FewToFewBiQueue::MaxItemsCount(const int groupASize, const int groupBSize, const int capacity) +{ + return capacity * groupASize * groupBSize * 2; +} + bool Ipc::FewToFewBiQueue::validProcessId(const Group group, const int processId) const { diff --git a/src/ipc/Queue.h b/src/ipc/Queue.h index 640342d973..c562379d5f 100644 --- a/src/ipc/Queue.h +++ b/src/ipc/Queue.h @@ -186,6 +186,9 @@ public: enum Group { groupA = 0, groupB = 1 }; FewToFewBiQueue(const String &id, const Group aLocalGroup, const int aLocalProcessId); + /// maximum number of items in the queue + static int MaxItemsCount(const int groupASize, const int groupBSize, const int capacity); + Group localGroup() const { return theLocalGroup; } Group remoteGroup() const { return theLocalGroup == groupA ? groupB : groupA; } diff --git a/src/ipc/mem/Pages.cc b/src/ipc/mem/Pages.cc index 054938bf8c..c97487d2fe 100644 --- a/src/ipc/mem/Pages.cc +++ b/src/ipc/mem/Pages.cc @@ -19,6 +19,7 @@ // TODO: make pool id more unique so it does not conflict with other Squids? static const char *PagePoolId = "squid-page-pool"; static Ipc::Mem::PagePool *ThePagePool = 0; +static int TheLimits[Ipc::Mem::PageId::maxPurpose]; // TODO: make configurable to avoid waste when mem-cached objects are small/big size_t @@ -60,16 +61,17 @@ Ipc::Mem::PageLimit() size_t Ipc::Mem::PageLimit(const int purpose) { - switch (purpose) { - case PageId::cachePage: - return Config.memMaxSize > 0 ? Config.memMaxSize / PageSize() : 0; - case PageId::ioPage: - // XXX: this should be independent from memory cache pages - return PageLimit(PageId::cachePage)/2; - default: - Must(false); - } - return 0; + Must(0 <= purpose && purpose <= PageId::maxPurpose); + return TheLimits[purpose]; +} + +// note: adjust this if we start recording needs during reconfigure +void +Ipc::Mem::NotePageNeed(const int purpose, const int count) +{ + Must(0 <= purpose && purpose <= PageId::maxPurpose); + Must(count >= 0); + TheLimits[purpose] += count; } size_t @@ -105,22 +107,8 @@ RunnerRegistrationEntry(rrAfterConfig, SharedMemPagesRr); void SharedMemPagesRr::run(const RunnerRegistry &r) { - if (!UsingSmp()) - return; - - // When cache_dirs start using shared memory pages, they would - // need to communicate their needs to us somehow. - if (Config.memMaxSize <= 0) - return; - - if (Ipc::Mem::PageLimit() <= 0) { - if (IamMasterProcess()) { - debugs(54, DBG_IMPORTANT, "WARNING: mem-cache size is too small (" - << (Config.memMaxSize / 1024.0) << " KB), should be >= " << - (Ipc::Mem::PageSize() / 1024.0) << " KB"); - } + if (Ipc::Mem::PageLimit() <= 0) return; - } Ipc::Mem::RegisteredRunner::run(r); } diff --git a/src/ipc/mem/Pages.h b/src/ipc/mem/Pages.h index aeea1890e1..bb626d10e3 100644 --- a/src/ipc/mem/Pages.h +++ b/src/ipc/mem/Pages.h @@ -51,6 +51,9 @@ inline size_t PagesAvailable(const int purpose) { return PageLimit(purpose) - Pa /// returns page size in bytes; all pages are assumed to be the same size size_t PageSize(); +/// claim the need for a number of pages for a given purpose +void NotePageNeed(const int purpose, const int count); + } // namespace Mem } // namespace Ipc diff --git a/src/main.cc b/src/main.cc index 546eb92bed..21e45a24ba 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1442,6 +1442,7 @@ SquidMain(int argc, char **argv) debugs(1,2, HERE << "Doing post-config initialization\n"); leave_suid(); + ActivateRegistered(rrClaimMemoryNeeds); ActivateRegistered(rrAfterConfig); enter_suid(); @@ -1811,6 +1812,7 @@ watch_child(char *argv[]) if (!TheKids.someRunning() && !TheKids.shouldRestartSome()) { leave_suid(); DeactivateRegistered(rrAfterConfig); + DeactivateRegistered(rrClaimMemoryNeeds); enter_suid(); if (TheKids.someSignaled(SIGINT) || TheKids.someSignaled(SIGTERM)) { @@ -1913,6 +1915,7 @@ SquidShutdown() StoreFileSystem::FreeAllFs(); DiskIOModule::FreeAllModules(); DeactivateRegistered(rrAfterConfig); + DeactivateRegistered(rrClaimMemoryNeeds); #if LEAK_CHECK_MODE && 0 /* doesn't work at the moment */ configFreeMemory();