From: Dmitry Kurochkin Date: Mon, 18 Apr 2011 10:28:07 +0000 (+0400) Subject: Improve statistic reporting for shared Rock caches. X-Git-Tag: take06~18 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=39c1e1d9d66a7a610e2afa2775698d5541f3d16e;p=thirdparty%2Fsquid.git Improve statistic reporting for shared Rock caches. Rock cache is shared between multiple processes. We need to make sure cache related stats are not counted twice by different processes. The patch makes Rock store statistics to be reported by the disker process only. Some global variables for cache related stats are replaced with Store class methods. This is needed because it may be difficult or impossible to correctly update these variables for shared caches. The patch also fixes cache manager output for some requests like mgr:storedir: Before the change stats from disker processes were not surrounded with "by kidN". --- diff --git a/src/MemStore.cc b/src/MemStore.cc index d90364f7fd..517a495143 100644 --- a/src/MemStore.cc +++ b/src/MemStore.cc @@ -29,7 +29,7 @@ MemStore::Init() delete map; // we just wanted to initialize shared memory segments } -MemStore::MemStore(): map(NULL) +MemStore::MemStore(): map(NULL), cur_size(0) { } @@ -59,9 +59,8 @@ MemStore::stat(StoreEntry &e) const const int limit = map->entryLimit(); storeAppendPrintf(&e, "Maximum entries: %9d\n", limit); if (limit > 0) { - const int entryCount = map->entryCount(); - storeAppendPrintf(&e, "Current entries: %9d %.2f%%\n", - entryCount, (100.0 * entryCount / limit)); + storeAppendPrintf(&e, "Current entries: %"PRId64" %.2f%%\n", + currentCount(), (100.0 * currentCount() / limit)); if (limit < 100) { // XXX: otherwise too expensive to count Ipc::ReadWriteLockStats stats; @@ -89,6 +88,18 @@ MemStore::maxSize() const return 0; // XXX: make configurable } +uint64_t +MemStore::currentSize() const +{ + return cur_size >> 10; +} + +uint64_t +MemStore::currentCount() const +{ + return map ? map->entryCount() : 0; +} + void MemStore::updateSize(int64_t eSize, int sign) { @@ -300,6 +311,7 @@ MemStore::copyToShm(StoreEntry &e, MemStoreMap::Extras &extras) debugs(20, 7, HERE << "mem-cached all " << eSize << " bytes of " << e << " in " << page); + cur_size += eSize; // remember storage location and size extras.page = page; extras.storedSize = copied; @@ -310,6 +322,7 @@ void MemStore::cleanReadable(const sfileno fileno) { Ipc::Mem::PutPage(map->extras(fileno).page); + cur_size -= map->extras(fileno).storedSize; } /// calculates maximum number of entries we need to store and map diff --git a/src/MemStore.h b/src/MemStore.h index 7709b7019d..b628e25898 100644 --- a/src/MemStore.h +++ b/src/MemStore.h @@ -25,6 +25,8 @@ public: virtual void init(); virtual uint64_t maxSize() const; virtual uint64_t minSize() const; + virtual uint64_t currentSize() const; + virtual uint64_t currentCount() const; virtual void stat(StoreEntry &) const; virtual StoreSearch *search(String const url, HttpRequest *); virtual void reference(StoreEntry &); @@ -49,6 +51,7 @@ protected: private: MemStoreMap *map; ///< index of mem-cached entries + uint64_t cur_size; ///< currently used space in the storage area }; // Why use Store as a base? MemStore and SwapDir are both "caches". diff --git a/src/Store.h b/src/Store.h index 2b2cc54d16..d12d1ffccf 100644 --- a/src/Store.h +++ b/src/Store.h @@ -300,6 +300,12 @@ public: /** The minimum size the store will shrink to via normal housekeeping */ virtual uint64_t minSize() const = 0; + /** current store size in kiloBytes */ + virtual uint64_t currentSize() const = 0; // TODO: return size in bytes + + /** the total number of objects stored */ + virtual uint64_t currentCount() const = 0; + /** * Output stats to the provided store entry. \todo make these calls asynchronous diff --git a/src/StoreHashIndex.h b/src/StoreHashIndex.h index 655d846eec..8ed0c00991 100644 --- a/src/StoreHashIndex.h +++ b/src/StoreHashIndex.h @@ -67,6 +67,10 @@ public: virtual uint64_t minSize() const; + virtual uint64_t currentSize() const; + + virtual uint64_t currentCount() const; + virtual void stat(StoreEntry&) const; virtual void reference(StoreEntry&); diff --git a/src/SwapDir.cc b/src/SwapDir.cc index 353e452c09..9460f7a6a1 100644 --- a/src/SwapDir.cc +++ b/src/SwapDir.cc @@ -39,7 +39,7 @@ #include "ConfigOption.h" SwapDir::SwapDir(char const *aType): theType(aType), - cur_size (0), max_size(0), + cur_size (0), max_size(0), n_disk_objects(0), path(NULL), index(-1), min_objsize(0), max_objsize (-1), repl(NULL), removals(0), scanned(0), cleanLog(NULL) @@ -71,6 +71,9 @@ SwapDir::unlink(StoreEntry &) {} void SwapDir::stat(StoreEntry &output) const { + if (!doReportStat()) + return; + storeAppendPrintf(&output, "Store Directory #%d (%s): %s\n", index, type(), path); storeAppendPrintf(&output, "FS Block Size %d Bytes\n", diff --git a/src/SwapDir.h b/src/SwapDir.h index 2c11895724..df0648f8a5 100644 --- a/src/SwapDir.h +++ b/src/SwapDir.h @@ -69,6 +69,10 @@ public: virtual uint64_t minSize() const; + virtual uint64_t currentSize() const; + + virtual uint64_t currentCount() const; + virtual void stat(StoreEntry &) const; virtual void sync(); /* Sync the store prior to shutdown */ @@ -124,6 +128,8 @@ public: virtual bool needsDiskStrand() const; ///< needs a dedicated kid process virtual bool active() const; ///< may be used in this strand + /// whether stat should be reported by this SwapDir + virtual bool doReportStat() const { return active(); } /* official Store interface functions */ virtual void diskFull(); @@ -135,6 +141,11 @@ public: virtual uint64_t maxSize() const { return max_size;} virtual uint64_t minSize() const; + + virtual uint64_t currentSize() const { return cur_size; } + + virtual uint64_t currentCount() const { return n_disk_objects; } + virtual void stat (StoreEntry &anEntry) const; virtual StoreSearch *search(String const url, HttpRequest *) = 0; @@ -159,8 +170,10 @@ private: char const *theType; public: - uint64_t cur_size; ///< currently used space in the storage area - uint64_t max_size; ///< maximum allocatable size of the storage area + // TODO: store cur_size and max_size in bytes + uint64_t cur_size; ///< currently used space in the storage area in kiloBytes + uint64_t max_size; ///< maximum allocatable size of the storage area in kiloBytes + uint64_t n_disk_objects; ///< total number of objects stored char *path; int index; /* This entry's index into the swapDirs array */ int64_t min_objsize; diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc index 6f033ba759..cd5b99e10e 100644 --- a/src/fs/coss/store_dir_coss.cc +++ b/src/fs/coss/store_dir_coss.cc @@ -427,9 +427,8 @@ storeCossRebuildFromSwapLog(void *data) /* * Make sure we don't unlink the file, it might be * in use by a subsequent entry. Also note that - * we don't have to subtract from store_swap_size - * because adding to store_swap_size happens in - * the cleanup procedure. + * we don't have to subtract from cur_size because + * adding to cur_size happens in the cleanup procedure. */ e->expireNow(); e->releaseRequest(); @@ -481,7 +480,6 @@ storeCossRebuildFromSwapLog(void *data) continue; } - /* update store_swap_size */ rb->counts.objcount++; e = storeCossAddDiskRestore(rb->sd, s.key, diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc index 49cef04cb4..57ed5ab684 100644 --- a/src/fs/rock/RockSwapDir.cc +++ b/src/fs/rock/RockSwapDir.cc @@ -95,6 +95,26 @@ void Rock::SwapDir::disconnect(StoreEntry &e) e.swap_status = SWAPOUT_NONE; } +uint64_t +Rock::SwapDir::currentSize() const +{ + return (HeaderSize + max_objsize * currentCount()) >> 10; +} + +uint64_t +Rock::SwapDir::currentCount() const +{ + return map ? map->entryCount() : 0; +} + +/// In SMP mode only the disker process reports stats to avoid +/// counting the same stats by multiple processes. +bool +Rock::SwapDir::doReportStat() const +{ + return ::SwapDir::doReportStat() && (!UsingSmp() || IamDiskProcess()); +} + // TODO: encapsulate as a tool; identical to CossSwapDir::create() void Rock::SwapDir::create() @@ -284,9 +304,6 @@ Rock::SwapDir::validateOptions() debugs(47, 0, "WARNING: Rock store config wastes space."); } */ - - // XXX: misplaced, map is not yet created - //cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; } void @@ -463,8 +480,6 @@ Rock::SwapDir::ioCompletedNotification() if (!map) map = new DirMap(path); - cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; - // TODO: lower debugging level debugs(47,1, "Rock cache_dir[" << index << "] limits: " << std::setw(12) << maximumSize() << " disk bytes and " << @@ -516,8 +531,6 @@ Rock::SwapDir::writeCompleted(int errflag, size_t rlen, RefCount< ::WriteRequest map->free(sio.swap_filen); // will mark as unusable, just in case } - // TODO: always compute cur_size based on map, do not store it - cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; assert(sio.diskOffset + sio.offset_ <= diskOffsetLimit()); // post-factum sio.finishedWriting(errflag); @@ -532,15 +545,7 @@ Rock::SwapDir::full() const void Rock::SwapDir::updateSize(int64_t size, int sign) { - // it is not clear what store_swap_size really is; TODO: move low-level - // size maintenance to individual store dir types - cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; - store_swap_size = cur_size; - - if (sign > 0) - ++n_disk_objects; - else if (sign < 0) - --n_disk_objects; + // stats are not stored but computed when needed } // storeSwapOutFileClosed calls this nethod on DISK_NO_SPACE_LEFT, @@ -571,8 +576,8 @@ Rock::SwapDir::maintain() if (StoreController::store_dirs_rebuilding) return; - debugs(47,3, HERE << "cache_dir[" << index << "] state: " << - map->full() << ' ' << currentSize() << " < " << diskOffsetLimit()); + debugs(47,3, HERE << "cache_dir[" << index << "] state: " << map->full() << + ' ' << (currentSize() << 10) << " < " << diskOffsetLimit()); // Hopefully, we find a removable entry much sooner (TODO: use time?) const int maxProbed = 10000; @@ -647,8 +652,8 @@ Rock::SwapDir::statfs(StoreEntry &e) const { storeAppendPrintf(&e, "\n"); storeAppendPrintf(&e, "Maximum Size: %"PRIu64" KB\n", max_size); - storeAppendPrintf(&e, "Current Size: %"PRIu64" KB %.2f%%\n", cur_size, - 100.0 * cur_size / max_size); + storeAppendPrintf(&e, "Current Size: %"PRIu64" KB %.2f%%\n", + currentSize(), 100.0 * currentSize() / max_size); if (map) { const int limit = map->entryLimit(); diff --git a/src/fs/rock/RockSwapDir.h b/src/fs/rock/RockSwapDir.h index 00ff9d2c4d..b0c58bf269 100644 --- a/src/fs/rock/RockSwapDir.h +++ b/src/fs/rock/RockSwapDir.h @@ -26,6 +26,9 @@ public: virtual StoreSearch *search(String const url, HttpRequest *); virtual StoreEntry *get(const cache_key *key); virtual void disconnect(StoreEntry &e); + virtual uint64_t currentSize() const; + virtual uint64_t currentCount() const; + virtual bool doReportStat() const; protected: /* protected ::SwapDir API */ @@ -62,7 +65,6 @@ protected: void ignoreReferences(StoreEntry &e); ///< delete from repl policy scope // TODO: change cur_size and max_size type to stop this madness - int64_t currentSize() const { return static_cast(cur_size) << 10;} int64_t maximumSize() const { return static_cast(max_size) << 10;} int64_t diskOffset(int filen) const; int64_t diskOffsetLimit() const; diff --git a/src/fs/ufs/ufscommon.cc b/src/fs/ufs/ufscommon.cc index d1a18cac30..ab5e5a23b9 100644 --- a/src/fs/ufs/ufscommon.cc +++ b/src/fs/ufs/ufscommon.cc @@ -508,9 +508,8 @@ RebuildState::rebuildFromSwapLog() /* * Make sure we don't unlink the file, it might be * in use by a subsequent entry. Also note that - * we don't have to subtract from store_swap_size - * because adding to store_swap_size happens in - * the cleanup procedure. + * we don't have to subtract from cur_size because + * adding to cur_size happens in the cleanup procedure. */ currentEntry()->expireNow(); currentEntry()->releaseRequest(); @@ -647,7 +646,6 @@ RebuildState::rebuildFromSwapLog() (void) 0; } - /* update store_swap_size */ counts.objcount++; currentEntry(sd->addDiskRestore(swapData.key, diff --git a/src/globals.h b/src/globals.h index d2195f7a2b..a9ec17454e 100644 --- a/src/globals.h +++ b/src/globals.h @@ -108,7 +108,6 @@ extern "C" { extern char *snmp_agentinfo; #endif - extern int n_disk_objects; /* 0 */ extern iostats IOStats; extern struct acl_deny_info_list *DenyInfoList; /* NULL */ @@ -117,7 +116,6 @@ extern "C" { extern int starting_up; /* 1 */ extern int shutting_down; /* 0 */ extern int reconfiguring; /* 0 */ - extern unsigned long store_swap_size; /* 0 */ extern time_t hit_only_mode_until; /* 0 */ extern StatCounters statCounter; extern double request_failure_ratio; /* 0.0 */ diff --git a/src/mgr/FunAction.cc b/src/mgr/FunAction.cc index 6fd0f97c1a..aa9e212a76 100644 --- a/src/mgr/FunAction.cc +++ b/src/mgr/FunAction.cc @@ -43,9 +43,9 @@ Mgr::FunAction::dump(StoreEntry* entry) { debugs(16, 5, HERE); Must(entry != NULL); - if (UsingSmp() && IamWorkerProcess()) + if (UsingSmp()) storeAppendPrintf(entry, "by kid%d {\n", KidIdentifier); handler(entry); - if (atomic() && UsingSmp() && IamWorkerProcess()) + if (atomic() && UsingSmp()) storeAppendPrintf(entry, "} by kid%d\n\n", KidIdentifier); } diff --git a/src/mgr/InfoAction.cc b/src/mgr/InfoAction.cc index eb5760ba0b..905a330e1c 100644 --- a/src/mgr/InfoAction.cc +++ b/src/mgr/InfoAction.cc @@ -175,10 +175,10 @@ Mgr::InfoAction::dump(StoreEntry* entry) Must(entry != NULL); #if XMALLOC_STATISTICS - if (UsingSmp() && IamWorkerProcess()) + if (UsingSmp()) storeAppendPrintf(entry, "by kid%d {\n", KidIdentifier); DumpMallocStatistics(entry); - if (UsingSmp() && IamWorkerProcess()) + if (UsingSmp()) storeAppendPrintf(entry, "} by kid%d\n\n", KidIdentifier); #endif if (IamPrimaryProcess()) diff --git a/src/snmp_agent.cc b/src/snmp_agent.cc index 17b0d463cb..9948c45b70 100644 --- a/src/snmp_agent.cc +++ b/src/snmp_agent.cc @@ -67,7 +67,7 @@ snmp_sysFn(variable_list * Var, snint * ErrP) case SYSSTOR: Answer = snmp_var_new_integer(Var->name, Var->name_length, - store_swap_size, + Store::Root().currentSize(), ASN_INTEGER); break; @@ -531,7 +531,7 @@ snmp_prfProtoFn(variable_list * Var, snint * ErrP) case PERF_PROTOSTAT_AGGR_CURSWAP: Answer = snmp_var_new_integer(Var->name, Var->name_length, - (snint) store_swap_size, + (snint) Store::Root().currentSize(), SMI_GAUGE32); break; diff --git a/src/stat.cc b/src/stat.cc index a1413ee2c6..6b0a88c3a7 100644 --- a/src/stat.cc +++ b/src/stat.cc @@ -546,14 +546,16 @@ GetInfo(Mgr::InfoActionData& stats) stats.request_hit_disk_ratio5 = statRequestHitDiskRatio(5); stats.request_hit_disk_ratio60 = statRequestHitDiskRatio(60); - stats.store_swap_size = store_swap_size; + stats.store_swap_size = Store::Root().currentSize(); stats.store_swap_max_size = Store::Root().maxSize(); stats.store_mem_size = mem_node::StoreMemSize(); stats.store_pages_max = store_pages_max; stats.store_mem_used = mem_node::InUseCount(); - stats.objects_size = n_disk_objects ? (double) store_swap_size / n_disk_objects : 0.0; + stats.n_disk_objects = Store::Root().currentCount(); + stats.objects_size = stats.n_disk_objects > 0 ? + (double)Store::Root().currentSize() / stats.n_disk_objects : 0.0; stats.unlink_requests = statCounter.unlink.requests; @@ -666,7 +668,6 @@ GetInfo(Mgr::InfoActionData& stats) stats.store_entries = StoreEntry::inUseCount(); stats.store_mem_entries = MemObject::inUseCount(); stats.hot_obj_count = hot_obj_count; - stats.n_disk_objects = n_disk_objects; } void diff --git a/src/store.cc b/src/store.cc index 6c0ca9dfb5..7d7adbcfe0 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1233,9 +1233,10 @@ StoreController::maintain() /* this should be emitted by the oversize dir, not globally */ - if (store_swap_size > Store::Root().maxSize()) { + if (Store::Root().currentSize() > Store::Root().maxSize()) { if (squid_curtime - last_warn_time > 10) { - debugs(20, 0, "WARNING: Disk space over limit: " << store_swap_size << " KB > " + debugs(20, 0, "WARNING: Disk space over limit: " + << Store::Root().currentSize() << " KB > " << Store::Root().maxSize() << " KB"); last_warn_time = squid_curtime; } diff --git a/src/store_digest.cc b/src/store_digest.cc index 4d419d5e5c..14649ab5f1 100644 --- a/src/store_digest.cc +++ b/src/store_digest.cc @@ -507,7 +507,7 @@ storeDigestCalcCap(void) * number of _entries_ we want to pre-allocate for. */ const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize; - const int lo_cap = 1 + store_swap_size / Config.Store.avgObjectSize; + const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize; const int e_count = StoreEntry::inUseCount(); int cap = e_count ? e_count :hi_cap; debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap << diff --git a/src/store_dir.cc b/src/store_dir.cc index 976d6caf47..4d64c018e5 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -334,7 +334,6 @@ SwapDir::updateSize(int64_t size, int sign) int64_t blks = (size + fs.blksize - 1) / fs.blksize; int64_t k = ((blks * fs.blksize) >> 10) * sign; cur_size += k; - store_swap_size += k; if (sign > 0) n_disk_objects++; @@ -351,10 +350,10 @@ StoreController::stat(StoreEntry &output) const storeAppendPrintf(&output, "Maximum Swap Size : %"PRIu64" KB\n", maxSize()); storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n", - store_swap_size); + currentSize()); storeAppendPrintf(&output, "Current Capacity : %"PRId64"%% used, %"PRId64"%% free\n", - Math::int64Percent(store_swap_size, maxSize()), - Math::int64Percent((maxSize() - store_swap_size), maxSize())); + Math::int64Percent(currentSize(), maxSize()), + Math::int64Percent((maxSize() - currentSize()), maxSize())); if (memStore) memStore->stat(output); @@ -378,6 +377,18 @@ StoreController::minSize() const return swapDir->minSize(); } +uint64_t +StoreController::currentSize() const +{ + return swapDir->currentSize(); +} + +uint64_t +StoreController::currentCount() const +{ + return swapDir->currentCount(); +} + void SwapDir::diskFull() { @@ -900,8 +911,10 @@ StoreHashIndex::maxSize() const { uint64_t result = 0; - for (int i = 0; i < Config.cacheSwap.n_configured; i++) - result += store(i)->maxSize(); + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + if (dir(i).doReportStat()) + result += store(i)->maxSize(); + } return result; } @@ -911,8 +924,36 @@ StoreHashIndex::minSize() const { uint64_t result = 0; - for (int i = 0; i < Config.cacheSwap.n_configured; i++) - result += store(i)->minSize(); + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + if (dir(i).doReportStat()) + result += store(i)->minSize(); + } + + return result; +} + +uint64_t +StoreHashIndex::currentSize() const +{ + uint64_t result = 0; + + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + if (dir(i).doReportStat()) + result += store(i)->currentSize(); + } + + return result; +} + +uint64_t +StoreHashIndex::currentCount() const +{ + uint64_t result = 0; + + for (int i = 0; i < Config.cacheSwap.n_configured; i++) { + if (dir(i).doReportStat()) + result += store(i)->currentCount(); + } return result; } diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc index 0ae8ef1780..1c3edea5f6 100644 --- a/src/store_rebuild.cc +++ b/src/store_rebuild.cc @@ -111,7 +111,7 @@ storeCleanup(void *datanotused) if (currentSearch->isDone()) { debugs(20, 1, " Completed Validation Procedure"); debugs(20, 1, " Validated " << validated << " Entries"); - debugs(20, 1, " store_swap_size = " << store_swap_size); + debugs(20, 1, " store_swap_size = " << Store::Root().currentSize()); StoreController::store_dirs_rebuilding--; assert(0 == StoreController::store_dirs_rebuilding);