From: Amos Jeffries Date: Sat, 4 Dec 2010 16:23:14 +0000 (-0700) Subject: Bug 3068: cache_dir capacity and usage overflows X-Git-Tag: SQUID_3_1_10~34 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7c7b952e676ef4b435cbb5b0c862de774c675536;p=thirdparty%2Fsquid.git Bug 3068: cache_dir capacity and usage overflows Makes usage calculations use size_t instead of int and updates the relevant fields storing the cache_dir capacity and usage fields as well. This fixes Squid filling cache_dir with files >2GB in size. Also allows Squid to store more than 2TB of data total in one dir. --- diff --git a/src/SquidMath.cc b/src/SquidMath.cc index 946d3f15c5..b4160ba6d1 100644 --- a/src/SquidMath.cc +++ b/src/SquidMath.cc @@ -7,6 +7,12 @@ Math::intPercent(const int a, const int b) return b ? ((int) (100.0 * a / b + 0.5)) : 0; } +int64_t +Math::int64Percent(const int64_t a, const int64_t b) +{ + return b ? ((int64_t) (100.0 * a / b + 0.5)) : 0; +} + double Math::doublePercent(const double a, const double b) { diff --git a/src/SquidMath.h b/src/SquidMath.h index a8fc22abb9..afb033825b 100644 --- a/src/SquidMath.h +++ b/src/SquidMath.h @@ -1,11 +1,14 @@ #ifndef _SQUID_SRC_SQUIDMATH_H #define _SQUID_SRC_SQUIDMATH_H +#include "config.h" + /* Math functions we define locally for Squid */ namespace Math { extern int intPercent(const int a, const int b); +extern int64_t int64Percent(const int64_t a, const int64_t b); extern double doublePercent(const double, const double); extern int intAverage(const int, const int, int, const int); extern double doubleAverage(const double, const double, int, const int); diff --git a/src/Store.h b/src/Store.h index 3fbb6c7855..6b360a4b7c 100644 --- a/src/Store.h +++ b/src/Store.h @@ -273,10 +273,10 @@ public: * The maximum size the store will support in normal use. Inaccuracy is permitted, * but may throw estimates for memory etc out of whack. */ - virtual size_t maxSize() const = 0; + virtual uint64_t maxSize() const = 0; /** The minimum size the store will shrink to via normal housekeeping */ - virtual size_t minSize() const = 0; + virtual uint64_t minSize() const = 0; /** * Output stats to the provided store entry. diff --git a/src/StoreHashIndex.h b/src/StoreHashIndex.h index bc602a9f14..1b76377eb7 100644 --- a/src/StoreHashIndex.h +++ b/src/StoreHashIndex.h @@ -63,9 +63,9 @@ public: virtual void sync(); - virtual size_t maxSize() const; + virtual uint64_t maxSize() const; - virtual size_t minSize() const; + virtual uint64_t minSize() const; virtual void stat(StoreEntry&) const; diff --git a/src/SwapDir.cc b/src/SwapDir.cc index 581e4ba74d..ff090b6ff5 100644 --- a/src/SwapDir.cc +++ b/src/SwapDir.cc @@ -81,11 +81,10 @@ SwapDir::statfs(StoreEntry &)const {} void SwapDir::maintain() {} -size_t +uint64_t SwapDir::minSize() const { - return (size_t) (((float) maxSize() * - (float) Config.Swap.lowWaterMark) / 100.0); + return ((maxSize() * Config.Swap.lowWaterMark) / 100); } void diff --git a/src/SwapDir.h b/src/SwapDir.h index e4a6205d7b..46020d20f9 100644 --- a/src/SwapDir.h +++ b/src/SwapDir.h @@ -54,19 +54,17 @@ public: virtual int callback(); virtual void create(); - virtual StoreEntry * get - (const cache_key *); + virtual StoreEntry * get(const cache_key *); - virtual void get - (String const, STOREGETCLIENT, void * cbdata); + virtual void get(String const, STOREGETCLIENT, void * cbdata); virtual void init(); virtual void maintain(); /* perform regular maintenance should be private and self registered ... */ - virtual size_t maxSize() const; + virtual uint64_t maxSize() const; - virtual size_t minSize() const; + virtual uint64_t minSize() const; virtual void stat(StoreEntry &) const; @@ -115,7 +113,7 @@ class SwapDir : public Store { public: - SwapDir(char const *aType) : theType (aType), cur_size (0), max_size(0), max_objsize (-1), cleanLog(NULL) { + SwapDir(char const *aType) : theType (aType), cur_size(0), max_size(0), max_objsize (-1), cleanLog(NULL) { fs.blksize = 1024; path = NULL; } @@ -127,15 +125,13 @@ public: /* official Store interface functions */ virtual void diskFull(); - virtual StoreEntry * get - (const cache_key *); + virtual StoreEntry * get(const cache_key *); - virtual void get - (String const, STOREGETCLIENT, void * cbdata); + virtual void get(String const, STOREGETCLIENT, void * cbdata); - virtual size_t maxSize() const { return max_size;} + virtual uint64_t maxSize() const { return max_size;} - virtual size_t minSize() const; + virtual uint64_t minSize() const; virtual void stat (StoreEntry &anEntry) const; virtual StoreSearch *search(String const url, HttpRequest *) = 0; @@ -157,8 +153,8 @@ private: char const *theType; public: - int cur_size; - int max_size; + uint64_t cur_size; + uint64_t max_size; char *path; int index; /* This entry's index into the swapDirs array */ int64_t max_objsize; diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc index b5c80c9dfc..6a4641d5c5 100644 --- a/src/fs/coss/store_dir_coss.cc +++ b/src/fs/coss/store_dir_coss.cc @@ -996,10 +996,10 @@ void CossSwapDir::statfs(StoreEntry & sentry) const { storeAppendPrintf(&sentry, "\n"); - storeAppendPrintf(&sentry, "Maximum Size: %d KB\n", max_size); - storeAppendPrintf(&sentry, "Current Size: %d KB\n", cur_size); + storeAppendPrintf(&sentry, "Maximum Size: %"PRIu64" KB\n", max_size); + storeAppendPrintf(&sentry, "Current Size: %"PRIu64" KB\n", cur_size); storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", - 100.0 * cur_size / max_size); + (100.0 * (double)cur_size / (double)max_size) ); storeAppendPrintf(&sentry, "Number of object collisions: %d\n", (int) numcollisions); #if 0 /* is this applicable? I Hope not .. */ @@ -1095,7 +1095,7 @@ CossSwapDir::reconfigure(int index, char *path) void CossSwapDir::dump(StoreEntry &entry)const { - storeAppendPrintf(&entry, " %d", max_size >> 10); + storeAppendPrintf(&entry, " %"PRIu64"", (max_size >> 10)); dumpOptions(&entry); } diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/store_dir_ufs.cc index be84eb12b9..99019ca477 100644 --- a/src/fs/ufs/store_dir_ufs.cc +++ b/src/fs/ufs/store_dir_ufs.cc @@ -71,15 +71,12 @@ UFSSwapDir::canStore(StoreEntry const &e)const void UFSSwapDir::parseSizeL1L2() { - int i; - int size; - - i = GetInteger(); - size = i << 10; /* Mbytes to kbytes */ - - if (size <= 0) + int i = GetInteger(); + if (i <= 0) fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); + size_t size = i << 10; /* Mbytes to kbytes */ + /* just reconfigure it */ if (reconfiguring) { if (size == max_size) @@ -314,10 +311,10 @@ UFSSwapDir::statfs(StoreEntry & sentry) const int x; storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); - storeAppendPrintf(&sentry, "Maximum Size: %d KB\n", max_size); - storeAppendPrintf(&sentry, "Current Size: %d KB\n", cur_size); + storeAppendPrintf(&sentry, "Maximum Size: %"PRIu64" KB\n", max_size); + storeAppendPrintf(&sentry, "Current Size: %"PRIu64" KB\n", cur_size); storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", - 100.0 * cur_size / max_size); + (double)(100.0 * cur_size) / (double)max_size); storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", map->n_files_in_map, map->max_n_files, Math::intPercent(map->n_files_in_map, map->max_n_files)); @@ -380,7 +377,7 @@ UFSSwapDir::maintain() walker = repl->PurgeInit(repl, max_scan); while (1) { - if (cur_size < (int) minSize()) /* cur_size should be unsigned */ + if (cur_size < minSize()) break; if (removed >= max_remove) @@ -1325,8 +1322,8 @@ UFSSwapDir::replacementRemove(StoreEntry * e) void UFSSwapDir::dump(StoreEntry & entry) const { - storeAppendPrintf(&entry, " %d %d %d", - max_size >> 10, + storeAppendPrintf(&entry, " %"PRIu64" %d %d", + (max_size >> 10), l1, l2); dumpOptions(&entry); diff --git a/src/store_dir.cc b/src/store_dir.cc index a99590fd32..170ba76f3e 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -343,8 +343,8 @@ StoreController::updateSize(int64_t size, int sign) void SwapDir::updateSize(int64_t size, int sign) { - int blks = (size + fs.blksize - 1) / fs.blksize; - int k = (blks * fs.blksize >> 10) * sign; + int64_t blks = (size + fs.blksize - 1) / fs.blksize; + int64_t k = ((blks * fs.blksize) >> 10) * sign; cur_size += k; store_swap_size += k; @@ -360,13 +360,13 @@ StoreController::stat(StoreEntry &output) const storeAppendPrintf(&output, "Store Directory Statistics:\n"); storeAppendPrintf(&output, "Store Entries : %lu\n", (unsigned long int)StoreEntry::inUseCount()); - storeAppendPrintf(&output, "Maximum Swap Size : %8ld KB\n", - (long int) maxSize()); + storeAppendPrintf(&output, "Maximum Swap Size : %"PRIu64" KB\n", + maxSize()); storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n", store_swap_size); - storeAppendPrintf(&output, "Current Capacity : %d%% used, %d%% free\n", - Math::intPercent((int) store_swap_size, (int) maxSize()), - Math::intPercent((int) (maxSize() - store_swap_size), (int) maxSize())); + storeAppendPrintf(&output, "Current Capacity : %"PRId64"%% used, %"PRId64"%% free\n", + Math::int64Percent(store_swap_size, maxSize()), + Math::int64Percent((maxSize() - store_swap_size), maxSize())); /* FIXME Here we should output memory statistics */ /* now the swapDir */ @@ -374,14 +374,14 @@ StoreController::stat(StoreEntry &output) const } /* if needed, this could be taught to cache the result */ -size_t +uint64_t StoreController::maxSize() const { /* TODO: include memory cache ? */ return swapDir->maxSize(); } -size_t +uint64_t StoreController::minSize() const { /* TODO: include memory cache ? */ @@ -827,22 +827,21 @@ StoreHashIndex::init() } } -size_t +uint64_t StoreHashIndex::maxSize() const { - int i; - size_t result = 0; + uint64_t result = 0; - for (i = 0; i < Config.cacheSwap.n_configured; i++) + for (int i = 0; i < Config.cacheSwap.n_configured; i++) result += store(i)->maxSize(); return result; } -size_t +uint64_t StoreHashIndex::minSize() const { - size_t result = 0; + uint64_t result = 0; for (int i = 0; i < Config.cacheSwap.n_configured; i++) result += store(i)->minSize(); diff --git a/src/tests/TestSwapDir.cc b/src/tests/TestSwapDir.cc index c24fb83ad4..0ca06d32e2 100644 --- a/src/tests/TestSwapDir.cc +++ b/src/tests/TestSwapDir.cc @@ -3,7 +3,7 @@ #include "squid.h" #include "TestSwapDir.h" -size_t +uint64_t TestSwapDir::maxSize() const { return 3; diff --git a/src/tests/TestSwapDir.h b/src/tests/TestSwapDir.h index d93fcd6dbc..d58c4ace64 100644 --- a/src/tests/TestSwapDir.h +++ b/src/tests/TestSwapDir.h @@ -12,7 +12,7 @@ public: bool statsCalled; - virtual size_t maxSize() const; + virtual uint64_t maxSize() const; virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */ virtual void reconfigure(int, char*); diff --git a/src/tests/testStore.cc b/src/tests/testStore.cc index 6be5f03067..b80dc7e33f 100644 --- a/src/tests/testStore.cc +++ b/src/tests/testStore.cc @@ -31,13 +31,13 @@ void TestStore::init() {} -size_t +uint64_t TestStore::maxSize() const { return 3; } -size_t +uint64_t TestStore::minSize() const { return 1; diff --git a/src/tests/testStore.h b/src/tests/testStore.h index 804a92f44d..fbdf7e0b41 100644 --- a/src/tests/testStore.h +++ b/src/tests/testStore.h @@ -55,9 +55,9 @@ public: virtual void maintain() {}; - virtual size_t maxSize() const; + virtual uint64_t maxSize() const; - virtual size_t minSize() const; + virtual uint64_t minSize() const; virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */