Bug 3068: cache_dir capacity and usage overflows

Makes usage calculations use size_t instead of int and updates the
relevant fields storing the cache_dir capacity and usage fields as well.

This fixes Squid filling cache_dir with files >2GB in size.
Also allows Squid to store more than 2TB of data total in one dir.

diff --git a/src/SquidMath.cc b/src/SquidMath.cc
index 946d3f15c51ff9c8607a0912098d1436254249f1..b4160ba6d1f452d369d7c4488300bbcc2fef27d3 100644 (file)
--- a/src/SquidMath.cc
+++ b/src/SquidMath.cc
@@ -7,6 +7,12 @@ Math::intPercent(const int a, const int b)
     return b ? ((int) (100.0 * a / b + 0.5)) : 0;
 }
 
+int64_t
+Math::int64Percent(const int64_t a, const int64_t b)
+{
+    return b ? ((int64_t) (100.0 * a / b + 0.5)) : 0;
+}
+
 double
 Math::doublePercent(const double a, const double b)
 {

diff --git a/src/SquidMath.h b/src/SquidMath.h
index a8fc22abb9ade1fd9f5db5e9accc693b3ae9a96a..afb033825b34bda1ad9c0cbd66e5bf58b6be4f2d 100644 (file)
--- a/src/SquidMath.h
+++ b/src/SquidMath.h
@@ -1,11 +1,14 @@
 #ifndef _SQUID_SRC_SQUIDMATH_H
 #define _SQUID_SRC_SQUIDMATH_H
 
+#include "config.h"
+
 /* Math functions we define locally for Squid */
 namespace Math
 {
 
 extern int intPercent(const int a, const int b);
+extern int64_t int64Percent(const int64_t a, const int64_t b);
 extern double doublePercent(const double, const double);
 extern int intAverage(const int, const int, int, const int);
 extern double doubleAverage(const double, const double, int, const int);

diff --git a/src/Store.h b/src/Store.h
index 3fbb6c7855e5a00a210686fd446d38c7a5e05e84..6b360a4b7c46c1b2355cc224ef6d8d4f363ffe2f 100644 (file)
--- a/src/Store.h
+++ b/src/Store.h
@@ -273,10 +273,10 @@ public:
      * The maximum size the store will support in normal use. Inaccuracy is permitted,
      * but may throw estimates for memory etc out of whack.
      */
-    virtual size_t maxSize() const = 0;
+    virtual uint64_t maxSize() const = 0;
 
     /** The minimum size the store will shrink to via normal housekeeping */
-    virtual size_t minSize() const = 0;
+    virtual uint64_t minSize() const = 0;
 
     /**
      * Output stats to the provided store entry.

diff --git a/src/StoreHashIndex.h b/src/StoreHashIndex.h
index bc602a9f142532db859267fe36291381f2e90117..1b76377eb789cc481cf9a94247b16e27c57da806 100644 (file)
--- a/src/StoreHashIndex.h
+++ b/src/StoreHashIndex.h
@@ -63,9 +63,9 @@ public:
 
     virtual void sync();
 
-    virtual size_t maxSize() const;
+    virtual uint64_t maxSize() const;
 
-    virtual size_t minSize() const;
+    virtual uint64_t minSize() const;
 
     virtual void stat(StoreEntry&) const;
 

diff --git a/src/SwapDir.cc b/src/SwapDir.cc
index 581e4ba74dd5d7a167ac10cff7c557bf3e8ee943..ff090b6ff58d50e45e9451ea24e04b7bbd1229a7 100644 (file)
--- a/src/SwapDir.cc
+++ b/src/SwapDir.cc
@@ -81,11 +81,10 @@ SwapDir::statfs(StoreEntry &)const {}
 void
 SwapDir::maintain() {}
 
-size_t
+uint64_t
 SwapDir::minSize() const
 {
-    return (size_t) (((float) maxSize() *
-                      (float) Config.Swap.lowWaterMark) / 100.0);
+    return ((maxSize() * Config.Swap.lowWaterMark) / 100);
 }
 
 void

diff --git a/src/SwapDir.h b/src/SwapDir.h
index e4a6205d7b57041ed79608e01ba3baa01c4bf9da..46020d20f9e72c39cd92af10ffd613fb2dace249 100644 (file)
--- a/src/SwapDir.h
+++ b/src/SwapDir.h
@@ -54,19 +54,17 @@ public:
     virtual int callback();
     virtual void create();
 
-    virtual StoreEntry * get
-    (const cache_key *);
+    virtual StoreEntry * get(const cache_key *);
 
-    virtual void get
-    (String const, STOREGETCLIENT, void * cbdata);
+    virtual void get(String const, STOREGETCLIENT, void * cbdata);
 
     virtual void init();
 
     virtual void maintain(); /* perform regular maintenance should be private and self registered ... */
 
-    virtual size_t maxSize() const;
+    virtual uint64_t maxSize() const;
 
-    virtual size_t minSize() const;
+    virtual uint64_t minSize() const;
 
     virtual void stat(StoreEntry &) const;
 
@@ -115,7 +113,7 @@ class SwapDir : public Store
 {
 
 public:
-    SwapDir(char const *aType) : theType (aType), cur_size (0), max_size(0), max_objsize (-1), cleanLog(NULL) {
+    SwapDir(char const *aType) : theType (aType), cur_size(0), max_size(0), max_objsize (-1), cleanLog(NULL) {
         fs.blksize = 1024;
         path = NULL;
     }
@@ -127,15 +125,13 @@ public:
     /* official Store interface functions */
     virtual void diskFull();
 
-    virtual StoreEntry * get
-    (const cache_key *);
+    virtual StoreEntry * get(const cache_key *);
 
-    virtual void get
-    (String const, STOREGETCLIENT, void * cbdata);
+    virtual void get(String const, STOREGETCLIENT, void * cbdata);
 
-    virtual size_t maxSize() const { return max_size;}
+    virtual uint64_t maxSize() const { return max_size;}
 
-    virtual size_t minSize() const;
+    virtual uint64_t minSize() const;
     virtual void stat (StoreEntry &anEntry) const;
     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 
@@ -157,8 +153,8 @@ private:
     char const *theType;
 
 public:
-    int cur_size;
-    int max_size;
+    uint64_t cur_size;
+    uint64_t max_size;
     char *path;
     int index;                 /* This entry's index into the swapDirs array */
     int64_t max_objsize;

diff --git a/src/fs/coss/store_dir_coss.cc b/src/fs/coss/store_dir_coss.cc
index b5c80c9dfc7c8bb5369b80959dbb9f2f4b66d76a..6a4641d5c5370507ed92306adb48fa0840ff0984 100644 (file)
--- a/src/fs/coss/store_dir_coss.cc
+++ b/src/fs/coss/store_dir_coss.cc
@@ -996,10 +996,10 @@ void
 CossSwapDir::statfs(StoreEntry & sentry) const
 {
     storeAppendPrintf(&sentry, "\n");
-    storeAppendPrintf(&sentry, "Maximum Size: %d KB\n", max_size);
-    storeAppendPrintf(&sentry, "Current Size: %d KB\n", cur_size);
+    storeAppendPrintf(&sentry, "Maximum Size: %"PRIu64" KB\n", max_size);
+    storeAppendPrintf(&sentry, "Current Size: %"PRIu64" KB\n", cur_size);
     storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
-                      100.0 * cur_size / max_size);
+                      (100.0 * (double)cur_size / (double)max_size) );
     storeAppendPrintf(&sentry, "Number of object collisions: %d\n", (int) numcollisions);
 #if 0
     /* is this applicable? I Hope not .. */
@@ -1095,7 +1095,7 @@ CossSwapDir::reconfigure(int index, char *path)
 void
 CossSwapDir::dump(StoreEntry &entry)const
 {
-    storeAppendPrintf(&entry, " %d", max_size >> 10);
+    storeAppendPrintf(&entry, " %"PRIu64"", (max_size >> 10));
     dumpOptions(&entry);
 }
 

diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/store_dir_ufs.cc
index be84eb12b9b4888b43eb04e4d56098f32bcb0978..99019ca477403490caa24fb254935487bb5d8ed8 100644 (file)
--- a/src/fs/ufs/store_dir_ufs.cc
+++ b/src/fs/ufs/store_dir_ufs.cc
@@ -71,15 +71,12 @@ UFSSwapDir::canStore(StoreEntry const &e)const
 void
 UFSSwapDir::parseSizeL1L2()
 {
-    int i;
-    int size;
-
-    i = GetInteger();
-    size = i << 10;            /* Mbytes to kbytes */
-
-    if (size <= 0)
+    int i = GetInteger();
+    if (i <= 0)
         fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
 
+    size_t size = i << 10;             /* Mbytes to kbytes */
+
     /* just reconfigure it */
     if (reconfiguring) {
         if (size == max_size)
@@ -314,10 +311,10 @@ UFSSwapDir::statfs(StoreEntry & sentry) const
     int x;
     storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
     storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
-    storeAppendPrintf(&sentry, "Maximum Size: %d KB\n", max_size);
-    storeAppendPrintf(&sentry, "Current Size: %d KB\n", cur_size);
+    storeAppendPrintf(&sentry, "Maximum Size: %"PRIu64" KB\n", max_size);
+    storeAppendPrintf(&sentry, "Current Size: %"PRIu64" KB\n", cur_size);
     storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
-                      100.0 * cur_size / max_size);
+                      (double)(100.0 * cur_size) / (double)max_size);
     storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
                       map->n_files_in_map, map->max_n_files,
                       Math::intPercent(map->n_files_in_map, map->max_n_files));
@@ -380,7 +377,7 @@ UFSSwapDir::maintain()
     walker = repl->PurgeInit(repl, max_scan);
 
     while (1) {
-        if (cur_size < (int) minSize()) /* cur_size should be unsigned */
+        if (cur_size < minSize())
             break;
 
         if (removed >= max_remove)
@@ -1325,8 +1322,8 @@ UFSSwapDir::replacementRemove(StoreEntry * e)
 void
 UFSSwapDir::dump(StoreEntry & entry) const
 {
-    storeAppendPrintf(&entry, " %d %d %d",
-                      max_size >> 10,
+    storeAppendPrintf(&entry, " %"PRIu64" %d %d",
+                      (max_size >> 10),
                       l1,
                       l2);
     dumpOptions(&entry);

diff --git a/src/store_dir.cc b/src/store_dir.cc
index a99590fd3204e04b294c34e86e25bb7be0145ab3..170ba76f3ebb361c614a81ffeee9bd0923134771 100644 (file)
--- a/src/store_dir.cc
+++ b/src/store_dir.cc
@@ -343,8 +343,8 @@ StoreController::updateSize(int64_t size, int sign)
 void
 SwapDir::updateSize(int64_t size, int sign)
 {
-    int blks = (size + fs.blksize - 1) / fs.blksize;
-    int k = (blks * fs.blksize >> 10) * sign;
+    int64_t blks = (size + fs.blksize - 1) / fs.blksize;
+    int64_t k = ((blks * fs.blksize) >> 10) * sign;
     cur_size += k;
     store_swap_size += k;
 
@@ -360,13 +360,13 @@ StoreController::stat(StoreEntry &output) const
     storeAppendPrintf(&output, "Store Directory Statistics:\n");
     storeAppendPrintf(&output, "Store Entries          : %lu\n",
                       (unsigned long int)StoreEntry::inUseCount());
-    storeAppendPrintf(&output, "Maximum Swap Size      : %8ld KB\n",
-                      (long int) maxSize());
+    storeAppendPrintf(&output, "Maximum Swap Size      : %"PRIu64" KB\n",
+                      maxSize());
     storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n",
                       store_swap_size);
-    storeAppendPrintf(&output, "Current Capacity       : %d%% used, %d%% free\n",
-                      Math::intPercent((int) store_swap_size, (int) maxSize()),
-                      Math::intPercent((int) (maxSize() - store_swap_size), (int) maxSize()));
+    storeAppendPrintf(&output, "Current Capacity       : %"PRId64"%% used, %"PRId64"%% free\n",
+                      Math::int64Percent(store_swap_size, maxSize()),
+                      Math::int64Percent((maxSize() - store_swap_size), maxSize()));
     /* FIXME Here we should output memory statistics */
 
     /* now the swapDir */
@@ -374,14 +374,14 @@ StoreController::stat(StoreEntry &output) const
 }
 
 /* if needed, this could be taught to cache the result */
-size_t
+uint64_t
 StoreController::maxSize() const
 {
     /* TODO: include memory cache ? */
     return swapDir->maxSize();
 }
 
-size_t
+uint64_t
 StoreController::minSize() const
 {
     /* TODO: include memory cache ? */
@@ -827,22 +827,21 @@ StoreHashIndex::init()
     }
 }
 
-size_t
+uint64_t
 StoreHashIndex::maxSize() const
 {
-    int i;
-    size_t result = 0;
+    uint64_t result = 0;
 
-    for (i = 0; i < Config.cacheSwap.n_configured; i++)
+    for (int i = 0; i < Config.cacheSwap.n_configured; i++)
         result += store(i)->maxSize();
 
     return result;
 }
 
-size_t
+uint64_t
 StoreHashIndex::minSize() const
 {
-    size_t result = 0;
+    uint64_t result = 0;
 
     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
         result += store(i)->minSize();

diff --git a/src/tests/TestSwapDir.cc b/src/tests/TestSwapDir.cc
index c24fb83ad418f72c6c03b24da64f15f8fb44845d..0ca06d32e22c2a5086b051b40df3ff2def668d4b 100644 (file)
--- a/src/tests/TestSwapDir.cc
+++ b/src/tests/TestSwapDir.cc
@@ -3,7 +3,7 @@
 #include "squid.h"
 #include "TestSwapDir.h"
 
-size_t
+uint64_t
 TestSwapDir::maxSize() const
 {
     return 3;

diff --git a/src/tests/TestSwapDir.h b/src/tests/TestSwapDir.h
index d93fcd6dbcaf529eb138bebdef4b5780ad2bb374..d58c4ace643e9fb1d643586ea43d2fff3b57767c 100644 (file)
--- a/src/tests/TestSwapDir.h
+++ b/src/tests/TestSwapDir.h
@@ -12,7 +12,7 @@ public:
 
     bool statsCalled;
 
-    virtual size_t maxSize() const;
+    virtual uint64_t maxSize() const;
     virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */
 
     virtual void reconfigure(int, char*);

diff --git a/src/tests/testStore.cc b/src/tests/testStore.cc
index 6be5f03067704c5401e52f0f95f1fcbe7a1d0bb7..b80dc7e33ff09d0aabf387c94eb8dc8789b1f861 100644 (file)
--- a/src/tests/testStore.cc
+++ b/src/tests/testStore.cc
@@ -31,13 +31,13 @@ void
 TestStore::init()
 {}
 
-size_t
+uint64_t
 TestStore::maxSize() const
 {
     return 3;
 }
 
-size_t
+uint64_t
 TestStore::minSize() const
 {
     return 1;

diff --git a/src/tests/testStore.h b/src/tests/testStore.h
index 804a92f44dc82c30f2e926a4e7ae2deee6797721..fbdf7e0b4106b0e7742c2c61776b55c91ea7f4db 100644 (file)
--- a/src/tests/testStore.h
+++ b/src/tests/testStore.h
@@ -55,9 +55,9 @@ public:
 
     virtual void maintain() {};
 
-    virtual size_t maxSize() const;
+    virtual uint64_t maxSize() const;
 
-    virtual size_t minSize() const;
+    virtual uint64_t minSize() const;
 
     virtual void stat(StoreEntry &) const; /* output stats to the provided store entry */