dynamic filemap patch from 2.3 branch

plus print "so far" debug lines more often

diff --git a/src/filemap.cc b/src/filemap.cc
index dd3e2809c8c715658c5af5673513ca3207746537..0392d35cf6911c84d90c3f5db55ef2f1d8bd82e3 100644 (file)
--- a/src/filemap.cc
+++ b/src/filemap.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: filemap.cc,v 1.31 1998/07/22 20:37:19 wessels Exp $
+ * $Id: filemap.cc,v 1.32 1999/12/01 04:28:07 wessels Exp $
  *
  * DEBUG: section 8     Swap File Bitmap
  * AUTHOR: Harvest Derived
@@ -53,35 +53,47 @@
 #define ALL_ONES (unsigned long) 0xFFFFFFFF
 #endif
 
+#define FM_INITIAL_NUMBER (1<<14)
+
 fileMap *
-file_map_create(int n)
+file_map_create(void)
 {
     fileMap *fm = xcalloc(1, sizeof(fileMap));
-    fm->max_n_files = n;
-    fm->nwords = n >> LONG_BIT_SHIFT;
-    debug(8, 3) ("file_map_create: creating space for %d files\n", n);
+    fm->max_n_files = FM_INITIAL_NUMBER;
+    fm->nwords = fm->max_n_files >> LONG_BIT_SHIFT;
+    debug(8, 3) ("file_map_create: creating space for %d files\n", fm->max_n_files);
     debug(8, 5) ("--> %d words of %d bytes each\n",
-       fm->nwords, sizeof(unsigned long));
-    fm->file_map = xcalloc(fm->nwords, sizeof(unsigned long));
+       fm->nwords, sizeof(*fm->file_map));
+    fm->file_map = xcalloc(fm->nwords, sizeof(*fm->file_map));
     /* XXX account fm->file_map */
     return fm;
 }
 
+static void
+file_map_grow(fileMap * fm)
+{
+    int old_sz = fm->nwords * sizeof(*fm->file_map);
+    void *old_map = fm->file_map;
+    fm->max_n_files <<= 1;
+    assert(fm->max_n_files <= (1 << 30));
+    fm->nwords = fm->max_n_files >> LONG_BIT_SHIFT;
+    debug(8, 3) ("file_map_grow: creating space for %d files\n", fm->max_n_files);
+    fm->file_map = xcalloc(fm->nwords, sizeof(*fm->file_map));
+    debug(8, 3) ("copying %d old bytes\n", old_sz);
+    xmemcpy(fm->file_map, old_map, old_sz);
+    xfree(old_map);
+    /* XXX account fm->file_map */
+}
+
 int
 file_map_bit_set(fileMap * fm, int file_number)
 {
     unsigned long bitmask = (1L << (file_number & LONG_BIT_MASK));
+    while (file_number >= fm->max_n_files)
+       file_map_grow(fm);
     fm->file_map[file_number >> LONG_BIT_SHIFT] |= bitmask;
     fm->n_files_in_map++;
-    if (!fm->toggle && (fm->n_files_in_map > ((fm->max_n_files * 7) >> 3))) {
-       fm->toggle++;
-       debug(8, 0) ("WARNING: filemap utilization at %d%%\n"
-           "\tConsider decreasing store_avg_object_size in squid.conf\n",
-           percent(fm->n_files_in_map, fm->max_n_files));
-    } else if (fm->n_files_in_map == fm->max_n_files) {
-       fatal_dump("You've run out of swap file numbers.");
-    }
-    return (file_number);
+    return file_number;
 }
 
 void
@@ -96,6 +108,8 @@ int
 file_map_bit_test(fileMap * fm, int file_number)
 {
     unsigned long bitmask = (1L << (file_number & LONG_BIT_MASK));
+    if (file_number >= fm->max_n_files)
+       return 0;
     /* be sure the return value is an int, not a u_long */
     return (fm->file_map[file_number >> LONG_BIT_SHIFT] & bitmask ? 1 : 0);
 }
@@ -123,8 +137,9 @@ file_map_allocate(fileMap * fm, int suggestion)
            return file_map_bit_set(fm, suggestion);
        }
     }
-    fatal("file_map_allocate: Exceeded filemap limit");
-    return 0;                  /* NOTREACHED */
+    debug(8, 3) ("growing from file_map_allocate\n");
+    file_map_grow(fm);
+    return file_map_allocate(fm, fm->max_n_files >> 1);
 }
 
 void
@@ -134,15 +149,6 @@ filemapFreeMemory(fileMap * fm)
     safe_free(fm);
 }
 
-void
-filemapCopy(fileMap * old, fileMap * new)
-{
-    assert(old->max_n_files <= new->max_n_files);
-    assert(0 == new->n_files_in_map);
-    xmemcpy(new->file_map, old->file_map, old->nwords * sizeof(unsigned long));
-    new->n_files_in_map = old->n_files_in_map;
-}
-
 #ifdef TEST
 
 #define TEST_SIZE 1<<16

diff --git a/src/protos.h b/src/protos.h
index 9106de49679dacb23f10c1c554374578d0ae21a6..4a970668c285c004d56d6557e61fb20657e38a40 100644 (file)
--- a/src/protos.h
+++ b/src/protos.h
@@ -1,6 +1,6 @@
 
 /*
- * $Id: protos.h,v 1.347 1999/10/04 05:05:22 wessels Exp $
+ * $Id: protos.h,v 1.348 1999/12/01 04:28:08 wessels Exp $
  *
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -250,13 +250,12 @@ extern void fdDumpOpen(void);
 extern int fdNFree(void);
 extern void fdAdjustReserved(void);
 
-extern fileMap *file_map_create(int);
+extern fileMap *file_map_create(void);
 extern int file_map_allocate(fileMap *, int);
 extern int file_map_bit_set(fileMap *, int);
 extern int file_map_bit_test(fileMap *, int);
 extern void file_map_bit_reset(fileMap *, int);
 extern void filemapFreeMemory(fileMap *);
-extern void filemapCopy(fileMap * old, fileMap * new);
 
 
 extern void fqdncache_nbgethostbyaddr(struct in_addr, FQDNH *, void *);
@@ -948,7 +947,7 @@ extern int storeDirMapBitTest(int fn);
 extern int storeDirMapBitsInUse(void);
 extern int storeDirNumber(int fileno);
 extern int storeDirProperFileno(int dirn, int fn);
-extern int storeDirValidFileno(int fn);
+extern int storeDirValidFileno(int fn, int);
 extern int storeDirWriteCleanLogs(int reopen);
 extern int storeVerifySwapDirs(void);
 extern void storeCreateSwapDirectories(void);

diff --git a/src/store_dir.cc b/src/store_dir.cc
index c644c67b792b8f8942317fbcc42d21bc25c463ee..dffe34da7547a45c50864174137b446a091e34d7 100644 (file)
--- a/src/store_dir.cc
+++ b/src/store_dir.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: store_dir.cc,v 1.98 1999/06/24 20:20:14 wessels Exp $
+ * $Id: store_dir.cc,v 1.99 1999/12/01 04:28:08 wessels Exp $
  *
  * DEBUG: section 47    Store Directory Routines
  * AUTHOR: Duane Wessels
@@ -167,7 +167,7 @@ storeDirSelectSwapDir(void)
 }
 
 int
-storeDirValidFileno(int fn)
+storeDirValidFileno(int fn, int flag)
 {
     int dirn = fn >> SWAP_DIR_SHIFT;
     int filn = fn & SWAP_FILE_MASK;
@@ -177,8 +177,13 @@ storeDirValidFileno(int fn)
        return 0;
     if (filn < 0)
        return 0;
-    if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
-       return 0;
+    /*
+     * If flag is set it means out-of-range file number should
+     * be considered invalid.
+     */
+    if (flag)
+       if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files)
+           return 0;
     return 1;
 }
 
@@ -309,25 +314,13 @@ void
 storeDirConfigure(void)
 {
     SwapDir *SD;
-    int n;
     int i;
-    fileMap *fm;
     Config.Swap.maxSize = 0;
     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
        SD = &Config.cacheSwap.swapDirs[i];;
        Config.Swap.maxSize += SD->max_size;
-       n = 2 * SD->max_size / Config.Store.avgObjectSize;
-       if (NULL == SD->map) {
-           /* first time */
-           SD->map = file_map_create(n);
-       } else if (n > SD->map->max_n_files) {
-           /* it grew, need to expand */
-           fm = file_map_create(n);
-           filemapCopy(SD->map, fm);
-           filemapFreeMemory(SD->map);
-           SD->map = fm;
-       }
-       /* else it shrunk, and we leave the old one in place */
+       if (NULL == SD->map)
+           SD->map = file_map_create();
     }
 }