Big improvement to cache_mem. Now count pool pages, not bytes for low/high water

mark.  Share pool among all in memory objects.

diff --git a/src/cache_cf.cc b/src/cache_cf.cc
index 5545c16c14a5dd4e3ffc9b225ef254a8bbb74bb8..ecdbb4d1bfa95b7a673d1a47f78580a4fd5546e6 100644 (file)
--- a/src/cache_cf.cc
+++ b/src/cache_cf.cc
@@ -1,5 +1,5 @@
 /*
- * $Id: cache_cf.cc,v 1.107 1996/10/10 16:33:37 wessels Exp $
+ * $Id: cache_cf.cc,v 1.108 1996/10/10 18:48:47 wessels Exp $
  *
  * DEBUG: section 3     Configuration File Parsing
  * AUTHOR: Harvest Derived
@@ -107,7 +107,7 @@
 
 struct SquidConfig Config;
 
-#define DefaultMemMaxSize      (16 << 20)      /* 16 MB */
+#define DefaultMemMaxSize      (8 << 20)       /* 8 MB */
 #define DefaultMemHighWaterMark 90     /* 90% */
 #define DefaultMemLowWaterMark  75     /* 75% */
 #define DefaultSwapMaxSize     (100 << 10)     /* 100 MB (100*1024 kbytes) */
@@ -196,7 +196,6 @@ struct SquidConfig Config;
 #define DefaultIpcacheSize     1024
 #define DefaultIpcacheLow      90
 #define DefaultIpcacheHigh     95
-#define DefaultMaxHotvmObjSize (256<<10)       /* 256k */
 #define DefaultMinDirectHops   4
 
 int httpd_accel_mode = 0;      /* for fast access */
@@ -1383,9 +1382,6 @@ parseConfigFile(char *file_name)
        else if (!strcmp(token, "forwarded_for"))
            parseOnOff(&opt_forwarded_for);
 
-       else if (!strcmp(token, "max_hotvm_obj_size"))
-           parseIntegerValue(&Config.maxHotvmObjSize);
-
        else if (!strcmp(token, "minimum_direct_hops"))
            parseIntegerValue(&Config.minDirectHops);
 
@@ -1591,7 +1587,6 @@ configSetFactoryDefaults(void)
     Config.ipcache.size = DefaultIpcacheSize;
     Config.ipcache.low = DefaultIpcacheLow;
     Config.ipcache.high = DefaultIpcacheHigh;
-    Config.maxHotvmObjSize = DefaultMaxHotvmObjSize;
     Config.minDirectHops = DefaultMinDirectHops;
 }
 

diff --git a/src/icmp.cc b/src/icmp.cc
index 66255086d5fab8f36b50e3392c32a8368f621fc5..cff9b3e6e08bf825da8869f1e0a7995649b5d196 100644 (file)
--- a/src/icmp.cc
+++ b/src/icmp.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: icmp.cc,v 1.14 1996/10/09 22:49:34 wessels Exp $
+ * $Id: icmp.cc,v 1.15 1996/10/10 18:48:49 wessels Exp $
  *
  * DEBUG: section 37    ICMP Routines
  * AUTHOR: Duane Wessels
@@ -122,6 +122,7 @@ void
 icmpClose(void)
 {
     icmpQueueData *queue;
+    debug(29,0,"Closing ICMP socket on FD %d\n", icmp_sock);
     comm_close(icmp_sock);
     while ((queue = IcmpQueueHead)) {
        IcmpQueueHead = queue->next;

diff --git a/src/net_db.cc b/src/net_db.cc
index a214ba44b677e5f663c34e5952e6a85d0d92d53a..90cba5cd04fbea396299274e329d0d6893c56700 100644 (file)
--- a/src/net_db.cc
+++ b/src/net_db.cc
@@ -302,7 +302,7 @@ netdbHops(struct in_addr addr)
 void
 netdbFreeMemory(void)
 {
-    netdbEntry *e;
+    netdbEntry *n;
     netdbEntry **L1;
     hash_link *h;
     hash_link **L2;
@@ -311,14 +311,20 @@ netdbFreeMemory(void)
     int j;
 
     L1 = xcalloc(meta_data.netdb_addrs, sizeof(netdbEntry));
-    e = (netdbEntry *) hash_first(addr_table);
-    while (e && i < meta_data.netdb_addrs) {
-       *(L1 + i) = e;
+    n = (netdbEntry *) hash_first(addr_table);
+    while (n && i < meta_data.netdb_addrs) {
+       *(L1 + i) = n;
        i++;
-       e = (netdbEntry *) hash_next(addr_table);
+       n = (netdbEntry *) hash_next(addr_table);
+    }
+    for (j = 0; j < i; j++) {
+       n = *(L1 + j);
+        while ((x = n->hosts)) {
+           n->hosts = x->next;
+           safe_free(x);
+        }
+       xfree(n);
     }
-    for (j = 0; j < i; j++)
-       xfree(*(L1 + j));
     xfree(L1);
 
     i = 0;
@@ -331,9 +337,7 @@ netdbFreeMemory(void)
     }
     for (j = 0; j < i; j++) {
        h = *(L2 + j);
-       x = (struct _net_db_name *) h->item;
-       xfree(x->name);
-       xfree(x);
+       xfree(h->key);
        xfree(h);
     }
     xfree(L2);

diff --git a/src/stat.cc b/src/stat.cc
index 434ff0b8374e32984d42b238cf220eaff8651575..fe69b03a8f05f8f16f553afce2ab760062e672f0 100644 (file)
--- a/src/stat.cc
+++ b/src/stat.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: stat.cc,v 1.81 1996/10/10 16:38:50 wessels Exp $
+ * $Id: stat.cc,v 1.82 1996/10/10 18:48:51 wessels Exp $
  *
  * DEBUG: section 18    Cache Manager Statistics
  * AUTHOR: Harvest Derived
@@ -705,7 +705,7 @@ info_get(cacheinfo * obj, StoreEntry * sentry)
     storeAppendPrintf(sentry, "{\tStorage Swap size:\t%d MB}\n",
        storeGetSwapSize() >> 10);
     storeAppendPrintf(sentry, "{\tStorage Mem size:\t%d KB}\n",
-       storeGetMemSize() >> 10);
+       store_mem_size >> 10);
 
 #if HAVE_GETRUSAGE && defined(RUSAGE_SELF)
     storeAppendPrintf(sentry, "{Resource usage for %s:}\n", appname);

diff --git a/src/stmem.cc b/src/stmem.cc
index baf45d593d58409e80d57b81117dbabf3f1711b6..2e0532f5925bbf0f3efc1bbe03090e7618740ddc 100644 (file)
--- a/src/stmem.cc
+++ b/src/stmem.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: stmem.cc,v 1.29 1996/10/09 15:34:38 wessels Exp $
+ * $Id: stmem.cc,v 1.30 1996/10/10 18:48:52 wessels Exp $
  *
  * DEBUG: section 19    Memory Primitives
  * AUTHOR: Harvest Derived
@@ -412,7 +412,7 @@ stmemInit(void)
     sm_stats.page_size = SM_PAGE_SIZE;
     sm_stats.total_pages_allocated = 0;
     sm_stats.n_pages_in_use = 0;
-    sm_stats.max_pages = (Config.Mem.maxSize / SM_PAGE_SIZE) >> 1;
+    sm_stats.max_pages = Config.Mem.maxSize / SM_PAGE_SIZE;
 
     disk_stats.page_size = DISK_PAGE_SIZE;
     disk_stats.total_pages_allocated = 0;

diff --git a/src/store.cc b/src/store.cc
index 20135635fbd77c649b257bf7a178c107c5d887ff..b20e25fc4ceca368494c09cac00c1c0edd2b37e7 100644 (file)
--- a/src/store.cc
+++ b/src/store.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: store.cc,v 1.126 1996/10/09 15:34:39 wessels Exp $
+ * $Id: store.cc,v 1.127 1996/10/10 18:48:52 wessels Exp $
  *
  * DEBUG: section 20    Storeage Manager
  * AUTHOR: Harvest Derived
@@ -250,9 +250,10 @@ static HashID store_table = 0;
 static HashID in_mem_table = 0;
 
 /* current memory storage size */
-static unsigned long store_mem_size = 0;
-static unsigned long store_mem_high = 0;
-static unsigned long store_mem_low = 0;
+unsigned long store_mem_size = 0;
+
+static int store_pages_high = 0;
+static int store_pages_low = 0;
 
 /* current file name, swap file, use number as a filename */
 static unsigned long swapfileno = 0;
@@ -356,12 +357,9 @@ new_MemObjectData(void)
 static void
 destroy_MemObjectData(MemObject * mem)
 {
-    debug(20, 3, "destroy_MemObjectData: destroying %p\n", mem->data);
+    debug(20, 3, "destroy_MemObjectData: destroying %p, %d bytes\n",
+       mem->data, mem->e_current_len);
     store_mem_size -= mem->e_current_len - mem->e_lowest_offset;
-    debug(20, 8, "destroy_MemObjectData: Freeing %d in-memory bytes\n",
-       mem->e_current_len);
-    debug(20, 8, "destroy_MemObjectData: store_mem_size = %d\n",
-       store_mem_size);
     if (mem->data) {
        mem->data->mem_free(mem->data);
        mem->data = NULL;
@@ -603,6 +601,8 @@ storeUnlockObject(StoreEntry * e)
        e->lock_count--;
     } else if (storeShouldPurgeMem(e)) {
        storePurgeMem(e);
+    } else {
+       requestUnlink(mem->request);
     }
     return 0;
 }
@@ -1654,13 +1654,6 @@ storeGetSwapSize(void)
     return store_swap_size;
 }
 
-/* return current swap size in bytes */
-int
-storeGetMemSize(void)
-{
-    return store_mem_size;
-}
-
 static int
 storeCheckSwapable(StoreEntry * e)
 {
@@ -1870,15 +1863,17 @@ storeGetMemSpace(int size)
     int i;
     static time_t last_warning = 0;
     static time_t last_check = 0;
+    int pages_needed;
 
     if (squid_curtime == last_check)
        return 0;
     last_check = squid_curtime;
-    if ((store_mem_size + size) < store_mem_high)
+    pages_needed = (size / SM_PAGE_SIZE) + 1;
+    if (sm_stats.n_pages_in_use + pages_needed < sm_stats.max_pages)
        return 0;
     if (store_rebuilding == STORE_REBUILDING_FAST)
        return 0;
-    debug(20, 2, "storeGetMemSpace: Starting...\n");
+    debug(20, 2, "storeGetMemSpace: Starting, need %d pages\n", pages_needed);
 
     list = xcalloc(meta_data.mem_obj_count, sizeof(ipcache_entry *));
     for (e = storeGetInMemFirst(); e; e = storeGetInMemNext()) {
@@ -1900,12 +1895,6 @@ storeGetMemSpace(int size)
        }
        debug(20, 3, "storeGetMemSpace: Adding '%s'\n", e->url);
     }
-#ifdef EXTRA_DEBUGGING
-    debug(20, 5, "storeGetMemSpace: Current size:     %7d bytes\n", store_mem_size);
-    debug(20, 5, "storeGetMemSpace: High W Mark:      %7d bytes\n", store_mem_high);
-    debug(20, 5, "storeGetMemSpace: Low W Mark:       %7d bytes\n", store_mem_low);
-    debug(20, 5, "storeGetMemSpace: Entry count:      %7d items\n", meta_data.store_entries);
-#endif
     debug(20, 5, "storeGetMemSpace: Sorting LRU_list: %7d items\n", list_count);
     qsort((char *) list,
        list_count,
@@ -1914,13 +1903,14 @@ storeGetMemSpace(int size)
 
     /* Kick LRU out until we have enough memory space */
     for (i = 0; i < list_count; i++) {
-       if (store_mem_size + size < store_mem_low)
+       if (sm_stats.n_pages_in_use + pages_needed < store_pages_low)
            break;
        e = *(list + i);
        if (storeCheckPurgeMem(e)) {
            storePurgeMem(e);
            n_purged++;
        } else if (!storeEntryLocked(e)) {
+           /* These will be neg-cached objects */
            storeRelease(e);
            n_released++;
        } else {
@@ -1929,11 +1919,12 @@ storeGetMemSpace(int size)
     }
 
     i = 3;
-    if (store_mem_size + size > store_mem_high) {
+    if (sm_stats.n_pages_in_use + pages_needed > store_pages_high) {
        if (squid_curtime - last_warning > 600) {
-           debug(20, 0, "WARNING: Over store_mem high-water mark (%d > %d)\n",
-               store_mem_size + size, store_mem_high);
+           debug(20, 0, "WARNING: Over store_pages high-water mark (%d > %d)\n",
+               sm_stats.n_pages_in_use + pages_needed, store_pages_high);
            last_warning = squid_curtime;
+           debug(20, 0, "Perhaps you should increase cache_mem?\n");
            i = 0;
        }
     }
@@ -2553,6 +2544,8 @@ storeInit(void)
 void
 storeConfigure(void)
 {
+    int store_mem_high = 0;
+    int store_mem_low = 0;
     store_mem_high = (long) (Config.Mem.maxSize / 100) *
        Config.Mem.highWaterMark;
     store_mem_low = (long) (Config.Mem.maxSize / 100) *
@@ -2562,6 +2555,9 @@ storeConfigure(void)
            (float) Config.Swap.highWaterMark) / (float) 100);
     store_swap_low = (long) (((float) Config.Swap.maxSize *
            (float) Config.Swap.lowWaterMark) / (float) 100);
+
+    store_pages_high = store_mem_high / SM_PAGE_SIZE;
+    store_pages_low = store_mem_low / SM_PAGE_SIZE;
 }
 
 /* 
@@ -2810,7 +2806,7 @@ storeShouldPurgeMem(StoreEntry * e)
 {
     if (storeCheckPurgeMem(e) == 0)
        return 0;
-    if (e->object_len > Config.maxHotvmObjSize)
+    if (sm_stats.n_pages_in_use > store_pages_low)
        return 1;
     return 0;
 }