From: wessels <> Date: Sat, 24 Aug 1996 03:25:59 +0000 (+0000) Subject: - Rewrote storeGetMemSpace() X-Git-Tag: SQUID_3_0_PRE1~5932 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b32508fb160c3c83081159da38c9ab9c46659814;p=thirdparty%2Fsquid.git - Rewrote storeGetMemSpace() - Fixed hot_vm accounting. --- diff --git a/src/store.cc b/src/store.cc index 0949af2857..8bf0890357 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,5 +1,5 @@ /* - * $Id: store.cc,v 1.86 1996/08/21 20:09:52 wessels Exp $ + * $Id: store.cc,v 1.87 1996/08/23 21:25:59 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -215,6 +215,8 @@ static char *storeDescribeStatus _PARAMS((StoreEntry *)); static int compareLastRef _PARAMS((StoreEntry ** e1, StoreEntry ** e2)); static int compareSize _PARAMS((StoreEntry ** e1, StoreEntry ** e2)); static int storeClientListSearch _PARAMS((MemObject *, int fd)); +static void storeHashMemInsert _PARAMS((StoreEntry *)); +static void storeHashMemDelete _PARAMS((StoreEntry *)); /* Now, this table is inaccessible to outsider. They have to use a method * to access a value in internal storage data structure. */ @@ -323,7 +325,7 @@ static void destroy_StoreEntry(e) static mem_ptr new_MemObjectData() { debug(20, 3, "new_MemObjectData: calling memInit()\n"); - meta_data.hot_vm++; + /* meta_data.hot_vm++; */ return memInit(); } @@ -339,7 +341,7 @@ static void destroy_MemObjectData(mem) if (mem->data) { mem->data->mem_free(mem->data); mem->data = NULL; - meta_data.hot_vm--; + /* meta_data.hot_vm--; */ } mem->e_current_len = 0; } @@ -359,9 +361,22 @@ HashID storeCreateHashTable(cmp_func) return store_table; } -/* - * if object is in memory, also insert into in_mem_table - */ +static void storeHashMemInsert(e) + StoreEntry *e; +{ + hash_insert(in_mem_table, e->key, e); + meta_data.hot_vm++; +} + +static void storeHashMemDelete(e) + StoreEntry *e; +{ + hash_link *hptr = hash_lookup(in_mem_table, e->key); + if (hptr == NULL) + fatal_dump("storeHashMemDelete: key not found"); + hash_delete_link(in_mem_table, hptr); + meta_data.hot_vm--; +} static int storeHashInsert(e) StoreEntry *e; @@ -369,45 +384,32 @@ static int storeHashInsert(e) debug(20, 3, "storeHashInsert: Inserting Entry %p key '%s'\n", e, e->key); if (e->mem_status == IN_MEMORY) - hash_insert(in_mem_table, e->key, e); - return (hash_join(store_table, (hash_link *) e)); + storeHashMemInsert(e); + return hash_join(store_table, (hash_link *) e); } -/* - * if object in memory, also remove from in_mem_table - */ - static int storeHashDelete(e) StoreEntry *e; { - hash_link *hptr = NULL; - if (e->mem_status == IN_MEMORY && e->key) { - if ((hptr = hash_lookup(in_mem_table, e->key))) - hash_delete_link(in_mem_table, hptr); - } - return (hash_remove_link(store_table, (hash_link *) e)); + if (e->mem_status == IN_MEMORY) + storeHashMemDelete(e); + return hash_remove_link(store_table, (hash_link *) e); } /* * maintain the in-mem hash table according to the changes of mem_status * This routine replaces the instruction "e->store_status = status;" */ - void storeSetMemStatus(e, status) StoreEntry *e; mem_status_t status; { - hash_link *ptr = NULL; - if (e->key == NULL) fatal_dump("storeSetMemStatus: NULL key"); - - if (status != IN_MEMORY && e->mem_status == IN_MEMORY) { - if ((ptr = hash_lookup(in_mem_table, e->key))) - hash_delete_link(in_mem_table, ptr); - } else if (status == IN_MEMORY && e->mem_status != IN_MEMORY) { - hash_insert(in_mem_table, e->key, e); - } + if (status != IN_MEMORY && e->mem_status == IN_MEMORY) + storeHashMemDelete(e); + else if (status == IN_MEMORY && e->mem_status != IN_MEMORY) + storeHashMemInsert(e); e->mem_status = status; } @@ -742,8 +744,6 @@ StoreEntry *storeCreateEntry(url, req_hdr, flags, method) MemObject *mem = NULL; debug(20, 3, "storeCreateEntry: '%s' icp flags=%x\n", url, flags); - if (meta_data.hot_vm > store_hotobj_high) - storeGetMemSpace(0, 1); e = new_StoreEntry(WITH_MEMOBJ); e->lock_count = 1; /* Note lock here w/o calling storeLock() */ mem = e->mem_obj; @@ -1714,6 +1714,8 @@ int storeAbort(e, msg) LOCAL_ARRAY(char, abort_msg, 2000); MemObject *mem = e->mem_obj; + if (e->store_status != STORE_PENDING) /* XXX remove later */ + fatal_dump("storeAbort: bad store_status"); if (mem == NULL) /* XXX remove later */ fatal_dump("storeAbort: null mem"); @@ -1883,168 +1885,78 @@ int storeGetMemSpace(size, check_vm_number) int size; int check_vm_number; { - static int over_highwater = 0; - static int over_max = 0; - StoreEntry *LRU = NULL, *e = NULL; - dynamic_array *LRU_list = NULL; - dynamic_array *pending_entry_list = NULL; - int entry_to_delete_behind = 0; - int n_deleted_behind = 0; - int n_scanned = 0; + StoreEntry *e = NULL; + StoreEntry **list = NULL; + int list_count = 0; int n_expired = 0; - int n_purged = 0; int n_released = 0; int i; - int n_inmem = 0; /* extra debugging */ - int n_locked = 0; /* extra debugging */ - int locked_bytes = 0; /* extra debugging */ - int compareLastRef(); - int compareSize(); + static time_t last_warning = 0; + static time_t last_check = 0; - if (!check_vm_number && ((store_mem_size + size) < store_mem_high)) + if (squid_curtime == last_check) return 0; + last_check = squid_curtime; + if (meta_data.hot_vm < store_hotobj_high && ((store_mem_size + size) < store_mem_high)) + return 0; debug(20, 2, "storeGetMemSpace: Starting...\n"); - LRU_list = create_dynamic_array(meta_data.store_in_mem_objects, MEM_LRUSCAN_BLOCK); - pending_entry_list = create_dynamic_array(meta_data.store_in_mem_objects, MEM_LRUSCAN_BLOCK); - + list = xcalloc(meta_data.store_in_mem_objects, sizeof(ipcache_entry *)); for (e = storeGetInMemFirst(); e; e = storeGetInMemNext()) { - n_scanned++; - n_inmem++; - if (e->store_status == STORE_PENDING) { - if (!(e->flag & DELETE_BEHIND)) { - /* it's not deleting behind, we can do something about it. */ - insert_dynamic_array(pending_entry_list, e); - } + if (list_count == meta_data.store_in_mem_objects) + break; + if (storeEntryLocked(e)) continue; - } if (squid_curtime > e->expires) { debug(20, 2, "storeGetMemSpace: Expired: %s\n", e->url); n_expired++; - /* Delayed release */ storeRelease(e); continue; } - if (squid_curtime + Config.negativeTtl > e->expires) { - debug(20, 2, "storeGetMemSpace: '%s' expires with Negative TTL time\n", e->url); - continue; - } - if (!storeEntryLocked(e)) { - insert_dynamic_array(LRU_list, e); - } else { - n_locked++; - locked_bytes += e->mem_obj->e_current_len; - debug(20, 5, "storeGetMemSpace: Locked: %s\n", - storeDescribeStatus(e)); - } + debug(20, 3, "storeGetMemSpace: Adding '%s'\n", e->url); + *(list + list_count) = e; + list_count++; } #ifdef EXTRA_DEBUGGING debug(20, 5, "storeGetMemSpace: Current size: %7d bytes\n", store_mem_size); debug(20, 5, "storeGetMemSpace: High W Mark: %7d bytes\n", store_mem_high); debug(20, 5, "storeGetMemSpace: Low W Mark: %7d bytes\n", store_mem_low); debug(20, 5, "storeGetMemSpace: Entry count: %7d items\n", meta_data.store_entries); - debug(20, 5, "storeGetMemSpace: Scanned: %7d items\n", n_scanned); - debug(20, 5, "storeGetMemSpace: In memory: %7d items\n", n_inmem); - debug(20, 5, "storeGetMemSpace: Hot vm count: %7d items\n", meta_data.hot_vm); - debug(20, 5, "storeGetMemSpace: Expired: %7d items\n", n_expired); - debug(20, 5, "storeGetMemSpace: Can't purge: %7d items\n", n_locked); - debug(20, 5, "storeGetMemSpace: Can't purge size: %7d bytes\n", locked_bytes); - debug(20, 5, "storeGetMemSpace: Sorting LRU_list: %7d items\n", LRU_list->index); #endif - qsort((char *) LRU_list->collection, - LRU_list->index, + debug(20, 5, "storeGetMemSpace: Sorting LRU_list: %7d items\n", list_count); + qsort((char *) list, + list_count, sizeof(StoreEntry *), - (QS) compareLastRef); + (QS) compareSize); /* Kick LRU out until we have enough memory space */ - - if (check_vm_number) { - /* look for vm slot */ - for (i = 0; (i < LRU_list->index) && (meta_data.hot_vm > store_hotobj_low); i++) { - if ((LRU = (StoreEntry *) LRU_list->collection[i])) - if ((LRU->store_status != STORE_PENDING) && (LRU->swap_status == NO_SWAP)) { - n_released++; - storeRelease(LRU); - } else { - n_purged++; - storePurgeMem(LRU); - } - } - } else { - /* look for space */ - for (i = 0; (i < LRU_list->index) && ((store_mem_size + size) > store_mem_low); i++) { - if ((LRU = (StoreEntry *) LRU_list->collection[i])) - if ((LRU->store_status != STORE_PENDING) && (LRU->swap_status == NO_SWAP)) { - n_released++; - storeRelease(LRU); - } else { - n_purged++; - storePurgeMem(LRU); - } - } + for (i = 0; i < list_count; i++) { + if (meta_data.hot_vm > store_hotobj_low) { + storeRelease(*(list + i)); + n_released++; + } else if ((store_mem_size + size) > store_mem_low) { + storeRelease(*(list + i)); + n_released++; + } else + break; } - destroy_dynamic_array(LRU_list); - debug(20, 2, "storeGetMemSpace: After freeing size: %7d bytes\n", store_mem_size); - debug(20, 2, "storeGetMemSpace: Purged: %7d items\n", n_purged); debug(20, 2, "storeGetMemSpace: Released: %7d items\n", n_released); - - - if (check_vm_number) { - /* don't check for size */ - destroy_dynamic_array(pending_entry_list); - debug(20, 2, "storeGetMemSpace: Done.\n"); - return 0; - } - if ((store_mem_size + size) < store_mem_high) { - /* we don't care for hot_vm count here, just the storage size. */ - over_highwater = over_max = 0; - destroy_dynamic_array(pending_entry_list); - debug(20, 2, "storeGetMemSpace: Done.\n"); - return 0; - } - if ((store_mem_size + size) < Config.Mem.maxSize) { - /* We're over high water mark here, but still under absolute max */ - if (!over_highwater) { - /* print only once when the condition occur until it clears. */ - debug(20, 1, "storeGetMemSpace: Allocating beyond the high water mark with total size of %d\n", - store_mem_size + size); - over_highwater = 1; + if (meta_data.hot_vm > store_hotobj_high) { + if (squid_curtime - last_warning > 10) { + debug(20, 0, "WARNING: Over hot_vm object count (%d > %d)\n", + meta_data.hot_vm, store_hotobj_high); + last_warning = squid_curtime; } - /* we can delete more than one if we want to be more aggressive. */ - entry_to_delete_behind = 1; - } else { - /* We're over absolute max */ - if (!over_max) { - /* print only once when the condition occur until it clears. */ - debug(20, 1, "storeGetMemSpace: Allocating beyond the MAX Store with total size of %d\n", - store_mem_size + size); - debug(20, 1, " Start Deleting Behind for every pending objects\n"); - debug(20, 1, " You should really adjust your cache_mem, high/low water mark,\n"); - debug(20, 1, " max object size to suit your need.\n"); - over_max = 1; + } else if ((store_mem_size + size) > store_mem_high) { + if (squid_curtime - last_warning > 10) { + debug(20, 0, "WARNING: Over store_mem high-water mark (%d > %d)\n", + store_mem_size + size, store_mem_high); + last_warning = squid_curtime; } - /* delete all of them, we desperate for a space. */ - entry_to_delete_behind = pending_entry_list->index; } - - /* sort the stuff by size */ - qsort((char *) pending_entry_list->collection, - pending_entry_list->index, - sizeof(StoreEntry *), - (QS) compareSize); - for (i = 0; (i < pending_entry_list->index) && (i < entry_to_delete_behind); i++) - if (pending_entry_list->collection[i]) { - n_deleted_behind++; - storeStartDeleteBehind(pending_entry_list->collection[i]); - } - if (n_deleted_behind) { - debug(20, 1, "storeGetMemSpace: Due to memory flucuation, put %d objects to DELETE_BEHIND MODE.\n", - n_deleted_behind); - } - destroy_dynamic_array(pending_entry_list); debug(20, 2, "storeGetMemSpace: Done.\n"); return 0; } @@ -2701,6 +2613,14 @@ int storeInit() store_hotobj_low = (int) (Config.hotVmFactor * store_mem_low / (1 << 20)); + /* Note, there will always be a few 'hot_vm' objects as the + * make the transition from STORE_PENDING to SWAPPING_OUT. + Set some minimum values to prevent thrasing and warning + messages. */ + if (store_hotobj_high < 32) { + store_hotobj_high = 32; + store_hotobj_low = 16; + } /* check for validity */ if (store_hotobj_low > store_hotobj_high) store_hotobj_low = store_hotobj_high;