/*
- * $Id: store.cc,v 1.513 2000/02/01 05:17:58 wessels Exp $
+ * $Id: store.cc,v 1.554 2003/01/23 00:37:26 robertc Exp $
*
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
*
- * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * SQUID Web Proxy Cache http://www.squid-cache.org/
* ----------------------------------------------------------
*
- * Squid is the result of efforts by numerous individuals from the
- * Internet community. Development is led by Duane Wessels of the
- * National Laboratory for Applied Network Research and funded by the
- * National Science Foundation. Squid is Copyrighted (C) 1998 by
- * Duane Wessels and the University of California San Diego. Please
- * see the COPYRIGHT file for full details. Squid incorporates
- * software developed and/or copyrighted by other sources. Please see
- * the CREDITS file for full details.
+ * Squid is the result of efforts by numerous individuals from
+ * the Internet community; see the CONTRIBUTORS file for full
+ * details. Many organizations have provided support for Squid's
+ * development; see the SPONSORS file for full details. Squid is
+ * Copyrighted (C) 2001 by the Regents of the University of
+ * California; see the COPYRIGHT file for full details. Squid
+ * incorporates software developed and/or copyrighted by other
+ * sources; see the CREDITS file for full details.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
- *
+ *
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
- *
+ *
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*/
#include "squid.h"
+#include "Store.h"
+#include "StoreClient.h"
+#include "stmem.h"
+#include "HttpReply.h"
+#include "HttpRequest.h"
+#include "MemObject.h"
+#include "mem_node.h"
+#include "StoreMeta.h"
+#include "SwapDir.h"
+
+static STMCB storeWriteComplete;
#define REBUILD_TIMESTAMP_DELTA_MAX 2
StoreEntry *e;
} lock_ctrl_t;
+extern OBJH storeIOStats;
+
/*
* local function prototypes
*/
-static int storeCheckExpired(const StoreEntry *);
-static int storeEntryLocked(const StoreEntry *);
-static int storeEntryValidLength(const StoreEntry *);
static void storeGetMemSpace(int);
static void storeHashDelete(StoreEntry *);
-static MemObject *new_MemObject(const char *, const char *);
static void destroy_MemObject(StoreEntry *);
-static FREE destroy_StoreEntry;
+static FREE destroyStoreEntry;
static void storePurgeMem(StoreEntry *);
+static void storeEntryReferenced(StoreEntry *);
+static void storeEntryDereferenced(StoreEntry *);
static int getKeyCounter(void);
static int storeKeepInMemory(const StoreEntry *);
static OBJH storeCheckCachableStats;
static EVH storeLateRelease;
-#if HEAP_REPLACEMENT
-static heap_key_func HeapKeyGen_StoreEntry_LFUDA;
-static heap_key_func HeapKeyGen_StoreEntry_GDSF;
-static heap_key_func HeapKeyGen_StoreEntry_LRU;
-#endif
/*
* local variables
*/
-#if HEAP_REPLACEMENT
-/*
- * The heap equivalent of inmem_list, inmem_heap, is in globals.c so other
- * modules can access it when updating object metadata (e.g., refcount)
- */
-#else
-static dlink_list inmem_list;
-#endif
-static int store_pages_max = 0;
-static int store_swap_high = 0;
-static int store_swap_low = 0;
static Stack LateReleaseStack;
+MemPool *StoreEntry::pool = NULL;
-#if URL_CHECKSUM_DEBUG
-unsigned int
-url_checksum(const char *url)
-{
- unsigned int ck;
- MD5_CTX M;
- static unsigned char digest[16];
- MD5Init(&M);
- MD5Update(&M, (unsigned char *) url, strlen(url));
- MD5Final(digest, &M);
- xmemcpy(&ck, digest, sizeof(ck));
- return ck;
+void *
+StoreEntry::operator new (size_t bytecount)
+{
+ assert (bytecount == sizeof (StoreEntry));
+ if (!pool) {
+ pool = memPoolCreate ("StoreEntry", bytecount);
+ memPoolSetChunkSize(pool, 2048 * 1024);
+ }
+ return memPoolAlloc (pool);
+}
+
+void
+StoreEntry::operator delete (void *address)
+{
+ memPoolFree(pool, address);
+}
+
+size_t
+StoreEntry::inUseCount()
+{
+ if (!pool)
+ return 0;
+ MemPoolStats stats;
+ memPoolGetStats (&stats, pool);
+ return stats.items_inuse;
+}
+
+const char *
+StoreEntry::getMD5Text() const
+{
+ return storeKeyText((const cache_key *)key);
}
-#endif
-static MemObject *
-new_MemObject(const char *url, const char *log_url)
+int
+StoreEntry::CheckDeferRead(int fd, void *data)
{
- MemObject *mem = memAllocate(MEM_MEMOBJECT);
- mem->reply = httpReplyCreate();
- mem->url = xstrdup(url);
+ assert (data);
+ return ((StoreEntry *)data)->checkDeferRead(fd);
+}
+
+int
+StoreEntry::checkDeferRead(int fd) const
+{
+ int rc = 0;
+ if (mem_obj == NULL)
+ return 0;
#if URL_CHECKSUM_DEBUG
- mem->chksum = url_checksum(mem->url);
+ mem_obj->checkUrlChecksum();
#endif
- mem->log_url = xstrdup(log_url);
- mem->object_sz = -1;
- mem->fd = -1;
- /* XXX account log_url */
- debug(20, 3) ("new_MemObject: returning %p\n", mem);
- return mem;
+#if DELAY_POOLS
+ if (fd < 0)
+ (void) 0;
+ else if (! delayIsNoDelay(fd)) {
+ int i = delayMostBytesWanted(mem_obj, INT_MAX);
+ if (0 == i)
+ return 1;
+ /* was: rc = -(rc != INT_MAX); */
+ else if (INT_MAX == i)
+ rc = 0;
+ else
+ rc = -1;
+ }
+#endif
+ if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT))
+ return rc;
+ if (mem_obj->readAheadPolicyCanRead())
+ return rc;
+ return 1;
+}
+
+store_client_t
+StoreEntry::storeClientType() const
+{
+ if (mem_obj->inmem_lo)
+ return STORE_DISK_CLIENT;
+ if (EBIT_TEST(flags, ENTRY_ABORTED)) {
+ /* I don't think we should be adding clients to aborted entries */
+ debug(20, 1) ("storeClientType: adding to ENTRY_ABORTED entry\n");
+ return STORE_MEM_CLIENT;
+ }
+ if (store_status == STORE_OK) {
+ if (mem_obj->inmem_lo == 0 && !isEmpty())
+ return STORE_MEM_CLIENT;
+ else
+ return STORE_DISK_CLIENT;
+ }
+ /* here and past, entry is STORE_PENDING */
+ /*
+ * If this is the first client, let it be the mem client
+ */
+ if (mem_obj->nclients == 1)
+ return STORE_MEM_CLIENT;
+ /*
+ * If there is no disk file to open yet, we must make this a
+ * mem client. If we can't open the swapin file before writing
+ * to the client, there is no guarantee that we will be able
+ * to open it later when we really need it.
+ */
+ if (swap_status == SWAPOUT_NONE)
+ return STORE_MEM_CLIENT;
+ /*
+ * otherwise, make subsequent clients read from disk so they
+ * can not delay the first, and vice-versa.
+ */
+ return STORE_DISK_CLIENT;
}
StoreEntry *
new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url)
{
StoreEntry *e = NULL;
- e = memAllocate(MEM_STOREENTRY);
+ e = new StoreEntry;
if (mem_obj_flag)
- e->mem_obj = new_MemObject(url, log_url);
- debug(20, 3) ("new_StoreEntry: returning %p\n", e);
+ e->mem_obj = new MemObject(url, log_url);
+ debug(20, 3) ("newStoreEntry: returning %p\n", e);
e->expires = e->lastmod = e->lastref = e->timestamp = -1;
- e->swap_file_number = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
return e;
}
static void
destroy_MemObject(StoreEntry * e)
{
+ storeSetMemStatus(e, NOT_IN_MEMORY);
MemObject *mem = e->mem_obj;
- const Ctx ctx = ctx_enter(mem->url);
- debug(20, 3) ("destroy_MemObject: destroying %p\n", mem);
-#if URL_CHECKSUM_DEBUG
- assert(mem->chksum == url_checksum(mem->url));
-#endif
e->mem_obj = NULL;
- if (!shutting_down)
- assert(mem->swapout.sio == NULL);
- stmemFree(&mem->data_hdr);
- mem->inmem_hi = 0;
- /*
- * There is no way to abort FD-less clients, so they might
- * still have mem->clients set if mem->fd == -1
- */
- assert(mem->fd == -1 || mem->clients == NULL);
- httpReplyDestroy(mem->reply);
- requestUnlink(mem->request);
- mem->request = NULL;
- ctx_exit(ctx); /* must exit before we free mem->url */
- safe_free(mem->url);
- safe_free(mem->log_url); /* XXX account log_url */
- memFree(mem, MEM_MEMOBJECT);
+ delete mem;
}
static void
-destroy_StoreEntry(void *data)
+destroyStoreEntry(void *data)
{
- StoreEntry *e = data;
- debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e);
+ StoreEntry *e = static_cast<StoreEntry *>(data);
+ debug(20, 3) ("destroyStoreEntry: destroying %p\n", e);
assert(e != NULL);
- if (e->mem_obj)
- destroy_MemObject(e);
+ if (e == NullStoreEntry::getInstance())
+ return;
+ destroy_MemObject(e);
storeHashDelete(e);
assert(e->key == NULL);
- memFree(e, MEM_STOREENTRY);
+ delete e;
}
/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n",
e, storeKeyText(key));
e->key = storeKeyDup(key);
- hash_join(store_table, (hash_link *) e);
-#if HEAP_REPLACEMENT
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- (void) 0;
- } else {
- e->node = heap_insert(store_heap, e);
- debug(20, 4) ("storeHashInsert: inserted node %p\n", e->node);
- }
-#endif
+ hash_join(store_table, e);
}
static void
storeHashDelete(StoreEntry * e)
{
- hash_remove_link(store_table, (hash_link *) e);
-#if HEAP_REPLACEMENT
- if (e->node) {
- debug(20, 4) ("storeHashDelete: deleting node %p\n", e->node);
- heap_delete(store_heap, e->node);
- e->node = NULL;
- }
-#endif
- storeKeyFree(e->key);
+ hash_remove_link(store_table, e);
+ storeKeyFree((const cache_key *)e->key);
e->key = NULL;
}
/* get rid of memory copy of the object */
-/* Only call this if storeCheckPurgeMem(e) returns 1 */
static void
storePurgeMem(StoreEntry * e)
{
if (e->mem_obj == NULL)
return;
debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n",
- storeKeyText(e->key));
- storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->getMD5Text());
destroy_MemObject(e);
if (e->swap_status != SWAPOUT_DONE)
storeRelease(e);
}
+static void
+storeEntryReferenced(StoreEntry * e)
+{
+ /* Notify the fs that we're referencing this object again */
+ if (e->swap_dirn > -1)
+ INDEXSD(e->swap_dirn)->reference(*e);
+ /* Notify the memory cache that we're referencing this object again */
+ if (e->mem_obj) {
+ if (mem_policy->Referenced)
+ mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+
+static void
+storeEntryDereferenced(StoreEntry * e)
+{
+ /* Notify the fs that we're not referencing this object any more */
+ if (e->swap_filen > -1)
+ INDEXSD(e->swap_dirn)->dereference(*e);
+ /* Notify the memory cache that we're not referencing this object any more */
+ if (e->mem_obj) {
+ if (mem_policy->Dereferenced)
+ mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+
void
storeLockObject(StoreEntry * e)
{
- if (e->lock_count++ == 0) {
-#if HEAP_REPLACEMENT
- /*
- * There is no reason to take any action here. Squid by
- * default is moving locked objects to the end of the LRU
- * list to keep them from getting bumped into by the
- * replacement algorithm. We can't do that so we will just
- * have to handle them.
- */
- debug(20, 4) ("storeLockObject: just locked node %p\n", e->node);
-#else
- storeDirLRUDelete(e);
- storeDirLRUAdd(e);
-#endif
- }
+ e->lock_count++;
debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
- storeKeyText(e->key), (int) e->lock_count);
+ e->getMD5Text(), (int) e->lock_count);
e->lastref = squid_curtime;
+ storeEntryReferenced(e);
}
void
{
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
return;
- debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->key));
+ debug(20, 3) ("storeReleaseRequest: '%s'\n", e->getMD5Text());
EBIT_SET(e->flags, RELEASE_REQUEST);
/*
* Clear cachable flag here because we might get called before
{
e->lock_count--;
debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
- storeKeyText(e->key), e->lock_count);
+ e->getMD5Text(), e->lock_count);
if (e->lock_count)
return (int) e->lock_count;
if (e->store_status == STORE_PENDING)
EBIT_SET(e->flags, RELEASE_REQUEST);
assert(storePendingNClients(e) == 0);
-#if HEAP_REPLACEMENT
- storeHeapPositionUpdate(e);
-#else
- storeDirLRUDelete(e);
- storeDirLRUAdd(e);
-#endif
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
storeRelease(e);
else if (storeKeepInMemory(e)) {
+ storeEntryDereferenced(e);
storeSetMemStatus(e, IN_MEMORY);
- requestUnlink(e->mem_obj->request);
- e->mem_obj->request = NULL;
+ e->mem_obj->unlinkRequest();
} else {
storePurgeMem(e);
+ storeEntryDereferenced(e);
if (EBIT_TEST(e->flags, KEY_PRIVATE))
debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__);
}
StoreEntry *
storeGet(const cache_key * key)
{
+ PROF_start(storeGet);
debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key));
- return (StoreEntry *) hash_lookup(store_table, key);
+ StoreEntry *p = static_cast<StoreEntry *>(hash_lookup(store_table, key));
+ PROF_stop(storeGet);
+ return p;
+}
+
+void
+StoreEntry::getPublicByRequestMethod (StoreClient *aClient, request_t * request, const method_t method)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublicByRequestMethod( request, method);
+ if (!result)
+ aClient->created (NullStoreEntry::getInstance());
+ else
+ aClient->created (result);
+}
+
+void
+StoreEntry::getPublicByRequest (StoreClient *aClient, request_t * request)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublicByRequest (request);
+ if (!result)
+ result = NullStoreEntry::getInstance();
+ aClient->created (result);
+}
+
+void
+StoreEntry::getPublic (StoreClient *aClient, const char *uri, const method_t method)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublic (uri, method);
+ if (!result)
+ result = NullStoreEntry::getInstance();
+ aClient->created (result);
}
StoreEntry *
return storeGet(storeKeyPublic(uri, method));
}
+StoreEntry *
+storeGetPublicByRequestMethod(request_t * req, const method_t method)
+{
+ return storeGet(storeKeyPublicByRequestMethod(req, method));
+}
+
+StoreEntry *
+storeGetPublicByRequest(request_t * req)
+{
+ StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
+ if (e == NULL && req->method == METHOD_HEAD)
+ /* We can generate a HEAD reply from a cached GET object */
+ e = storeGetPublicByRequestMethod(req, METHOD_GET);
+ return e;
+}
+
static int
getKeyCounter(void)
{
if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE))
return; /* is already private */
if (e->key) {
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_DEL);
storeHashDelete(e);
}
* If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
* be set, and storeSetPublicKey() should not be called.
*/
-#if HEAP_REPLACEMENT
+#if MORE_DEBUG_OUTPUT
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n",
e->key, mem->url);
#endif
assert(!EBIT_TEST(e->flags, RELEASE_REQUEST));
- newkey = storeKeyPublic(mem->url, mem->method);
+ if (mem->request) {
+ StoreEntry *pe;
+ request_t *request = mem->request;
+ if (!mem->vary_headers) {
+ /* First handle the case where the object no longer varies */
+ safe_free(request->vary_headers);
+ } else {
+ if (request->vary_headers && strcmp(request->vary_headers, mem->vary_headers) != 0) {
+ /* Oops.. the variance has changed. Kill the base object
+ * to record the new variance key
+ */
+ safe_free(request->vary_headers); /* free old "bad" variance key */
+ pe = storeGetPublic(mem->url, mem->method);
+ if (pe)
+ storeRelease(pe);
+ }
+ /* Make sure the request knows the variance status */
+ if (!request->vary_headers) {
+ const char *vary = httpMakeVaryMark(request, mem->getReply());
+ if (vary)
+ request->vary_headers = xstrdup(vary);
+ }
+ }
+ if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) {
+ /* Create "vary" base object */
+ http_version_t version;
+ String vary;
+ pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method);
+ httpBuildVersion(&version, 1, 0);
+ /* We are allowed to do this typecast */
+ httpReplySetHeaders((HttpReply *)pe->getReply(), version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
+ vary = httpHeaderGetList(&mem->getReply()->header, HDR_VARY);
+ if (vary.size()) {
+ /* Again, we own this structure layout */
+ httpHeaderPutStr((HttpHeader *)&pe->getReply()->header, HDR_VARY, vary.buf());
+ vary.clean();
+ }
+#if X_ACCELERATOR_VARY
+ vary = httpHeaderGetList(&mem->getReply()->header, HDR_X_ACCELERATOR_VARY);
+ if (vary.buf()) {
+ httpHeaderPutStr(&pe->getReply()->header, HDR_X_ACCELERATOR_VARY, vary.buf());
+ vary.clean();
+ }
+#endif
+ storeSetPublicKey(pe);
+ /* TODO: remove this when the metadata is separated */
+ {
+ Packer p;
+ packerToStoreInit(&p, pe);
+ httpReplyPackHeadersInto(pe->getReply(), &p);
+ packerClean(&p);
+ }
+ storeBufferFlush(pe);
+ storeTimestampsSet(pe);
+ pe->complete();
+ storeUnlockObject(pe);
+ }
+ newkey = storeKeyPublicByRequest(mem->request);
+ } else
+ newkey = storeKeyPublic(mem->url, mem->method);
if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url);
storeSetPrivateKey(e2);
storeRelease(e2);
- newkey = storeKeyPublic(mem->url, mem->method);
+ if (mem->request)
+ newkey = storeKeyPublicByRequest(mem->request);
+ else
+ newkey = storeKeyPublic(mem->url, mem->method);
}
if (e->key)
storeHashDelete(e);
EBIT_CLR(e->flags, KEY_PRIVATE);
storeHashInsert(e, newkey);
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_ADD);
}
e->store_status = STORE_PENDING;
storeSetMemStatus(e, NOT_IN_MEMORY);
e->swap_status = SWAPOUT_NONE;
- e->swap_file_number = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
e->refcount = 0;
e->lastref = squid_curtime;
- e->timestamp = 0; /* set in storeTimestampsSet() */
+ e->timestamp = -1; /* set in storeTimestampsSet() */
e->ping_status = PING_NONE;
EBIT_SET(e->flags, ENTRY_VALIDATED);
return e;
void
storeExpireNow(StoreEntry * e)
{
- debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->key));
+ debug(20, 3) ("storeExpireNow: '%s'\n", e->getMD5Text());
e->expires = squid_curtime;
}
+void
+storeWriteComplete (void *data, StoreIOBuffer wroteBuffer)
+{
+ StoreEntry *e = (StoreEntry *)data;
+ if (EBIT_TEST(e->flags, DELAY_SENDING))
+ return;
+ InvokeHandlers(e);
+}
+
+void
+StoreEntry::write (StoreIOBuffer writeBuffer)
+{
+ assert(mem_obj != NULL);
+ assert(writeBuffer.length >= 0);
+ /* This assert will change when we teach the store to update */
+ assert(store_status == STORE_PENDING);
+ if (!writeBuffer.length)
+ return;
+
+ debug(20, 5) ("storeWrite: writing %u bytes for '%s'\n",
+ writeBuffer.length, getMD5Text());
+ storeGetMemSpace(writeBuffer.length);
+ mem_obj->write (writeBuffer, storeWriteComplete, this);
+}
+
/* Append incoming data from a primary server to an entry. */
void
storeAppend(StoreEntry * e, const char *buf, int len)
assert(mem != NULL);
assert(len >= 0);
assert(e->store_status == STORE_PENDING);
- if (len) {
- debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
- len,
- storeKeyText(e->key));
- storeGetMemSpace(len);
- stmemAppend(&mem->data_hdr, buf, len);
- mem->inmem_hi += len;
- }
- if (EBIT_TEST(e->flags, DELAY_SENDING))
- return;
- InvokeHandlers(e);
- storeSwapOut(e);
+
+ StoreIOBuffer tempBuffer;
+ tempBuffer.data = (char *)buf;
+ tempBuffer.length = len;
+ tempBuffer.offset = mem->endOffset() - (e->getReply() ? e->getReply()->hdr_sz : 0);
+ e->write(tempBuffer);
}
void
int wrong_content_length;
int negative_cached;
int too_big;
+ int too_small;
int private_key;
int too_many_open_files;
int too_many_open_fds;
return 0;
}
+static int
+storeCheckTooSmall(StoreEntry * e)
+{
+ MemObject * const mem = e->mem_obj;
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+ return 0;
+ if (STORE_OK == e->store_status)
+ if (mem->object_sz < 0 ||
+ static_cast<size_t>(mem->object_sz) < Config.Store.minObjectSize)
+ return 1;
+ if (e->getReply()->content_length > -1)
+ if (e->getReply()->content_length < (int) Config.Store.minObjectSize)
+ return 1;
+ return 0;
+}
+
int
storeCheckCachable(StoreEntry * e)
{
debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
store_check_cachable_hist.no.negative_cached++;
return 0; /* avoid release call below */
- } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
+ } else if ((e->getReply()->content_length > 0 &&
+ static_cast<size_t>(e->getReply()->content_length) > Config.Store.maxObjectSize) ||
+ static_cast<size_t>(e->mem_obj->endOffset()) > Config.Store.maxObjectSize) {
debug(20, 2) ("storeCheckCachable: NO: too big\n");
store_check_cachable_hist.no.too_big++;
- } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) {
+ } else if (e->getReply()->content_length > (int) Config.Store.maxObjectSize) {
debug(20, 2) ("storeCheckCachable: NO: too big\n");
store_check_cachable_hist.no.too_big++;
+ } else if (storeCheckTooSmall(e)) {
+ debug(20, 2) ("storeCheckCachable: NO: too small\n");
+ store_check_cachable_hist.no.too_small++;
} else if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
debug(20, 3) ("storeCheckCachable: NO: private key\n");
store_check_cachable_hist.no.private_key++;
store_check_cachable_hist.no.negative_cached);
storeAppendPrintf(sentry, "no.too_big\t%d\n",
store_check_cachable_hist.no.too_big);
+ storeAppendPrintf(sentry, "no.too_small\t%d\n",
+ store_check_cachable_hist.no.too_small);
storeAppendPrintf(sentry, "no.private_key\t%d\n",
store_check_cachable_hist.no.private_key);
storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
store_check_cachable_hist.yes.Default);
}
-/* Complete transfer into the local cache. */
void
-storeComplete(StoreEntry * e)
+StoreEntry::complete()
{
- debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key));
- if (e->store_status != STORE_PENDING) {
+ debug(20, 3) ("storeComplete: '%s'\n", getMD5Text());
+ if (store_status != STORE_PENDING) {
/*
* if we're not STORE_PENDING, then probably we got aborted
* and there should be NO clients on this entry
*/
- assert(EBIT_TEST(e->flags, ENTRY_ABORTED));
- assert(e->mem_obj->nclients == 0);
+ assert(EBIT_TEST(flags, ENTRY_ABORTED));
+ assert(mem_obj->nclients == 0);
return;
}
- e->mem_obj->object_sz = e->mem_obj->inmem_hi;
- e->store_status = STORE_OK;
- assert(e->mem_status == NOT_IN_MEMORY);
- if (!storeEntryValidLength(e)) {
- EBIT_SET(e->flags, ENTRY_BAD_LENGTH);
- storeReleaseRequest(e);
+ mem_obj->object_sz = mem_obj->endOffset();
+ store_status = STORE_OK;
+ assert(mem_status == NOT_IN_MEMORY);
+ if (!validLength()) {
+ EBIT_SET(flags, ENTRY_BAD_LENGTH);
+ storeReleaseRequest(this);
}
#if USE_CACHE_DIGESTS
- if (e->mem_obj->request)
- e->mem_obj->request->hier.store_complete_stop = current_time;
+ if (mem_obj->request)
+ mem_obj->request->hier.store_complete_stop = current_time;
#endif
- InvokeHandlers(e);
- storeSwapOut(e);
+ /*
+ * We used to call InvokeHandlers, then storeSwapOut. However,
+ * Madhukar Reddy <myreddy@persistence.com> reported that
+ * responses without content length would sometimes get released
+ * in client_side, thinking that the response is incomplete.
+ */
+ InvokeHandlers(this);
}
/*
void
storeAbort(StoreEntry * e)
{
+ statCounter.aborted_requests++;
MemObject *mem = e->mem_obj;
assert(e->store_status == STORE_PENDING);
assert(mem != NULL);
- debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->key));
+ debug(20, 6) ("storeAbort: %s\n", e->getMD5Text());
storeLockObject(e); /* lock while aborting */
storeNegativeCache(e);
storeReleaseRequest(e);
* We assign an object length here. The only other place we assign
* the object length is in storeComplete()
*/
- mem->object_sz = mem->inmem_hi;
+ /* RBC: What do we need an object length for? we've just aborted the
+ * request, the request is private and negatively cached. Surely
+ * the object length is inappropriate to set.
+ */
+ mem->object_sz = mem->endOffset();
/* Notify the server side */
if (mem->abort.callback) {
eventAdd("mem->abort.callback",
mem->abort.callback = NULL;
mem->abort.data = NULL;
}
+ /* XXX Should we reverse these two, so that there is no
+ * unneeded disk swapping triggered?
+ */
/* Notify the client side */
InvokeHandlers(e);
- /* Do we need to close the swapout file? */
- /* Not if we never started swapping out */
- if (e->swap_file_number > -1) {
- storeSwapOutFileClose(e);
- }
+ /* Close any swapout file */
+ storeSwapOutFileClose(e);
storeUnlockObject(e); /* unlock */
}
StoreEntry *e = NULL;
int released = 0;
static time_t last_check = 0;
- int pages_needed;
- int locked = 0;
-#if !HEAP_REPLACEMENT
- dlink_node *head;
- dlink_node *m;
- dlink_node *prev = NULL;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#endif
+ size_t pages_needed;
+ RemovalPurgeWalker *walker;
if (squid_curtime == last_check)
return;
last_check = squid_curtime;
pages_needed = (size / SM_PAGE_SIZE) + 1;
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
- return;
- if (store_dirs_rebuilding)
+ if (mem_node::InUseCount() + pages_needed < store_pages_max)
return;
debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed);
-#if HEAP_REPLACEMENT
- while (heap_nodes(inmem_heap) > 0) {
- age = heap_peepminkey(inmem_heap);
- e = heap_extractmin(inmem_heap);
- e->mem_obj->node = NULL; /* no longer in the heap */
- if (storeEntryLocked(e)) {
- locked++;
- debug(20, 5) ("storeGetMemSpace: locked key %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
- released++;
- debug(20, 3) ("Released memory object with key %f size %d refs %d url %s\n",
- age, e->swap_file_sz, e->refcount, e->mem_obj->url);
- min_age = age;
+ /* XXX what to set as max_scan here? */
+ walker = mem_policy->PurgeInit(mem_policy, 100000);
+ while ((e = walker->Next(walker))) {
storePurgeMem(e);
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
- break;
- }
- /*
- * Increase the heap age factor.
- */
- if (min_age > 0)
- inmem_heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->mem_obj->node = heap_insert(inmem_heap, e);
-#else
- head = inmem_list.head;
- for (m = inmem_list.tail; m; m = prev) {
- if (m == head)
- break;
- prev = m->prev;
- e = m->data;
- if (storeEntryLocked(e)) {
- locked++;
- dlinkDelete(m, &inmem_list);
- dlinkAdd(e, m, &inmem_list);
- continue;
- }
released++;
- storePurgeMem(e);
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
+ if (mem_node::InUseCount() + pages_needed < store_pages_max)
break;
}
-#endif
- debug(20, 3) ("storeGetMemSpace: released %d/%d locked %d\n",
- released, hot_obj_count, locked);
+ walker->Done(walker);
debug(20, 3) ("storeGetMemSpace stats:\n");
debug(20, 3) (" %6d HOT objects\n", hot_obj_count);
debug(20, 3) (" %6d were released\n", released);
/*
* This routine is to be called by main loop in main.c.
* It removes expired objects on only one bucket for each time called.
- * returns the number of objects removed
*
* This should get called 1/s from main().
*/
void
storeMaintainSwapSpace(void *datanotused)
{
- StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
- int max_scan;
- int max_remove;
int i;
- int j;
- static int ndir = 0;
- double f;
+ SwapDir *SD;
static time_t last_warn_time = 0;
-#if !HEAP_REPLACEMENT
- SwapDir *sd;
-#else
- heap_key age;
- heap_key min_age = 0.0;
- link_list *locked_entries = NULL;
-#if HEAP_REPLACEMENT_DEBUG
- if (!verify_heap_property(store_heap)) {
- debug(20, 1) ("Heap property violated!\n");
- }
-#endif
-#endif
- /* We can't delete objects while rebuilding swap */
- if (store_dirs_rebuilding) {
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
- return;
- } else {
- f = (double) (store_swap_size - store_swap_low) / (store_swap_high - store_swap_low);
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
- max_scan = (int) (f * 400.0 + 100.0);
- if ((max_remove = stat5minClientRequests()) < 10)
- max_remove = 10;
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0 - f, 1);
- }
- debug(20, 3) ("storeMaintainSwapSpace: f=%f, max_scan=%d, max_remove=%d\n",
- f, max_scan, max_remove);
-#if HEAP_REPLACEMENT
- while (heap_nodes(store_heap) > 0) {
- if (store_swap_size < store_swap_low)
- break;
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
- age = heap_peepminkey(store_heap);
- e = heap_extractmin(store_heap);
- e->node = NULL; /* no longer in the heap */
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * Entry is in use ... put it in a linked list to ignore it.
- */
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- /*
- * If this was a "SPECIAL" do not add it back into the heap.
- * It will always be "SPECIAL" and therefore never removed.
- */
- debug(20, 4) ("storeMaintainSwapSpace: locked url %s\n",
- (e->mem_obj && e->mem_obj->url) ? e->mem_obj->url : storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- }
- locked++;
- continue;
- } else if (storeCheckExpired(e)) {
- /*
- * Note: This will not check the reference age ifdef
- * HEAP_REPLACEMENT, but it does some other useful
- * checks...
- */
- expired++;
- debug(20, 3) ("Released store object age %f size %d refs %d key %s\n",
- age, e->swap_file_sz, e->refcount, storeKeyText(e->key));
- min_age = age;
- storeRelease(e);
- } else {
- /*
- * Did not expire the object so we need to add it back
- * into the heap!
- */
- debug(20, 5) ("storeMaintainSwapSpace: non-expired %s\n",
- storeKeyText(e->key));
- linklistPush(&locked_entries, e);
- continue;
- }
- if (store_swap_size < store_swap_low)
- break;
- else if (expired >= max_remove)
- break;
- else if (scanned >= max_scan)
- break;
- }
- /*
- * Bump the heap age factor.
- */
- if (min_age > 0.0)
- store_heap->age = min_age;
- /*
- * Reinsert all bumped locked entries back into heap...
- */
- while ((e = linklistShift(&locked_entries)))
- e->node = heap_insert(store_heap, e);
-#else
+
+ PROF_start(storeMaintainSwapSpace);
+ /* walk each fs */
for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- sd = &Config.cacheSwap.swapDirs[i];
- sd->lru_walker = sd->lru_list.tail;
+ /* call the maintain function .. */
+ SD = INDEXSD(i);
+ /* XXX FixMe: This should be done "in parallell" on the different
+ * cache_dirs, not one at a time.
+ */
+ SD->maintainfs();
}
- do {
- j = 0;
- for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- if (ndir >= Config.cacheSwap.n_configured)
- ndir = ndir % Config.cacheSwap.n_configured;
- sd = &Config.cacheSwap.swapDirs[ndir++];
- if (sd->cur_size < sd->high_size)
- continue;
- if (NULL == sd->lru_walker)
- continue;
- e = sd->lru_walker->data;
- sd->lru_walker = sd->lru_walker->prev;
- j++;
- scanned++;
- sd->scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- storeDirLRUDelete(e);
- if (!EBIT_TEST(e->flags, ENTRY_SPECIAL))
- storeDirLRUAdd(e);
- }
- locked++;
- } else if (storeCheckExpired(e)) {
- expired++;
- sd->removals++;
- storeRelease(e);
- }
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
+ if (store_swap_size > Config.Swap.maxSize) {
+ if (squid_curtime - last_warn_time > 10) {
+ debug(20, 0) ("WARNING: Disk space over limit: %ld KB > %ld KB\n",
+ (long int) store_swap_size, (long int) Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
}
- } while (j > 0 && expired < max_remove && scanned < max_scan);
-#endif
- debug(20, (expired ? 2 : 3)) ("storeMaintainSwapSpace: scanned %d/%d removed %d/%d locked %d f=%.03f\n",
- scanned, max_scan, expired, max_remove, locked, f);
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+ }
+ /* Reregister a maintain event .. */
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
+ PROF_stop(storeMaintainSwapSpace);
}
/* release an object from a cache */
-/* return number of objects released. */
void
storeRelease(StoreEntry * e)
{
- debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->key));
+ PROF_start(storeRelease);
+ debug(20, 3) ("storeRelease: Releasing: '%s'\n", e->getMD5Text());
/* If, for any reason we can't discard this object because of an
* outstanding request, mark it for pending release */
if (storeEntryLocked(e)) {
storeExpireNow(e);
debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n");
storeReleaseRequest(e);
+ PROF_stop(storeRelease);
return;
}
- if (store_dirs_rebuilding && e->swap_file_number > -1) {
+ if (store_dirs_rebuilding && e->swap_filen > -1) {
storeSetPrivateKey(e);
- if (e->mem_obj) {
- storeSetMemStatus(e, NOT_IN_MEMORY);
+ if (e->mem_obj)
destroy_MemObject(e);
+ if (e->swap_filen > -1) {
+ /*
+ * Fake a call to storeLockObject(). When rebuilding is done,
+ * we'll just call storeUnlockObject() on these.
+ */
+ e->lock_count++;
+ EBIT_SET(e->flags, RELEASE_REQUEST);
+ stackPush(&LateReleaseStack, e);
+ PROF_stop(storeRelease);
+ return;
+ } else {
+ destroyStoreEntry(e);
}
- /*
- * Fake a call to storeLockObject(). When rebuilding is done,
- * we'll just call storeUnlockObject() on these.
- */
- e->lock_count++;
- EBIT_SET(e->flags, RELEASE_REQUEST);
- stackPush(&LateReleaseStack, e);
- return;
}
storeLog(STORE_LOG_RELEASE, e);
- if (e->swap_file_number > -1) {
- storeUnlink(e->swap_file_number);
+ if (e->swap_filen > -1) {
+ storeUnlink(e);
if (e->swap_status == SWAPOUT_DONE)
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1);
+ storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, -1);
if (!EBIT_TEST(e->flags, KEY_PRIVATE))
storeDirSwapLog(e, SWAP_LOG_DEL);
+#if 0
+ /* From 2.4. I think we do this in storeUnlink? */
storeSwapFileNumberSet(e, -1);
+#endif
}
storeSetMemStatus(e, NOT_IN_MEMORY);
- destroy_StoreEntry(e);
+ destroyStoreEntry(e);
+ PROF_stop(storeRelease);
}
static void
return;
}
for (i = 0; i < 10; i++) {
- e = stackPop(&LateReleaseStack);
+ e = static_cast<StoreEntry*>(stackPop(&LateReleaseStack));
if (e == NULL) {
/* done! */
debug(20, 1) ("storeLateRelease: released %d objects\n", n);
}
/* return 1 if a store entry is locked */
-static int
+int
storeEntryLocked(const StoreEntry * e)
{
if (e->lock_count)
return 0;
}
-static int
-storeEntryValidLength(const StoreEntry * e)
+bool
+StoreEntry::validLength() const
{
int diff;
const HttpReply *reply;
- assert(e->mem_obj != NULL);
- reply = e->mem_obj->reply;
- debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key));
+ assert(mem_obj != NULL);
+ reply = getReply();
+ debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", getMD5Text());
debug(20, 5) ("storeEntryValidLength: object_len = %d\n",
- objectLen(e));
+ objectLen(this));
debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n",
reply->hdr_sz);
debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
reply->content_length);
if (reply->content_length < 0) {
debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
if (reply->hdr_sz == 0) {
debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
- if (e->mem_obj->method == METHOD_HEAD) {
+ if (mem_obj->method == METHOD_HEAD) {
debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
if (reply->sline.status == HTTP_NOT_MODIFIED)
return 1;
if (reply->sline.status == HTTP_NO_CONTENT)
return 1;
- diff = reply->hdr_sz + reply->content_length - objectLen(e);
+ diff = reply->hdr_sz + reply->content_length - objectLen(this);
if (diff == 0)
return 1;
debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
diff < 0 ? -diff : diff,
diff < 0 ? "big" : "small",
- storeKeyText(e->key));
+ getMD5Text());
return 0;
}
static void
storeInitHashValues(void)
{
- int i;
+ long int i;
/* Calculate size of hash table (maximum currently 64k buckets). */
i = Config.Swap.maxSize / Config.Store.avgObjectSize;
- debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n",
- Config.Swap.maxSize, i);
+ debug(20, 1) ("Swap maxSize %ld KB, estimated %ld objects\n",
+ (long int) Config.Swap.maxSize, i);
i /= Config.Store.objectsPerBucket;
- debug(20, 1) ("Target number of buckets: %d\n", i);
+ debug(20, 1) ("Target number of buckets: %ld\n", i);
/* ideally the full scan period should be configurable, for the
* moment it remains at approximately 24 hours. */
store_hash_buckets = storeKeyHashBuckets(i);
debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets);
- debug(20, 1) ("Max Mem size: %d KB\n", Config.memMaxSize >> 10);
- debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize);
+ debug(20, 1) ("Max Mem size: %ld KB\n", (long int) Config.memMaxSize >> 10);
+ debug(20, 1) ("Max Swap size: %ld KB\n", (long int) Config.Swap.maxSize);
}
-#if HEAP_REPLACEMENT
-#include "store_heap_replacement.c"
-#endif
-
void
storeInit(void)
{
storeInitHashValues();
store_table = hash_create(storeKeyHashCmp,
store_hash_buckets, storeKeyHashHash);
+ mem_policy = createRemovalPolicy(Config.memPolicy);
storeDigestInit();
storeLogOpen();
-#if HEAP_REPLACEMENT
- /*
- * Create new heaps with cache replacement policies attached to them.
- * The cache replacement policy is specified as either GDSF or LFUDA in
- * the squid.conf configuration file. Note that the replacement policy
- * applies only to the disk replacement algorithm. Memory replacement
- * always uses GDSF since we want to maximize object hit rate.
- */
- inmem_heap = new_heap(1000, HeapKeyGen_StoreEntry_GDSF);
- if (Config.replPolicy) {
- if (tolower(Config.replPolicy[0]) == 'g') {
- debug(20, 1) ("Using GDSF disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- } else if (tolower(Config.replPolicy[0]) == 'l') {
- if (tolower(Config.replPolicy[1]) == 'f') {
- debug(20, 1) ("Using LFUDA disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LFUDA);
- } else if (tolower(Config.replPolicy[1]) == 'r') {
- debug(20, 1) ("Using LRU heap disk replacement policy\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_LRU);
- }
- } else {
- debug(20, 1) ("Unrecognized replacement_policy; using GDSF\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
- } else {
- debug(20, 1) ("Using default disk replacement policy (GDSF)\n");
- store_heap = new_heap(10000, HeapKeyGen_StoreEntry_GDSF);
- }
-#else
- inmem_list.head = inmem_list.tail = NULL;
-#endif
stackInit(&LateReleaseStack);
eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
storeDirInit();
cachemgrRegister("store_check_cachable_stats",
"storeCheckCachable() Stats",
storeCheckCachableStats, 0, 1);
+ cachemgrRegister("store_io",
+ "Store IO Interface Stats",
+ storeIOStats, 0, 1);
}
void
return mem->inmem_lo == 0;
}
-static int
-storeCheckExpired(const StoreEntry * e)
+int
+storeCheckNegativeHit(StoreEntry * e)
{
- if (storeEntryLocked(e))
+ if (!EBIT_TEST(e->flags, ENTRY_NEGCACHED))
+ return 0;
+ if (e->expires <= squid_curtime)
+ return 0;
+ if (e->store_status != STORE_OK)
return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
return 1;
}
-#if !HEAP_REPLACEMENT
-/*
- * storeExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-time_t
-storeExpiredReferenceAge(void)
-{
- double x;
- double z;
- time_t age;
- x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > 31536000)
- age = 31536000;
- return age;
-}
-#endif
-
void
storeNegativeCache(StoreEntry * e)
{
void
storeFreeMemory(void)
{
- hashFreeItems(store_table, destroy_StoreEntry);
+ hashFreeItems(store_table, destroyStoreEntry);
hashFreeMemory(store_table);
store_table = NULL;
#if USE_CACHE_DIGESTS
void
storeTimestampsSet(StoreEntry * entry)
{
- const HttpReply *reply = entry->mem_obj->reply;
+ const HttpReply *reply = entry->getReply();
time_t served_date = reply->date;
+ int age = httpHeaderGetInt(&reply->header, HDR_AGE);
+ /*
+ * The timestamp calculations below tries to mimic the properties
+ * of the age calculation in RFC2616 section 13.2.3. The implementaion
+ * isn't complete, and the most notable exception from the RFC is that
+ * this does not account for response_delay, but it probably does
+ * not matter much as this is calculated immediately when the headers
+ * are received, not when the whole response has been received.
+ */
/* make sure that 0 <= served_date <= squid_curtime */
if (served_date < 0 || served_date > squid_curtime)
served_date = squid_curtime;
+ /*
+ * Compensate with Age header if origin server clock is ahead
+ * of us and there is a cache in between us and the origin
+ * server. But DONT compensate if the age value is larger than
+ * squid_curtime because it results in a negative served_date.
+ */
+ if (age > squid_curtime - served_date)
+ if (squid_curtime > age)
+ served_date = squid_curtime - age;
entry->expires = reply->expires;
entry->lastmod = reply->last_modified;
entry->timestamp = served_date;
mem->abort.callback = NULL;
}
-void
-storeMemObjectDump(MemObject * mem)
-{
- debug(20, 1) ("MemObject->data.head: %p\n",
- mem->data_hdr.head);
- debug(20, 1) ("MemObject->data.tail: %p\n",
- mem->data_hdr.tail);
- debug(20, 1) ("MemObject->data.origin_offset: %d\n",
- mem->data_hdr.origin_offset);
- debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
- (int) mem->start_ping.tv_sec,
- (int) mem->start_ping.tv_usec);
- debug(20, 1) ("MemObject->inmem_hi: %d\n",
- (int) mem->inmem_hi);
- debug(20, 1) ("MemObject->inmem_lo: %d\n",
- (int) mem->inmem_lo);
- debug(20, 1) ("MemObject->clients: %p\n",
- mem->clients);
- debug(20, 1) ("MemObject->nclients: %d\n",
- mem->nclients);
- debug(20, 1) ("MemObject->reply: %p\n",
- mem->reply);
- debug(20, 1) ("MemObject->request: %p\n",
- mem->request);
- debug(20, 1) ("MemObject->log_url: %p %s\n",
- mem->log_url,
- checkNullString(mem->log_url));
-}
-
void
storeEntryDump(const StoreEntry * e, int l)
{
- debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->key));
+ debug(20, l) ("StoreEntry->key: %s\n", e->getMD5Text());
debug(20, l) ("StoreEntry->next: %p\n", e->next);
debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj);
debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp);
debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
- debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number);
+ debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn);
+ debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen);
debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count);
debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status);
debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status);
* NOTE, this function assumes only two mem states
*/
void
-storeSetMemStatus(StoreEntry * e, int new_status)
+storeSetMemStatus(StoreEntry * e, mem_status_t new_status)
{
MemObject *mem = e->mem_obj;
if (new_status == e->mem_status)
assert(mem != NULL);
if (new_status == IN_MEMORY) {
assert(mem->inmem_lo == 0);
-#if HEAP_REPLACEMENT
- if (mem->node == NULL) {
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
- debug(20, 4) ("storeSetMemStatus: not inserting special %s\n",
- mem->url);
- } else {
- mem->node = heap_insert(inmem_heap, e);
- debug(20, 4) ("storeSetMemStatus: inserted mem node %p\n",
- mem->node);
- }
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n",
+ mem->url);
+ } else {
+ mem_policy->Add(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n",
+ mem->url);
}
-#else
- dlinkAdd(e, &mem->lru, &inmem_list);
-#endif
hot_obj_count++;
} else {
-#if HEAP_REPLACEMENT
- /*
- * It's being removed from the memory heap; is it already gone?
- */
- if (mem->node) {
- heap_delete(inmem_heap, mem->node);
- debug(20, 4) ("storeSetMemStatus: deleted mem node %p\n",
- mem->node);
- mem->node = NULL;
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: special entry %s\n",
+ mem->url);
+ } else {
+ mem_policy->Remove(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: removed mem node %s\n",
+ mem->url);
}
-#else
- dlinkDelete(&mem->lru, &inmem_list);
-#endif
hot_obj_count--;
}
e->mem_status = new_status;
{
if (e->mem_obj)
return;
- e->mem_obj = new_MemObject(url, log_url);
+ e->mem_obj = new MemObject(url, log_url);
}
/* this just sets DELAY_SENDING */
{
EBIT_CLR(e->flags, DELAY_SENDING);
InvokeHandlers(e);
- storeSwapOut(e);
}
-int
+ssize_t
objectLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
contentLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
- assert(e->mem_obj->reply != NULL);
- return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz;
+ assert(e->getReply() != NULL);
+ return objectLen(e) - e->getReply()->hdr_sz;
+
}
-HttpReply *
-storeEntryReply(StoreEntry * e)
+HttpReply const *
+StoreEntry::getReply () const
{
- if (NULL == e)
- return NULL;
- if (NULL == e->mem_obj)
+ if (NULL == mem_obj)
return NULL;
- return e->mem_obj->reply;
+ return mem_obj->getReply();
}
void
storeEntryReset(StoreEntry * e)
{
MemObject *mem = e->mem_obj;
+ assert (mem);
debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e));
- assert(mem->swapout.sio == NULL);
- stmemFree(&mem->data_hdr);
- mem->inmem_hi = mem->inmem_lo = 0;
- httpReplyDestroy(mem->reply);
- mem->reply = httpReplyCreate();
+ mem->reset();
+ httpReplyReset((HttpReply *)e->getReply());
e->expires = e->lastmod = e->timestamp = -1;
}
-#if HEAP_REPLACEMENT
+/*
+ * storeFsInit
+ *
+ * This routine calls the SETUP routine for each fs type.
+ * I don't know where the best place for this is, and I'm not going to shuffle
+ * around large chunks of code right now (that can be done once its working.)
+ */
void
-storeHeapPositionUpdate(StoreEntry * e)
+storeFsInit(void)
{
- if (e->node)
- heap_update(store_heap, e->node, e);
- if (e->mem_obj && e->mem_obj->node)
- heap_update(inmem_heap, e->mem_obj->node, e);
+ storeReplSetup();
+ storeFsSetup();
+}
+
+
+/*
+ * similar to above, but is called when a graceful shutdown is to occur
+ * of each fs module.
+ */
+void
+storeFsDone(void)
+{
+ int i = 0;
+
+ while (storefs_list[i].typestr != NULL) {
+ storefs_list[i].donefunc();
+ i++;
+ }
+}
+
+/*
+ * called to add another store fs module
+ * RBC: doesn't belong here. Move IT.
+ */
+void
+StoreEntry::FsAdd(const char *type, STSETUP * setup)
+{
+ int i;
+ /* find the number of currently known storefs types */
+ for (i = 0; storefs_list && storefs_list[i].typestr; i++) {
+ assert(strcmp(storefs_list[i].typestr, type) != 0);
+ }
+ /* add the new type */
+ storefs_list = static_cast<storefs_entry_t *>(xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t)));
+ memset(&storefs_list[i + 1], 0, sizeof(storefs_entry_t));
+ storefs_list[i].typestr = type;
+ /* Call the FS to set up capabilities and initialize the FS driver */
+ setup(&storefs_list[i]);
+}
+
+/*
+ * called to add another store removal policy module
+ */
+void
+storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
+{
+ int i;
+ /* find the number of currently known repl types */
+ for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
+ assert(strcmp(storerepl_list[i].typestr, type) != 0);
+ }
+ /* add the new type */
+ storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
+ memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
+ storerepl_list[i].typestr = type;
+ storerepl_list[i].create = create;
+}
+
+/*
+ * Create a removal policy instance
+ */
+RemovalPolicy *
+createRemovalPolicy(RemovalPolicySettings * settings)
+{
+ storerepl_entry_t *r;
+ for (r = storerepl_list; r && r->typestr; r++) {
+ if (strcmp(r->typestr, settings->type) == 0)
+ return r->create(settings->args);
+ }
+ debug(20, 1) ("ERROR: Unknown policy %s\n", settings->type);
+ debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n");
+ debug(20, 1) ("ERROR: and memory_replacement_policy in squid.conf!\n");
+ fatalf("ERROR: Unknown policy %s\n", settings->type);
+ return NULL; /* NOTREACHED */
}
-#endif
+#if 0
void
storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
{
storeDirLRUAdd(e);
}
}
+#endif
+
+/* Replace a store entry with
+ * a new reply. This eats the reply.
+ */
+void
+storeEntryReplaceObject(StoreEntry * e, HttpReply * rep)
+{
+ MemObject * const mem = e->mem_obj;
+ HttpReply *myrep;
+ Packer p;
+ debug(20, 3) ("storeEntryReplaceObject: %s\n", storeUrl(e));
+ if (!mem) {
+ debug (20,0)("Attempt to replace object with no in-memory representation\n");
+ return;
+ }
+ /* TODO: check that there is at most 1 store client ? */
+ myrep = (HttpReply *)e->getReply(); /* we are allowed to do this */
+ /* move info to the mem_obj->reply */
+ httpReplyAbsorb(myrep, rep);
+
+ /* TODO: when we store headers serparately remove the header portion */
+ /* TODO: mark the length of the headers ? */
+ /* We ONLY want the headers */
+ packerToStoreInit(&p, e);
+ assert (e->isEmpty());
+ httpReplyPackHeadersInto(e->getReply(), &p);
+ myrep->hdr_sz = e->mem_obj->endOffset();
+ httpBodyPackInto(&e->getReply()->body, &p);
+ packerClean(&p);
+}
+
+
+char const *
+StoreEntry::getSerialisedMetaData()
+{
+ StoreMeta *tlv_list = storeSwapMetaBuild(this);
+ int swap_hdr_sz;
+ char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
+ storeSwapTLVFree(tlv_list);
+ assert (swap_hdr_sz >= 0);
+ mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz;
+ return result;
+}
+
+bool
+StoreEntry::swapoutPossible()
+{
+ /* should we swap something out to disk? */
+ debug(20, 7) ("storeSwapOut: %s\n", storeUrl(this));
+ debug(20, 7) ("storeSwapOut: store_status = %s\n",
+ storeStatusStr[store_status]);
+ if (EBIT_TEST(flags, ENTRY_ABORTED)) {
+ assert(EBIT_TEST(flags, RELEASE_REQUEST));
+ storeSwapOutFileClose(this);
+ return false;
+ }
+ if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
+ debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(this));
+ return false;
+ }
+ return true;
+}
+
+void
+StoreEntry::trimMemory()
+{
+ if (mem_obj->policyLowestOffsetToKeep() == 0)
+ /* Nothing to do */
+ return;
+ assert (mem_obj->policyLowestOffsetToKeep() > 0);
+ if (!storeSwapOutAble(this)) {
+ /*
+ * Its not swap-able, and we're about to delete a chunk,
+ * so we must make it PRIVATE. This is tricky/ugly because
+ * for the most part, we treat swapable == cachable here.
+ */
+ storeReleaseRequest(this);
+ mem_obj->trimUnSwappable ();
+ } else {
+ mem_obj->trimSwappable ();
+ }
+}
+/* NullStoreEntry */
+
+NullStoreEntry NullStoreEntry::_instance;
+
+NullStoreEntry *
+NullStoreEntry::getInstance()
+{
+ return &_instance;
+}
+
+char const *
+NullStoreEntry::getMD5Text() const
+{
+ return "N/A";
+}
+
+char const *
+NullStoreEntry::getSerialisedMetaData()
+{
+ return NULL;
+}
+
+#ifndef _USE_INLINE_
+#include "Store.cci"
+#endif
projects / thirdparty / squid.git / blobdiff