/*
- * $Id: store.cc,v 1.462 1998/09/19 17:06:12 wessels Exp $
+ * $Id: store.cc,v 1.554 2003/01/23 00:37:26 robertc Exp $
*
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
*
- * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * SQUID Web Proxy Cache http://www.squid-cache.org/
* ----------------------------------------------------------
*
- * Squid is the result of efforts by numerous individuals from the
- * Internet community. Development is led by Duane Wessels of the
- * National Laboratory for Applied Network Research and funded by the
- * National Science Foundation. Squid is Copyrighted (C) 1998 by
- * Duane Wessels and the University of California San Diego. Please
- * see the COPYRIGHT file for full details. Squid incorporates
- * software developed and/or copyrighted by other sources. Please see
- * the CREDITS file for full details.
+ * Squid is the result of efforts by numerous individuals from
+ * the Internet community; see the CONTRIBUTORS file for full
+ * details. Many organizations have provided support for Squid's
+ * development; see the SPONSORS file for full details. Squid is
+ * Copyrighted (C) 2001 by the Regents of the University of
+ * California; see the COPYRIGHT file for full details. Squid
+ * incorporates software developed and/or copyrighted by other
+ * sources; see the CREDITS file for full details.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
*/
#include "squid.h"
+#include "Store.h"
+#include "StoreClient.h"
+#include "stmem.h"
+#include "HttpReply.h"
+#include "HttpRequest.h"
+#include "MemObject.h"
+#include "mem_node.h"
+#include "StoreMeta.h"
+#include "SwapDir.h"
+
+static STMCB storeWriteComplete;
#define REBUILD_TIMESTAMP_DELTA_MAX 2
{
"PING_NONE",
"PING_WAITING",
- "PING_TIMEOUT",
"PING_DONE"
};
const char *storeStatusStr[] =
{
"STORE_OK",
- "STORE_PENDING",
- "STORE_ABORTED"
+ "STORE_PENDING"
};
const char *swapStatusStr[] =
{
"SWAPOUT_NONE",
- "SWAPOUT_OPENING",
"SWAPOUT_WRITING",
"SWAPOUT_DONE"
};
StoreEntry *e;
} lock_ctrl_t;
+extern OBJH storeIOStats;
+
/*
* local function prototypes
*/
-static int storeCheckExpired(const StoreEntry *);
-static int storeEntryLocked(const StoreEntry *);
-static int storeEntryValidLength(const StoreEntry *);
static void storeGetMemSpace(int);
static void storeHashDelete(StoreEntry *);
-static MemObject *new_MemObject(const char *, const char *);
static void destroy_MemObject(StoreEntry *);
-static FREE destroy_StoreEntry;
+static FREE destroyStoreEntry;
static void storePurgeMem(StoreEntry *);
+static void storeEntryReferenced(StoreEntry *);
+static void storeEntryDereferenced(StoreEntry *);
static int getKeyCounter(void);
static int storeKeepInMemory(const StoreEntry *);
static OBJH storeCheckCachableStats;
+static EVH storeLateRelease;
/*
* local variables
*/
-static dlink_list inmem_list;
-static int store_pages_high = 0;
-static int store_pages_low = 0;
-static int store_swap_high = 0;
-static int store_swap_low = 0;
-static int store_swap_mid = 0;
-static int store_maintain_rate;
-
-static MemObject *
-new_MemObject(const char *url, const char *log_url)
-{
- MemObject *mem = memAllocate(MEM_MEMOBJECT);
- mem->reply = httpReplyCreate();
- mem->url = xstrdup(url);
- mem->log_url = xstrdup(log_url);
- mem->swapout.fd = -1;
- mem->object_sz = -1;
- mem->fd = -1;
- /* XXX account log_url */
- debug(20, 3) ("new_MemObject: returning %p\n", mem);
- return mem;
+static Stack LateReleaseStack;
+MemPool *StoreEntry::pool = NULL;
+
+void *
+StoreEntry::operator new (size_t bytecount)
+{
+ assert (bytecount == sizeof (StoreEntry));
+ if (!pool) {
+ pool = memPoolCreate ("StoreEntry", bytecount);
+ memPoolSetChunkSize(pool, 2048 * 1024);
+ }
+ return memPoolAlloc (pool);
+}
+
+void
+StoreEntry::operator delete (void *address)
+{
+ memPoolFree(pool, address);
+}
+
+size_t
+StoreEntry::inUseCount()
+{
+ if (!pool)
+ return 0;
+ MemPoolStats stats;
+ memPoolGetStats (&stats, pool);
+ return stats.items_inuse;
+}
+
+const char *
+StoreEntry::getMD5Text() const
+{
+ return storeKeyText((const cache_key *)key);
+}
+
+int
+StoreEntry::CheckDeferRead(int fd, void *data)
+{
+ assert (data);
+ return ((StoreEntry *)data)->checkDeferRead(fd);
+}
+
+int
+StoreEntry::checkDeferRead(int fd) const
+{
+ int rc = 0;
+ if (mem_obj == NULL)
+ return 0;
+#if URL_CHECKSUM_DEBUG
+ mem_obj->checkUrlChecksum();
+#endif
+#if DELAY_POOLS
+ if (fd < 0)
+ (void) 0;
+ else if (! delayIsNoDelay(fd)) {
+ int i = delayMostBytesWanted(mem_obj, INT_MAX);
+ if (0 == i)
+ return 1;
+ /* was: rc = -(rc != INT_MAX); */
+ else if (INT_MAX == i)
+ rc = 0;
+ else
+ rc = -1;
+ }
+#endif
+ if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT))
+ return rc;
+ if (mem_obj->readAheadPolicyCanRead())
+ return rc;
+ return 1;
+}
+
+store_client_t
+StoreEntry::storeClientType() const
+{
+ if (mem_obj->inmem_lo)
+ return STORE_DISK_CLIENT;
+ if (EBIT_TEST(flags, ENTRY_ABORTED)) {
+ /* I don't think we should be adding clients to aborted entries */
+ debug(20, 1) ("storeClientType: adding to ENTRY_ABORTED entry\n");
+ return STORE_MEM_CLIENT;
+ }
+ if (store_status == STORE_OK) {
+ if (mem_obj->inmem_lo == 0 && !isEmpty())
+ return STORE_MEM_CLIENT;
+ else
+ return STORE_DISK_CLIENT;
+ }
+ /* here and past, entry is STORE_PENDING */
+ /*
+ * If this is the first client, let it be the mem client
+ */
+ if (mem_obj->nclients == 1)
+ return STORE_MEM_CLIENT;
+ /*
+ * If there is no disk file to open yet, we must make this a
+ * mem client. If we can't open the swapin file before writing
+ * to the client, there is no guarantee that we will be able
+ * to open it later when we really need it.
+ */
+ if (swap_status == SWAPOUT_NONE)
+ return STORE_MEM_CLIENT;
+ /*
+ * otherwise, make subsequent clients read from disk so they
+ * can not delay the first, and vice-versa.
+ */
+ return STORE_DISK_CLIENT;
}
StoreEntry *
new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url)
{
StoreEntry *e = NULL;
- e = memAllocate(MEM_STOREENTRY);
+ e = new StoreEntry;
if (mem_obj_flag)
- e->mem_obj = new_MemObject(url, log_url);
- debug(20, 3) ("new_StoreEntry: returning %p\n", e);
+ e->mem_obj = new MemObject(url, log_url);
+ debug(20, 3) ("newStoreEntry: returning %p\n", e);
e->expires = e->lastmod = e->lastref = e->timestamp = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
return e;
}
static void
destroy_MemObject(StoreEntry * e)
{
+ storeSetMemStatus(e, NOT_IN_MEMORY);
MemObject *mem = e->mem_obj;
- const Ctx ctx = ctx_enter(mem->url);
- debug(20, 3) ("destroy_MemObject: destroying %p\n", mem);
e->mem_obj = NULL;
- if (!shutting_down)
- assert(mem->swapout.fd == -1);
- stmemFree(&mem->data_hdr);
- mem->inmem_hi = 0;
- /* XXX account log_url */
-#if USE_ASYNC_IO
- while (mem->clients != NULL)
- storeUnregister(e, mem->clients->callback_data);
-#endif
- /*
- * There is no way to abort FD-less clients, so they might
- * still have mem->clients set if mem->fd == -1
- */
- assert(mem->fd == -1 || mem->clients == NULL);
- httpReplyDestroy(mem->reply);
- requestUnlink(mem->request);
- mem->request = NULL;
- ctx_exit(ctx); /* must exit before we free mem->url */
- safe_free(mem->url);
- safe_free(mem->log_url);
- memFree(MEM_MEMOBJECT, mem);
+ delete mem;
}
static void
-destroy_StoreEntry(void *data)
+destroyStoreEntry(void *data)
{
- StoreEntry *e = data;
- debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e);
+ StoreEntry *e = static_cast<StoreEntry *>(data);
+ debug(20, 3) ("destroyStoreEntry: destroying %p\n", e);
assert(e != NULL);
- if (e->mem_obj)
- destroy_MemObject(e);
+ if (e == NullStoreEntry::getInstance())
+ return;
+ destroy_MemObject(e);
storeHashDelete(e);
assert(e->key == NULL);
- memFree(MEM_STOREENTRY, e);
+ delete e;
}
/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n",
e, storeKeyText(key));
e->key = storeKeyDup(key);
- hash_join(store_table, (hash_link *) e);
- dlinkAdd(e, &e->lru, &store_list);
+ hash_join(store_table, e);
}
static void
storeHashDelete(StoreEntry * e)
{
- hash_remove_link(store_table, (hash_link *) e);
- dlinkDelete(&e->lru, &store_list);
- storeKeyFree(e->key);
+ hash_remove_link(store_table, e);
+ storeKeyFree((const cache_key *)e->key);
e->key = NULL;
}
/* get rid of memory copy of the object */
-/* Only call this if storeCheckPurgeMem(e) returns 1 */
static void
storePurgeMem(StoreEntry * e)
{
if (e->mem_obj == NULL)
return;
debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n",
- storeKeyText(e->key));
- storeSetMemStatus(e, NOT_IN_MEMORY);
+ e->getMD5Text());
destroy_MemObject(e);
if (e->swap_status != SWAPOUT_DONE)
storeRelease(e);
}
+static void
+storeEntryReferenced(StoreEntry * e)
+{
+ /* Notify the fs that we're referencing this object again */
+ if (e->swap_dirn > -1)
+ INDEXSD(e->swap_dirn)->reference(*e);
+ /* Notify the memory cache that we're referencing this object again */
+ if (e->mem_obj) {
+ if (mem_policy->Referenced)
+ mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+
+static void
+storeEntryDereferenced(StoreEntry * e)
+{
+ /* Notify the fs that we're not referencing this object any more */
+ if (e->swap_filen > -1)
+ INDEXSD(e->swap_dirn)->dereference(*e);
+ /* Notify the memory cache that we're not referencing this object any more */
+ if (e->mem_obj) {
+ if (mem_policy->Dereferenced)
+ mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl);
+ }
+}
+
void
storeLockObject(StoreEntry * e)
{
- if (e->lock_count++ == 0) {
- dlinkDelete(&e->lru, &store_list);
- dlinkAdd(e, &e->lru, &store_list);
- }
+ e->lock_count++;
debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
- storeKeyText(e->key), (int) e->lock_count);
+ e->getMD5Text(), (int) e->lock_count);
e->lastref = squid_curtime;
+ storeEntryReferenced(e);
}
void
{
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
return;
- assert(storeEntryLocked(e));
- debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->key));
+ debug(20, 3) ("storeReleaseRequest: '%s'\n", e->getMD5Text());
EBIT_SET(e->flags, RELEASE_REQUEST);
/*
* Clear cachable flag here because we might get called before
{
e->lock_count--;
debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
- storeKeyText(e->key), e->lock_count);
+ e->getMD5Text(), e->lock_count);
if (e->lock_count)
return (int) e->lock_count;
- if (e->store_status == STORE_PENDING) {
- assert(!EBIT_TEST(e->flags, ENTRY_DISPATCHED));
+ if (e->store_status == STORE_PENDING)
EBIT_SET(e->flags, RELEASE_REQUEST);
- }
assert(storePendingNClients(e) == 0);
if (EBIT_TEST(e->flags, RELEASE_REQUEST))
storeRelease(e);
else if (storeKeepInMemory(e)) {
+ storeEntryDereferenced(e);
storeSetMemStatus(e, IN_MEMORY);
- requestUnlink(e->mem_obj->request);
- e->mem_obj->request = NULL;
+ e->mem_obj->unlinkRequest();
} else {
storePurgeMem(e);
- if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
- dlinkDelete(&e->lru, &store_list);
- dlinkAddTail(e, &e->lru, &store_list);
- }
+ storeEntryDereferenced(e);
+ if (EBIT_TEST(e->flags, KEY_PRIVATE))
+ debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__);
}
return 0;
}
-/* Lookup an object in the cache.
+/* Lookup an object in the cache.
* return just a reference to object, don't start swapping in yet. */
StoreEntry *
storeGet(const cache_key * key)
{
+ PROF_start(storeGet);
debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key));
- return (StoreEntry *) hash_lookup(store_table, key);
+ StoreEntry *p = static_cast<StoreEntry *>(hash_lookup(store_table, key));
+ PROF_stop(storeGet);
+ return p;
+}
+
+void
+StoreEntry::getPublicByRequestMethod (StoreClient *aClient, request_t * request, const method_t method)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublicByRequestMethod( request, method);
+ if (!result)
+ aClient->created (NullStoreEntry::getInstance());
+ else
+ aClient->created (result);
+}
+
+void
+StoreEntry::getPublicByRequest (StoreClient *aClient, request_t * request)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublicByRequest (request);
+ if (!result)
+ result = NullStoreEntry::getInstance();
+ aClient->created (result);
+}
+
+void
+StoreEntry::getPublic (StoreClient *aClient, const char *uri, const method_t method)
+{
+ assert (aClient);
+ StoreEntry *result = storeGetPublic (uri, method);
+ if (!result)
+ result = NullStoreEntry::getInstance();
+ aClient->created (result);
+}
+
+StoreEntry *
+storeGetPublic(const char *uri, const method_t method)
+{
+ return storeGet(storeKeyPublic(uri, method));
+}
+
+StoreEntry *
+storeGetPublicByRequestMethod(request_t * req, const method_t method)
+{
+ return storeGet(storeKeyPublicByRequestMethod(req, method));
+}
+
+StoreEntry *
+storeGetPublicByRequest(request_t * req)
+{
+ StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
+ if (e == NULL && req->method == METHOD_HEAD)
+ /* We can generate a HEAD reply from a cached GET object */
+ e = storeGetPublicByRequestMethod(req, METHOD_GET);
+ return e;
}
static int
if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE))
return; /* is already private */
if (e->key) {
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_DEL);
storeHashDelete(e);
}
* the object. If we're not swapping out, then subsequent
* store clients won't be able to access object data which has
* been freed from memory.
+ *
+ * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
+ * be set, and storeSetPublicKey() should not be called.
*/
+#if MORE_DEBUG_OUTPUT
+ if (EBIT_TEST(e->flags, RELEASE_REQUEST))
+ debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n",
+ e->key, mem->url);
+#endif
assert(!EBIT_TEST(e->flags, RELEASE_REQUEST));
- newkey = storeKeyPublic(mem->url, mem->method);
+ if (mem->request) {
+ StoreEntry *pe;
+ request_t *request = mem->request;
+ if (!mem->vary_headers) {
+ /* First handle the case where the object no longer varies */
+ safe_free(request->vary_headers);
+ } else {
+ if (request->vary_headers && strcmp(request->vary_headers, mem->vary_headers) != 0) {
+ /* Oops.. the variance has changed. Kill the base object
+ * to record the new variance key
+ */
+ safe_free(request->vary_headers); /* free old "bad" variance key */
+ pe = storeGetPublic(mem->url, mem->method);
+ if (pe)
+ storeRelease(pe);
+ }
+ /* Make sure the request knows the variance status */
+ if (!request->vary_headers) {
+ const char *vary = httpMakeVaryMark(request, mem->getReply());
+ if (vary)
+ request->vary_headers = xstrdup(vary);
+ }
+ }
+ if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) {
+ /* Create "vary" base object */
+ http_version_t version;
+ String vary;
+ pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method);
+ httpBuildVersion(&version, 1, 0);
+ /* We are allowed to do this typecast */
+ httpReplySetHeaders((HttpReply *)pe->getReply(), version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
+ vary = httpHeaderGetList(&mem->getReply()->header, HDR_VARY);
+ if (vary.size()) {
+ /* Again, we own this structure layout */
+ httpHeaderPutStr((HttpHeader *)&pe->getReply()->header, HDR_VARY, vary.buf());
+ vary.clean();
+ }
+#if X_ACCELERATOR_VARY
+ vary = httpHeaderGetList(&mem->getReply()->header, HDR_X_ACCELERATOR_VARY);
+ if (vary.buf()) {
+ httpHeaderPutStr(&pe->getReply()->header, HDR_X_ACCELERATOR_VARY, vary.buf());
+ vary.clean();
+ }
+#endif
+ storeSetPublicKey(pe);
+ /* TODO: remove this when the metadata is separated */
+ {
+ Packer p;
+ packerToStoreInit(&p, pe);
+ httpReplyPackHeadersInto(pe->getReply(), &p);
+ packerClean(&p);
+ }
+ storeBufferFlush(pe);
+ storeTimestampsSet(pe);
+ pe->complete();
+ storeUnlockObject(pe);
+ }
+ newkey = storeKeyPublicByRequest(mem->request);
+ } else
+ newkey = storeKeyPublic(mem->url, mem->method);
if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url);
storeSetPrivateKey(e2);
storeRelease(e2);
- newkey = storeKeyPublic(mem->url, mem->method);
+ if (mem->request)
+ newkey = storeKeyPublicByRequest(mem->request);
+ else
+ newkey = storeKeyPublic(mem->url, mem->method);
}
if (e->key)
storeHashDelete(e);
EBIT_CLR(e->flags, KEY_PRIVATE);
storeHashInsert(e, newkey);
- if (e->swap_file_number > -1)
+ if (e->swap_filen > -1)
storeDirSwapLog(e, SWAP_LOG_ADD);
}
e->store_status = STORE_PENDING;
storeSetMemStatus(e, NOT_IN_MEMORY);
e->swap_status = SWAPOUT_NONE;
- e->swap_file_number = -1;
+ e->swap_filen = -1;
+ e->swap_dirn = -1;
e->refcount = 0;
e->lastref = squid_curtime;
- e->timestamp = 0; /* set in storeTimestampsSet() */
+ e->timestamp = -1; /* set in storeTimestampsSet() */
e->ping_status = PING_NONE;
EBIT_SET(e->flags, ENTRY_VALIDATED);
-#ifdef PPNR_WIP
- EBIT_SET(e->flags, ENTRY_FWD_HDR_WAIT);
-#endif /* PPNR_WIP */
return e;
}
void
storeExpireNow(StoreEntry * e)
{
- debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->key));
+ debug(20, 3) ("storeExpireNow: '%s'\n", e->getMD5Text());
e->expires = squid_curtime;
}
+void
+storeWriteComplete (void *data, StoreIOBuffer wroteBuffer)
+{
+ StoreEntry *e = (StoreEntry *)data;
+ if (EBIT_TEST(e->flags, DELAY_SENDING))
+ return;
+ InvokeHandlers(e);
+}
+
+void
+StoreEntry::write (StoreIOBuffer writeBuffer)
+{
+ assert(mem_obj != NULL);
+ assert(writeBuffer.length >= 0);
+ /* This assert will change when we teach the store to update */
+ assert(store_status == STORE_PENDING);
+ if (!writeBuffer.length)
+ return;
+
+ debug(20, 5) ("storeWrite: writing %u bytes for '%s'\n",
+ writeBuffer.length, getMD5Text());
+ storeGetMemSpace(writeBuffer.length);
+ mem_obj->write (writeBuffer, storeWriteComplete, this);
+}
+
/* Append incoming data from a primary server to an entry. */
void
storeAppend(StoreEntry * e, const char *buf, int len)
MemObject *mem = e->mem_obj;
assert(mem != NULL);
assert(len >= 0);
- if (len) {
- debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
- len,
- storeKeyText(e->key));
- storeGetMemSpace(len);
- stmemAppend(&mem->data_hdr, buf, len);
- mem->inmem_hi += len;
- }
- if (EBIT_TEST(e->flags, DELAY_SENDING))
- return;
-#ifdef OPTIMISTIC_IO
- storeLockObject(e);
-#endif
- InvokeHandlers(e);
- storeCheckSwapOut(e);
-#ifdef OPTIMISTIC_IO
- storeUnlockObject(e);
-#endif
+ assert(e->store_status == STORE_PENDING);
+
+ StoreIOBuffer tempBuffer;
+ tempBuffer.data = (char *)buf;
+ tempBuffer.length = len;
+ tempBuffer.offset = mem->endOffset() - (e->getReply() ? e->getReply()->hdr_sz : 0);
+ e->write(tempBuffer);
}
-#ifdef __STDC__
void
+#if STDC_HEADERS
storeAppendPrintf(StoreEntry * e, const char *fmt,...)
-{
- va_list args;
- va_start(args, fmt);
#else
-void
storeAppendPrintf(va_alist)
va_dcl
+#endif
{
+#if STDC_HEADERS
+ va_list args;
+ va_start(args, fmt);
+#else
va_list args;
StoreEntry *e = NULL;
const char *fmt = NULL;
int wrong_content_length;
int negative_cached;
int too_big;
+ int too_small;
int private_key;
int too_many_open_files;
- int lru_age_too_low;
+ int too_many_open_fds;
} no;
struct {
int Default;
{
if (Config.max_open_disk_fds == 0)
return 0;
- if (open_disk_fd > Config.max_open_disk_fds)
+ if (store_open_disk_fd > Config.max_open_disk_fds)
return 1;
return 0;
}
+static int
+storeCheckTooSmall(StoreEntry * e)
+{
+ MemObject * const mem = e->mem_obj;
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+ return 0;
+ if (STORE_OK == e->store_status)
+ if (mem->object_sz < 0 ||
+ static_cast<size_t>(mem->object_sz) < Config.Store.minObjectSize)
+ return 1;
+ if (e->getReply()->content_length > -1)
+ if (e->getReply()->content_length < (int) Config.Store.minObjectSize)
+ return 1;
+ return 0;
+}
+
int
storeCheckCachable(StoreEntry * e)
{
debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
store_check_cachable_hist.no.negative_cached++;
return 0; /* avoid release call below */
- } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
+ } else if ((e->getReply()->content_length > 0 &&
+ static_cast<size_t>(e->getReply()->content_length) > Config.Store.maxObjectSize) ||
+ static_cast<size_t>(e->mem_obj->endOffset()) > Config.Store.maxObjectSize) {
+ debug(20, 2) ("storeCheckCachable: NO: too big\n");
+ store_check_cachable_hist.no.too_big++;
+ } else if (e->getReply()->content_length > (int) Config.Store.maxObjectSize) {
debug(20, 2) ("storeCheckCachable: NO: too big\n");
store_check_cachable_hist.no.too_big++;
+ } else if (storeCheckTooSmall(e)) {
+ debug(20, 2) ("storeCheckCachable: NO: too small\n");
+ store_check_cachable_hist.no.too_small++;
} else if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
debug(20, 3) ("storeCheckCachable: NO: private key\n");
store_check_cachable_hist.no.private_key++;
+ } else if (e->swap_status != SWAPOUT_NONE) {
+ /*
+ * here we checked the swap_status because the remaining
+ * cases are only relevant only if we haven't started swapping
+ * out the object yet.
+ */
+ return 1;
} else if (storeTooManyDiskFilesOpen()) {
debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n");
store_check_cachable_hist.no.too_many_open_files++;
- } else if (storeExpiredReferenceAge() < 300) {
- debug(20, 2) ("storeCheckCachable: NO: LRU Age = %d\n",
- storeExpiredReferenceAge());
- store_check_cachable_hist.no.lru_age_too_low++;
+ } else if (fdNFree() < RESERVED_FD) {
+ debug(20, 2) ("storeCheckCachable: NO: too many FD's open\n");
+ store_check_cachable_hist.no.too_many_open_fds++;
} else {
store_check_cachable_hist.yes.Default++;
return 1;
static void
storeCheckCachableStats(StoreEntry * sentry)
{
+ storeAppendPrintf(sentry, "Category\t Count\n");
+
storeAppendPrintf(sentry, "no.non_get\t%d\n",
store_check_cachable_hist.no.non_get);
storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
store_check_cachable_hist.no.negative_cached);
storeAppendPrintf(sentry, "no.too_big\t%d\n",
store_check_cachable_hist.no.too_big);
+ storeAppendPrintf(sentry, "no.too_small\t%d\n",
+ store_check_cachable_hist.no.too_small);
storeAppendPrintf(sentry, "no.private_key\t%d\n",
store_check_cachable_hist.no.private_key);
storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
store_check_cachable_hist.no.too_many_open_files);
- storeAppendPrintf(sentry, "no.lru_age_too_low\t%d\n",
- store_check_cachable_hist.no.lru_age_too_low);
+ storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
+ store_check_cachable_hist.no.too_many_open_fds);
storeAppendPrintf(sentry, "yes.default\t%d\n",
store_check_cachable_hist.yes.Default);
}
-/* Complete transfer into the local cache. */
void
-storeComplete(StoreEntry * e)
+StoreEntry::complete()
{
- debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key));
- assert(e->store_status == STORE_PENDING);
- e->mem_obj->object_sz = e->mem_obj->inmem_hi;
- e->store_status = STORE_OK;
- assert(e->mem_status == NOT_IN_MEMORY);
- if (!storeEntryValidLength(e))
- EBIT_SET(e->flags, ENTRY_BAD_LENGTH);
+ debug(20, 3) ("storeComplete: '%s'\n", getMD5Text());
+ if (store_status != STORE_PENDING) {
+ /*
+ * if we're not STORE_PENDING, then probably we got aborted
+ * and there should be NO clients on this entry
+ */
+ assert(EBIT_TEST(flags, ENTRY_ABORTED));
+ assert(mem_obj->nclients == 0);
+ return;
+ }
+ mem_obj->object_sz = mem_obj->endOffset();
+ store_status = STORE_OK;
+ assert(mem_status == NOT_IN_MEMORY);
+ if (!validLength()) {
+ EBIT_SET(flags, ENTRY_BAD_LENGTH);
+ storeReleaseRequest(this);
+ }
#if USE_CACHE_DIGESTS
- if (e->mem_obj->request)
- e->mem_obj->request->hier.store_complete_stop = current_time;
+ if (mem_obj->request)
+ mem_obj->request->hier.store_complete_stop = current_time;
#endif
- InvokeHandlers(e);
- storeCheckSwapOut(e);
-}
-
-#ifdef PPNR_WIP
-void
-storePPNR(StoreEntry * e)
-{
- assert(EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT));
- EBIT_CLR(e->flags, ENTRY_FWD_HDR_WAIT);
+ /*
+ * We used to call InvokeHandlers, then storeSwapOut. However,
+ * Madhukar Reddy <myreddy@persistence.com> reported that
+ * responses without content length would sometimes get released
+ * in client_side, thinking that the response is incomplete.
+ */
+ InvokeHandlers(this);
}
-#endif /* PPNR_WIP */
-
/*
* Someone wants to abort this transfer. Set the reason in the
* request structure, call the server-side callback and mark the
- * entry for releasing
+ * entry for releasing
*/
void
-storeAbort(StoreEntry * e, int cbflag)
+storeAbort(StoreEntry * e)
{
+ statCounter.aborted_requests++;
MemObject *mem = e->mem_obj;
- STABH *callback;
- void *data;
assert(e->store_status == STORE_PENDING);
assert(mem != NULL);
- debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->key));
+ debug(20, 6) ("storeAbort: %s\n", e->getMD5Text());
storeLockObject(e); /* lock while aborting */
storeNegativeCache(e);
storeReleaseRequest(e);
- e->store_status = STORE_ABORTED;
+ EBIT_SET(e->flags, ENTRY_ABORTED);
storeSetMemStatus(e, NOT_IN_MEMORY);
- /* No DISK swap for negative cached object */
- e->swap_status = SWAPOUT_NONE;
+ e->store_status = STORE_OK;
/*
* We assign an object length here. The only other place we assign
* the object length is in storeComplete()
*/
- mem->object_sz = mem->inmem_hi;
+ /* RBC: What do we need an object length for? we've just aborted the
+ * request, the request is private and negatively cached. Surely
+ * the object length is inappropriate to set.
+ */
+ mem->object_sz = mem->endOffset();
/* Notify the server side */
- if (cbflag && mem->abort.callback) {
- callback = mem->abort.callback;
- data = mem->abort.data;
+ if (mem->abort.callback) {
+ eventAdd("mem->abort.callback",
+ mem->abort.callback,
+ mem->abort.data,
+ 0.0,
+ 0);
mem->abort.callback = NULL;
mem->abort.data = NULL;
- callback(data);
}
+ /* XXX Should we reverse these two, so that there is no
+ * unneeded disk swapping triggered?
+ */
/* Notify the client side */
InvokeHandlers(e);
- /* Do we need to close the swapout file? */
- /* Not if we never started swapping out */
- /* But we may need to cancel an open/stat in progress if using ASYNC */
-#if USE_ASYNC_IO
- aioCancel(-1, e);
-#endif
- if (e->swap_file_number > -1) {
-#if USE_ASYNC_IO
- /* Need to cancel any pending ASYNC writes right now */
- if (mem->swapout.fd >= 0)
- aioCancel(mem->swapout.fd, NULL);
-#endif
- /* we have to close the disk file if there is no write pending */
- if (!storeSwapOutWriteQueued(mem))
- storeSwapOutFileClose(e);
- }
+ /* Close any swapout file */
+ storeSwapOutFileClose(e);
storeUnlockObject(e); /* unlock */
}
StoreEntry *e = NULL;
int released = 0;
static time_t last_check = 0;
- int pages_needed;
- dlink_node *m;
- dlink_node *head;
- dlink_node *prev = NULL;
+ size_t pages_needed;
+ RemovalPurgeWalker *walker;
if (squid_curtime == last_check)
return;
last_check = squid_curtime;
pages_needed = (size / SM_PAGE_SIZE) + 1;
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_high)
- return;
- if (store_rebuilding)
+ if (mem_node::InUseCount() + pages_needed < store_pages_max)
return;
debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed);
- head = inmem_list.head;
- for (m = inmem_list.tail; m; m = prev) {
- if (m == head)
- break;
- prev = m->prev;
- e = m->data;
- if (storeEntryLocked(e)) {
- dlinkDelete(m, &inmem_list);
- dlinkAdd(e, m, &inmem_list);
- continue;
- }
- released++;
+ /* XXX what to set as max_scan here? */
+ walker = mem_policy->PurgeInit(mem_policy, 100000);
+ while ((e = walker->Next(walker))) {
storePurgeMem(e);
- if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_high)
+ released++;
+ if (mem_node::InUseCount() + pages_needed < store_pages_max)
break;
}
+ walker->Done(walker);
debug(20, 3) ("storeGetMemSpace stats:\n");
debug(20, 3) (" %6d HOT objects\n", hot_obj_count);
debug(20, 3) (" %6d were released\n", released);
#define MAINTAIN_MAX_SCAN 1024
#define MAINTAIN_MAX_REMOVE 64
-/*
+/*
* This routine is to be called by main loop in main.c.
* It removes expired objects on only one bucket for each time called.
- * returns the number of objects removed
*
* This should get called 1/s from main().
*/
void
storeMaintainSwapSpace(void *datanotused)
{
- dlink_node *m;
- dlink_node *prev = NULL;
- StoreEntry *e = NULL;
- int scanned = 0;
- int locked = 0;
- int expired = 0;
- int max_scan;
- int max_remove;
+ int i;
+ SwapDir *SD;
static time_t last_warn_time = 0;
- /* We can't delete objects while rebuilding swap */
- if (store_rebuilding) {
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
- return;
- } else if (store_swap_size < store_swap_mid) {
- max_scan = 100;
- max_remove = 8;
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
- } else if (store_swap_size < store_swap_high) {
- max_scan = 200;
- max_remove = 8;
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.1, 1);
- } else {
- max_scan = 500;
- max_remove = 32;
- eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.0, 1);
+
+ PROF_start(storeMaintainSwapSpace);
+ /* walk each fs */
+ for (i = 0; i < Config.cacheSwap.n_configured; i++) {
+ /* call the maintain function .. */
+ SD = INDEXSD(i);
+ /* XXX FixMe: This should be done "in parallell" on the different
+ * cache_dirs, not one at a time.
+ */
+ SD->maintainfs();
}
- debug(20, 3) ("storeMaintainSwapSpace\n");
- for (m = store_list.tail; m; m = prev) {
- prev = m->prev;
- e = m->data;
- scanned++;
- if (storeEntryLocked(e)) {
- /*
- * If there is a locked entry at the tail of the LRU list,
- * move it to the beginning to get it out of the way.
- * Theoretically, we might have all locked objects at the
- * tail, and then we'll never remove anything here and the
- * LRU age will go to zero.
- */
- if (memInUse(MEM_STOREENTRY) > max_scan) {
- dlinkDelete(&e->lru, &store_list);
- dlinkAdd(e, &e->lru, &store_list);
- }
- locked++;
- } else if (storeCheckExpired(e)) {
- expired++;
- storeRelease(e);
+ if (store_swap_size > Config.Swap.maxSize) {
+ if (squid_curtime - last_warn_time > 10) {
+ debug(20, 0) ("WARNING: Disk space over limit: %ld KB > %ld KB\n",
+ (long int) store_swap_size, (long int) Config.Swap.maxSize);
+ last_warn_time = squid_curtime;
}
- if (expired >= max_remove)
- break;
- if (scanned >= max_scan)
- break;
}
- debug(20, 3) ("storeMaintainSwapSpace stats:\n");
- debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
- debug(20, 3) (" %6d were scanned\n", scanned);
- debug(20, 3) (" %6d were locked\n", locked);
- debug(20, 3) (" %6d were expired\n", expired);
- if (store_swap_size < Config.Swap.maxSize)
- return;
- if (squid_curtime - last_warn_time < 10)
- return;
- debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
- store_swap_size, Config.Swap.maxSize);
- last_warn_time = squid_curtime;
+ /* Reregister a maintain event .. */
+ eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
+ PROF_stop(storeMaintainSwapSpace);
}
/* release an object from a cache */
-/* return number of objects released. */
void
storeRelease(StoreEntry * e)
{
- debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->key));
+ PROF_start(storeRelease);
+ debug(20, 3) ("storeRelease: Releasing: '%s'\n", e->getMD5Text());
/* If, for any reason we can't discard this object because of an
* outstanding request, mark it for pending release */
if (storeEntryLocked(e)) {
storeExpireNow(e);
debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n");
storeReleaseRequest(e);
+ PROF_stop(storeRelease);
return;
}
- if (store_rebuilding) {
- debug(20, 2) ("storeRelease: Delaying release until store is rebuilt: '%s'\n",
- storeUrl(e));
- storeExpireNow(e);
+ if (store_dirs_rebuilding && e->swap_filen > -1) {
storeSetPrivateKey(e);
- EBIT_SET(e->flags, RELEASE_REQUEST);
- return;
+ if (e->mem_obj)
+ destroy_MemObject(e);
+ if (e->swap_filen > -1) {
+ /*
+ * Fake a call to storeLockObject(). When rebuilding is done,
+ * we'll just call storeUnlockObject() on these.
+ */
+ e->lock_count++;
+ EBIT_SET(e->flags, RELEASE_REQUEST);
+ stackPush(&LateReleaseStack, e);
+ PROF_stop(storeRelease);
+ return;
+ } else {
+ destroyStoreEntry(e);
+ }
}
-#if USE_ASYNC_IO
- /*
- * Make sure all forgotten async ops are cancelled
- */
- aioCancel(-1, e);
-#endif
storeLog(STORE_LOG_RELEASE, e);
- if (e->swap_file_number > -1) {
- storeUnlinkFileno(e->swap_file_number);
- storeDirMapBitReset(e->swap_file_number);
+ if (e->swap_filen > -1) {
+ storeUnlink(e);
if (e->swap_status == SWAPOUT_DONE)
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1);
+ storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, -1);
if (!EBIT_TEST(e->flags, KEY_PRIVATE))
storeDirSwapLog(e, SWAP_LOG_DEL);
+#if 0
+ /* From 2.4. I think we do this in storeUnlink? */
+ storeSwapFileNumberSet(e, -1);
+#endif
}
storeSetMemStatus(e, NOT_IN_MEMORY);
- destroy_StoreEntry(e);
+ destroyStoreEntry(e);
+ PROF_stop(storeRelease);
+}
+
+static void
+storeLateRelease(void *unused)
+{
+ StoreEntry *e;
+ int i;
+ static int n = 0;
+ if (store_dirs_rebuilding) {
+ eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
+ return;
+ }
+ for (i = 0; i < 10; i++) {
+ e = static_cast<StoreEntry*>(stackPop(&LateReleaseStack));
+ if (e == NULL) {
+ /* done! */
+ debug(20, 1) ("storeLateRelease: released %d objects\n", n);
+ return;
+ }
+ storeUnlockObject(e);
+ n++;
+ }
+ eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
}
/* return 1 if a store entry is locked */
-static int
+int
storeEntryLocked(const StoreEntry * e)
{
if (e->lock_count)
return 1;
- if (e->swap_status == SWAPOUT_OPENING)
- return 1;
if (e->swap_status == SWAPOUT_WRITING)
return 1;
if (e->store_status == STORE_PENDING)
return 1;
+ /*
+ * SPECIAL, PUBLIC entries should be "locked"
+ */
if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
- return 1;
+ if (!EBIT_TEST(e->flags, KEY_PRIVATE))
+ return 1;
return 0;
}
-static int
-storeEntryValidLength(const StoreEntry * e)
+bool
+StoreEntry::validLength() const
{
int diff;
const HttpReply *reply;
- assert(e->mem_obj != NULL);
- reply = e->mem_obj->reply;
- debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key));
+ assert(mem_obj != NULL);
+ reply = getReply();
+ debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", getMD5Text());
debug(20, 5) ("storeEntryValidLength: object_len = %d\n",
- objectLen(e));
+ objectLen(this));
debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n",
reply->hdr_sz);
debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
reply->content_length);
if (reply->content_length < 0) {
debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
if (reply->hdr_sz == 0) {
debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
- if (e->mem_obj->method == METHOD_HEAD) {
+ if (mem_obj->method == METHOD_HEAD) {
debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n",
- storeKeyText(e->key));
+ getMD5Text());
return 1;
}
if (reply->sline.status == HTTP_NOT_MODIFIED)
return 1;
if (reply->sline.status == HTTP_NO_CONTENT)
return 1;
- diff = reply->hdr_sz + reply->content_length - objectLen(e);
+ diff = reply->hdr_sz + reply->content_length - objectLen(this);
if (diff == 0)
return 1;
debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
diff < 0 ? -diff : diff,
- diff < 0 ? "small" : "big",
- storeKeyText(e->key));
+ diff < 0 ? "big" : "small",
+ getMD5Text());
return 0;
}
static void
storeInitHashValues(void)
{
- int i;
+ long int i;
/* Calculate size of hash table (maximum currently 64k buckets). */
i = Config.Swap.maxSize / Config.Store.avgObjectSize;
- debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n",
- Config.Swap.maxSize, i);
+ debug(20, 1) ("Swap maxSize %ld KB, estimated %ld objects\n",
+ (long int) Config.Swap.maxSize, i);
i /= Config.Store.objectsPerBucket;
- debug(20, 1) ("Target number of buckets: %d\n", i);
+ debug(20, 1) ("Target number of buckets: %ld\n", i);
/* ideally the full scan period should be configurable, for the
* moment it remains at approximately 24 hours. */
store_hash_buckets = storeKeyHashBuckets(i);
- store_maintain_rate = 86400 / store_hash_buckets;
- assert(store_maintain_rate > 0);
- debug(20, 1) ("Using %d Store buckets, replacement runs every %d second%s\n",
- store_hash_buckets,
- store_maintain_rate,
- store_maintain_rate == 1 ? null_string : "s");
- debug(20, 1) ("Max Mem size: %d KB\n", Config.Mem.maxSize >> 10);
- debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize);
+ debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets);
+ debug(20, 1) ("Max Mem size: %ld KB\n", (long int) Config.memMaxSize >> 10);
+ debug(20, 1) ("Max Swap size: %ld KB\n", (long int) Config.Swap.maxSize);
}
void
storeInitHashValues();
store_table = hash_create(storeKeyHashCmp,
store_hash_buckets, storeKeyHashHash);
+ mem_policy = createRemovalPolicy(Config.memPolicy);
storeDigestInit();
storeLogOpen();
- if (storeVerifyCacheDirs() < 0) {
- xstrncpy(tmp_error_buf,
- "\tFailed to verify one of the swap directories, Check cache.log\n"
- "\tfor details. Run 'squid -z' to create swap directories\n"
- "\tif needed, or if running Squid for the first time.",
- ERROR_BUF_SZ);
- fatal(tmp_error_buf);
- }
- storeDirOpenSwapLogs();
- store_list.head = store_list.tail = NULL;
- inmem_list.head = inmem_list.tail = NULL;
+ stackInit(&LateReleaseStack);
+ eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
+ storeDirInit();
storeRebuildStart();
cachemgrRegister("storedir",
"Store Directory Stats",
cachemgrRegister("store_check_cachable_stats",
"storeCheckCachable() Stats",
storeCheckCachableStats, 0, 1);
+ cachemgrRegister("store_io",
+ "Store IO Interface Stats",
+ storeIOStats, 0, 1);
}
void
storeConfigure(void)
{
- int store_mem_high = 0;
- int store_mem_low = 0;
- store_mem_high = (long) (Config.Mem.maxSize / 100) *
- Config.Mem.highWaterMark;
- store_mem_low = (long) (Config.Mem.maxSize / 100) *
- Config.Mem.lowWaterMark;
-
store_swap_high = (long) (((float) Config.Swap.maxSize *
(float) Config.Swap.highWaterMark) / (float) 100);
store_swap_low = (long) (((float) Config.Swap.maxSize *
(float) Config.Swap.lowWaterMark) / (float) 100);
- store_swap_mid = (store_swap_high >> 1) + (store_swap_low >> 1);
-
- store_pages_high = store_mem_high / SM_PAGE_SIZE;
- store_pages_low = store_mem_low / SM_PAGE_SIZE;
+ store_pages_max = Config.memMaxSize / SM_PAGE_SIZE;
}
static int
return mem->inmem_lo == 0;
}
-static int
-storeCheckExpired(const StoreEntry * e)
+int
+storeCheckNegativeHit(StoreEntry * e)
{
- if (storeEntryLocked(e))
+ if (!EBIT_TEST(e->flags, ENTRY_NEGCACHED))
return 0;
- if (EBIT_TEST(e->flags, RELEASE_REQUEST))
- return 1;
- if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
- return 1;
- if (squid_curtime - e->lastref > storeExpiredReferenceAge())
- return 1;
- return 0;
-}
-
-/*
- * storeExpiredReferenceAge
- *
- * The LRU age is scaled exponentially between 1 minute and
- * Config.referenceAge , when store_swap_low < store_swap_size <
- * store_swap_high. This keeps store_swap_size within the low and high
- * water marks. If the cache is very busy then store_swap_size stays
- * closer to the low water mark, if it is not busy, then it will stay
- * near the high water mark. The LRU age value can be examined on the
- * cachemgr 'info' page.
- */
-time_t
-storeExpiredReferenceAge(void)
-{
- double x;
- double z;
- time_t age;
- x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low);
- x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
- z = pow((double) (Config.referenceAge / 60), x);
- age = (time_t) (z * 60.0);
- if (age < 60)
- age = 60;
- else if (age > 31536000)
- age = 31536000;
- return age;
+ if (e->expires <= squid_curtime)
+ return 0;
+ if (e->store_status != STORE_OK)
+ return 0;
+ return 1;
}
void
void
storeFreeMemory(void)
{
- hashFreeItems(store_table, destroy_StoreEntry);
+ hashFreeItems(store_table, destroyStoreEntry);
hashFreeMemory(store_table);
store_table = NULL;
+#if USE_CACHE_DIGESTS
if (store_digest)
cacheDigestDestroy(store_digest);
+#endif
store_digest = NULL;
}
if (EBIT_TEST(e->flags, ENTRY_NEGCACHED))
if (e->expires <= squid_curtime)
return 0;
- if (e->store_status == STORE_ABORTED)
+ if (EBIT_TEST(e->flags, ENTRY_ABORTED))
return 0;
return 1;
}
void
storeTimestampsSet(StoreEntry * entry)
{
- time_t served_date = -1;
- const HttpReply *reply = entry->mem_obj->reply;
- served_date = reply->date;
- if (served_date < 0)
+ const HttpReply *reply = entry->getReply();
+ time_t served_date = reply->date;
+ int age = httpHeaderGetInt(&reply->header, HDR_AGE);
+ /*
+ * The timestamp calculations below tries to mimic the properties
+ * of the age calculation in RFC2616 section 13.2.3. The implementaion
+ * isn't complete, and the most notable exception from the RFC is that
+ * this does not account for response_delay, but it probably does
+ * not matter much as this is calculated immediately when the headers
+ * are received, not when the whole response has been received.
+ */
+ /* make sure that 0 <= served_date <= squid_curtime */
+ if (served_date < 0 || served_date > squid_curtime)
served_date = squid_curtime;
+ /*
+ * Compensate with Age header if origin server clock is ahead
+ * of us and there is a cache in between us and the origin
+ * server. But DONT compensate if the age value is larger than
+ * squid_curtime because it results in a negative served_date.
+ */
+ if (age > squid_curtime - served_date)
+ if (squid_curtime > age)
+ served_date = squid_curtime - age;
entry->expires = reply->expires;
entry->lastmod = reply->last_modified;
entry->timestamp = served_date;
}
void
-storeMemObjectDump(MemObject * mem)
-{
- debug(20, 1) ("MemObject->data.head: %p\n",
- mem->data_hdr.head);
- debug(20, 1) ("MemObject->data.tail: %p\n",
- mem->data_hdr.tail);
- debug(20, 1) ("MemObject->data.origin_offset: %d\n",
- mem->data_hdr.origin_offset);
- debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
- (int) mem->start_ping.tv_sec,
- (int) mem->start_ping.tv_usec);
- debug(20, 1) ("MemObject->inmem_hi: %d\n",
- (int) mem->inmem_hi);
- debug(20, 1) ("MemObject->inmem_lo: %d\n",
- (int) mem->inmem_lo);
- debug(20, 1) ("MemObject->clients: %p\n",
- mem->clients);
- debug(20, 1) ("MemObject->nclients: %d\n",
- mem->nclients);
- debug(20, 1) ("MemObject->swapout.fd: %d\n",
- mem->swapout.fd);
- debug(20, 1) ("MemObject->reply: %p\n",
- mem->reply);
- debug(20, 1) ("MemObject->request: %p\n",
- mem->request);
- debug(20, 1) ("MemObject->log_url: %p %s\n",
- mem->log_url,
- checkNullString(mem->log_url));
-}
-
-void
-storeEntryDump(StoreEntry * e, int l)
+storeEntryDump(const StoreEntry * e, int l)
{
- debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->key));
+ debug(20, l) ("StoreEntry->key: %s\n", e->getMD5Text());
debug(20, l) ("StoreEntry->next: %p\n", e->next);
debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj);
debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp);
debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
- debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number);
+ debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn);
+ debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen);
debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count);
debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status);
debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status);
debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status);
}
-/* NOTE, this function assumes only two mem states */
+/*
+ * NOTE, this function assumes only two mem states
+ */
void
-storeSetMemStatus(StoreEntry * e, int new_status)
+storeSetMemStatus(StoreEntry * e, mem_status_t new_status)
{
MemObject *mem = e->mem_obj;
if (new_status == e->mem_status)
assert(mem != NULL);
if (new_status == IN_MEMORY) {
assert(mem->inmem_lo == 0);
- dlinkAdd(e, &mem->lru, &inmem_list);
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n",
+ mem->url);
+ } else {
+ mem_policy->Add(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n",
+ mem->url);
+ }
hot_obj_count++;
} else {
- dlinkDelete(&mem->lru, &inmem_list);
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
+ debug(20, 4) ("storeSetMemStatus: special entry %s\n",
+ mem->url);
+ } else {
+ mem_policy->Remove(mem_policy, e, &mem->repl);
+ debug(20, 4) ("storeSetMemStatus: removed mem node %s\n",
+ mem->url);
+ }
hot_obj_count--;
}
e->mem_status = new_status;
{
if (e->mem_obj)
return;
- e->mem_obj = new_MemObject(url, log_url);
+ e->mem_obj = new MemObject(url, log_url);
}
/* this just sets DELAY_SENDING */
{
EBIT_CLR(e->flags, DELAY_SENDING);
InvokeHandlers(e);
- storeCheckSwapOut(e);
-}
-
-void
-storeUnlinkFileno(int fileno)
-{
- debug(20, 5) ("storeUnlinkFileno: %08X\n", fileno);
-#if USE_ASYNC_IO
- safeunlink(storeSwapFullPath(fileno, NULL), 1);
-#else
- unlinkdUnlink(storeSwapFullPath(fileno, NULL));
-#endif
}
-int
+ssize_t
objectLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
contentLen(const StoreEntry * e)
{
assert(e->mem_obj != NULL);
- assert(e->mem_obj->reply != NULL);
- return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz;
+ assert(e->getReply() != NULL);
+ return objectLen(e) - e->getReply()->hdr_sz;
+
}
-HttpReply *
-storeEntryReply(StoreEntry * e)
+HttpReply const *
+StoreEntry::getReply () const
{
- if (NULL == e)
- return NULL;
- if (NULL == e->mem_obj)
+ if (NULL == mem_obj)
return NULL;
- return e->mem_obj->reply;
+ return mem_obj->getReply();
}
+
+void
+storeEntryReset(StoreEntry * e)
+{
+ MemObject *mem = e->mem_obj;
+ assert (mem);
+ debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e));
+ mem->reset();
+ httpReplyReset((HttpReply *)e->getReply());
+ e->expires = e->lastmod = e->timestamp = -1;
+}
+
+/*
+ * storeFsInit
+ *
+ * This routine calls the SETUP routine for each fs type.
+ * I don't know where the best place for this is, and I'm not going to shuffle
+ * around large chunks of code right now (that can be done once its working.)
+ */
+void
+storeFsInit(void)
+{
+ storeReplSetup();
+ storeFsSetup();
+}
+
+
+/*
+ * similar to above, but is called when a graceful shutdown is to occur
+ * of each fs module.
+ */
+void
+storeFsDone(void)
+{
+ int i = 0;
+
+ while (storefs_list[i].typestr != NULL) {
+ storefs_list[i].donefunc();
+ i++;
+ }
+}
+
+/*
+ * called to add another store fs module
+ * RBC: doesn't belong here. Move IT.
+ */
+void
+StoreEntry::FsAdd(const char *type, STSETUP * setup)
+{
+ int i;
+ /* find the number of currently known storefs types */
+ for (i = 0; storefs_list && storefs_list[i].typestr; i++) {
+ assert(strcmp(storefs_list[i].typestr, type) != 0);
+ }
+ /* add the new type */
+ storefs_list = static_cast<storefs_entry_t *>(xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t)));
+ memset(&storefs_list[i + 1], 0, sizeof(storefs_entry_t));
+ storefs_list[i].typestr = type;
+ /* Call the FS to set up capabilities and initialize the FS driver */
+ setup(&storefs_list[i]);
+}
+
+/*
+ * called to add another store removal policy module
+ */
+void
+storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
+{
+ int i;
+ /* find the number of currently known repl types */
+ for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
+ assert(strcmp(storerepl_list[i].typestr, type) != 0);
+ }
+ /* add the new type */
+ storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
+ memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
+ storerepl_list[i].typestr = type;
+ storerepl_list[i].create = create;
+}
+
+/*
+ * Create a removal policy instance
+ */
+RemovalPolicy *
+createRemovalPolicy(RemovalPolicySettings * settings)
+{
+ storerepl_entry_t *r;
+ for (r = storerepl_list; r && r->typestr; r++) {
+ if (strcmp(r->typestr, settings->type) == 0)
+ return r->create(settings->args);
+ }
+ debug(20, 1) ("ERROR: Unknown policy %s\n", settings->type);
+ debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n");
+ debug(20, 1) ("ERROR: and memory_replacement_policy in squid.conf!\n");
+ fatalf("ERROR: Unknown policy %s\n", settings->type);
+ return NULL; /* NOTREACHED */
+}
+
+#if 0
+void
+storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
+{
+ if (e->swap_file_number == filn)
+ return;
+ if (filn < 0) {
+ assert(-1 == filn);
+ storeDirMapBitReset(e->swap_file_number);
+ storeDirLRUDelete(e);
+ e->swap_file_number = -1;
+ } else {
+ assert(-1 == e->swap_file_number);
+ storeDirMapBitSet(e->swap_file_number = filn);
+ storeDirLRUAdd(e);
+ }
+}
+#endif
+
+/* Replace a store entry with
+ * a new reply. This eats the reply.
+ */
+void
+storeEntryReplaceObject(StoreEntry * e, HttpReply * rep)
+{
+ MemObject * const mem = e->mem_obj;
+ HttpReply *myrep;
+ Packer p;
+ debug(20, 3) ("storeEntryReplaceObject: %s\n", storeUrl(e));
+ if (!mem) {
+ debug (20,0)("Attempt to replace object with no in-memory representation\n");
+ return;
+ }
+ /* TODO: check that there is at most 1 store client ? */
+ myrep = (HttpReply *)e->getReply(); /* we are allowed to do this */
+ /* move info to the mem_obj->reply */
+ httpReplyAbsorb(myrep, rep);
+
+ /* TODO: when we store headers serparately remove the header portion */
+ /* TODO: mark the length of the headers ? */
+ /* We ONLY want the headers */
+ packerToStoreInit(&p, e);
+ assert (e->isEmpty());
+ httpReplyPackHeadersInto(e->getReply(), &p);
+ myrep->hdr_sz = e->mem_obj->endOffset();
+ httpBodyPackInto(&e->getReply()->body, &p);
+ packerClean(&p);
+}
+
+
+char const *
+StoreEntry::getSerialisedMetaData()
+{
+ StoreMeta *tlv_list = storeSwapMetaBuild(this);
+ int swap_hdr_sz;
+ char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
+ storeSwapTLVFree(tlv_list);
+ assert (swap_hdr_sz >= 0);
+ mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz;
+ return result;
+}
+
+bool
+StoreEntry::swapoutPossible()
+{
+ /* should we swap something out to disk? */
+ debug(20, 7) ("storeSwapOut: %s\n", storeUrl(this));
+ debug(20, 7) ("storeSwapOut: store_status = %s\n",
+ storeStatusStr[store_status]);
+ if (EBIT_TEST(flags, ENTRY_ABORTED)) {
+ assert(EBIT_TEST(flags, RELEASE_REQUEST));
+ storeSwapOutFileClose(this);
+ return false;
+ }
+ if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
+ debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(this));
+ return false;
+ }
+ return true;
+}
+
+void
+StoreEntry::trimMemory()
+{
+ if (mem_obj->policyLowestOffsetToKeep() == 0)
+ /* Nothing to do */
+ return;
+ assert (mem_obj->policyLowestOffsetToKeep() > 0);
+ if (!storeSwapOutAble(this)) {
+ /*
+ * Its not swap-able, and we're about to delete a chunk,
+ * so we must make it PRIVATE. This is tricky/ugly because
+ * for the most part, we treat swapable == cachable here.
+ */
+ storeReleaseRequest(this);
+ mem_obj->trimUnSwappable ();
+ } else {
+ mem_obj->trimSwappable ();
+ }
+}
+/* NullStoreEntry */
+
+NullStoreEntry NullStoreEntry::_instance;
+
+NullStoreEntry *
+NullStoreEntry::getInstance()
+{
+ return &_instance;
+}
+
+char const *
+NullStoreEntry::getMD5Text() const
+{
+ return "N/A";
+}
+
+char const *
+NullStoreEntry::getSerialisedMetaData()
+{
+ return NULL;
+}
+
+#ifndef _USE_INLINE_
+#include "Store.cci"
+#endif
projects / thirdparty / squid.git / blobdiff