/*
- * $Id$
- *
* DEBUG: section 20 Storage Manager
* AUTHOR: Harvest Derived
*
*/
#include "squid.h"
+#include "CacheDigest.h"
#include "CacheManager.h"
#include "comm/Connection.h"
#include "ETag.h"
#include "event.h"
#include "fde.h"
-#include "Store.h"
-#include "mgr/Registration.h"
-#include "StoreClient.h"
-#include "stmem.h"
+#include "globals.h"
+#include "http.h"
#include "HttpReply.h"
#include "HttpRequest.h"
-#include "MemObject.h"
#include "mem_node.h"
+#include "MemObject.h"
+#include "mgr/Registration.h"
+#include "mgr/StoreIoAction.h"
+#include "profiler/Profiler.h"
+#include "repl_modules.h"
+#include "RequestFlags.h"
+#include "SquidConfig.h"
+#include "SquidTime.h"
+#include "Stack.h"
#include "StatCounters.h"
+#include "stmem.h"
+#include "Store.h"
+#include "store_digest.h"
+#include "store_key_md5.h"
+#include "store_key_md5.h"
+#include "store_log.h"
+#include "store_rebuild.h"
+#include "StoreClient.h"
+#include "StoreIOState.h"
#include "StoreMeta.h"
+#include "StrList.h"
+#include "swap_log_op.h"
#include "SwapDir.h"
-#include "StoreIOState.h"
+#include "tools.h"
#if USE_DELAY_POOLS
#include "DelayPools.h"
#endif
-#include "Stack.h"
-#include "SquidTime.h"
-#include "swap_log_op.h"
-#include "mgr/StoreIoAction.h"
+#if HAVE_LIMITS_H
+#include <limits.h>
+#endif
static STMCB storeWriteComplete;
#define STORE_IN_MEM_BUCKETS (229)
-
/** \todo Convert these string constants to enum string-arrays generated */
const char *memStatusStr[] = {
"SWAPOUT_DONE"
};
-
/*
* This defines an repl type
*/
static storerepl_entry_t *storerepl_list = NULL;
-
/*
* local function prototypes
*/
}
size_t
-StoreEntry::bytesWanted (Range<size_t> const aRange) const
+StoreEntry::bytesWanted (Range<size_t> const aRange, bool ignoreDelayPools) const
{
- assert (aRange.size());
-
if (mem_obj == NULL)
- return aRange.end - 1;
+ return aRange.end;
#if URL_CHECKSUM_DEBUG
#endif
- /* Always read *something* here - we haven't got the header yet */
- if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT))
- return aRange.end - 1;
-
if (!mem_obj->readAheadPolicyCanRead())
return 0;
- return mem_obj->mostBytesWanted(aRange.end - 1);
+ return mem_obj->mostBytesWanted(aRange.end, ignoreDelayPools);
}
bool
if (EBIT_TEST(flags, ENTRY_ABORTED)) {
/* I don't think we should be adding clients to aborted entries */
- debugs(20, 1, "storeClientType: adding to ENTRY_ABORTED entry");
+ debugs(20, DBG_IMPORTANT, "storeClientType: adding to ENTRY_ABORTED entry");
return STORE_MEM_CLIENT;
}
return STORE_DISK_CLIENT;
}
-StoreEntry::StoreEntry():
+StoreEntry::StoreEntry() :
+ mem_obj(NULL),
hidden_mem_obj(NULL),
- swap_file_sz(0)
+ timestamp(-1),
+ lastref(-1),
+ expires(-1),
+ lastmod(-1),
+ swap_file_sz(0),
+ refcount(0),
+ flags(0),
+ swap_filen(-1),
+ swap_dirn(-1),
+ lock_count(0),
+ mem_status(NOT_IN_MEMORY),
+ ping_status(PING_NONE),
+ store_status(STORE_PENDING),
+ swap_status(SWAPOUT_NONE)
{
debugs(20, 3, HERE << "new StoreEntry " << this);
- mem_obj = NULL;
-
- expires = lastmod = lastref = timestamp = -1;
-
- swap_status = SWAPOUT_NONE;
- swap_filen = -1;
- swap_dirn = -1;
}
-StoreEntry::StoreEntry(const char *aUrl, const char *aLogUrl):
+StoreEntry::StoreEntry(const char *aUrl, const char *aLogUrl) :
+ mem_obj(NULL),
hidden_mem_obj(NULL),
- swap_file_sz(0)
+ timestamp(-1),
+ lastref(-1),
+ expires(-1),
+ lastmod(-1),
+ swap_file_sz(0),
+ refcount(0),
+ flags(0),
+ swap_filen(-1),
+ swap_dirn(-1),
+ lock_count(0),
+ mem_status(NOT_IN_MEMORY),
+ ping_status(PING_NONE),
+ store_status(STORE_PENDING),
+ swap_status(SWAPOUT_NONE)
{
debugs(20, 3, HERE << "new StoreEntry " << this);
mem_obj = new MemObject(aUrl, aLogUrl);
-
- expires = lastmod = lastref = timestamp = -1;
-
- swap_status = SWAPOUT_NONE;
- swap_filen = -1;
- swap_dirn = -1;
}
StoreEntry::~StoreEntry()
/* -------------------------------------------------------------------------- */
-
/* get rid of memory copy of the object */
void
StoreEntry::purgeMem()
StoreEntry::lock()
{
- lock_count++;
+ ++lock_count;
debugs(20, 3, "StoreEntry::lock: key '" << getMD5Text() <<"' count=" <<
lock_count );
lastref = squid_curtime;
int
StoreEntry::unlock()
{
- lock_count--;
+ --lock_count;
debugs(20, 3, "StoreEntry::unlock: key '" << getMD5Text() << "' count=" << lock_count);
if (lock_count)
}
if (EBIT_TEST(flags, KEY_PRIVATE))
- debugs(20, 1, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
+ debugs(20, DBG_IMPORTANT, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
Store::Root().handleIdleEntry(*this); // may delete us
return 0;
{
StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
- if (e == NULL && req->method == METHOD_HEAD)
+ if (e == NULL && req->method == Http::METHOD_HEAD)
/* We can generate a HEAD reply from a cached GET object */
- e = storeGetPublicByRequestMethod(req, METHOD_GET);
+ e = storeGetPublicByRequestMethod(req, Http::METHOD_GET);
return e;
}
mem_obj->id = getKeyCounter();
newkey = storeKeyPrivate(mem_obj->url, mem_obj->method, mem_obj->id);
} else {
- newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter());
+ newkey = storeKeyPrivate("JUNK", Http::METHOD_NONE, getKeyCounter());
}
assert(hash_lookup(store_table, newkey) == NULL);
#if MORE_DEBUG_OUTPUT
if (EBIT_TEST(flags, RELEASE_REQUEST))
- debugs(20, 1, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
+ debugs(20, DBG_IMPORTANT, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
#endif
StoreEntry *pe = storeCreateEntry(mem_obj->url, mem_obj->log_url, request->flags, request->method);
/* We are allowed to do this typecast */
HttpReply *rep = new HttpReply;
- rep->setHeaders(HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
+ rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
vary = mem_obj->getReply()->header.getList(HDR_VARY);
if (vary.size()) {
}
StoreEntry *
-storeCreateEntry(const char *url, const char *log_url, request_flags flags, const HttpRequestMethod& method)
+storeCreateEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method)
{
StoreEntry *e = NULL;
MemObject *mem = NULL;
write(tempBuffer);
}
-
void
storeAppendPrintf(StoreEntry * e, const char *fmt,...)
{
{
#if CACHE_ALL_METHODS
- if (mem_obj->method != METHOD_GET) {
+ if (mem_obj->method != Http::METHOD_GET) {
debugs(20, 2, "StoreEntry::checkCachable: NO: non-GET method");
- store_check_cachable_hist.no.non_get++;
+ ++store_check_cachable_hist.no.non_get;
} else
#endif
if (store_status == STORE_OK && EBIT_TEST(flags, ENTRY_BAD_LENGTH)) {
debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
- store_check_cachable_hist.no.wrong_content_length++;
+ ++store_check_cachable_hist.no.wrong_content_length;
} else if (!EBIT_TEST(flags, ENTRY_CACHABLE)) {
debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
- store_check_cachable_hist.no.not_entry_cachable++;
+ ++store_check_cachable_hist.no.not_entry_cachable;
} else if (EBIT_TEST(flags, ENTRY_NEGCACHED)) {
debugs(20, 3, "StoreEntry::checkCachable: NO: negative cached");
- store_check_cachable_hist.no.negative_cached++;
+ ++store_check_cachable_hist.no.negative_cached;
return 0; /* avoid release call below */
} else if ((getReply()->content_length > 0 &&
- getReply()->content_length
- > Config.Store.maxObjectSize) ||
- mem_obj->endOffset() > Config.Store.maxObjectSize) {
- debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
- store_check_cachable_hist.no.too_big++;
- } else if (getReply()->content_length > Config.Store.maxObjectSize) {
+ getReply()->content_length > store_maxobjsize) ||
+ mem_obj->endOffset() > store_maxobjsize) {
debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
- store_check_cachable_hist.no.too_big++;
+ ++store_check_cachable_hist.no.too_big;
} else if (checkTooSmall()) {
debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
- store_check_cachable_hist.no.too_small++;
+ ++store_check_cachable_hist.no.too_small;
} else if (EBIT_TEST(flags, KEY_PRIVATE)) {
debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
- store_check_cachable_hist.no.private_key++;
+ ++store_check_cachable_hist.no.private_key;
} else if (swap_status != SWAPOUT_NONE) {
/*
* here we checked the swap_status because the remaining
return 1;
} else if (storeTooManyDiskFilesOpen()) {
debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
- store_check_cachable_hist.no.too_many_open_files++;
+ ++store_check_cachable_hist.no.too_many_open_files;
} else if (fdNFree() < RESERVED_FD) {
debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
- store_check_cachable_hist.no.too_many_open_fds++;
+ ++store_check_cachable_hist.no.too_many_open_fds;
} else {
- store_check_cachable_hist.yes.Default++;
+ ++store_check_cachable_hist.yes.Default;
return 1;
}
void
StoreEntry::abort()
{
- statCounter.aborted_requests++;
+ ++statCounter.aborted_requests;
assert(store_status == STORE_PENDING);
assert(mem_obj != NULL);
debugs(20, 6, "storeAbort: " << getMD5Text());
*/
if (mem_obj->abort.callback) {
if (!cbdataReferenceValid(mem_obj->abort.data))
- debugs(20,1,HERE << "queueing event when abort.data is not valid");
+ debugs(20, DBG_IMPORTANT,HERE << "queueing event when abort.data is not valid");
eventAdd("mem_obj->abort.callback",
mem_obj->abort.callback,
mem_obj->abort.data,
while ((e = walker->Next(walker))) {
e->purgeMem();
- released++;
+ ++released;
if (mem_node::InUseCount() + pages_needed < store_pages_max)
break;
PROF_stop(storeGetMemSpace);
}
-
/* thunk through to Store::Root().maintain(). Note that this would be better still
* if registered against the root store itself, but that requires more complex
* update logic - bigger fish to fry first. Long term each store when
* Fake a call to StoreEntry->lock() When rebuilding is done,
* we'll just call StoreEntry->unlock() on these.
*/
- lock_count++;
+ ++lock_count;
setReleaseFlag();
LateReleaseStack.push_back(this);
} else {
return;
}
- for (i = 0; i < 10; i++) {
+ for (i = 0; i < 10; ++i) {
e = LateReleaseStack.count ? LateReleaseStack.pop() : NULL;
if (e == NULL) {
/* done! */
- debugs(20, 1, "storeLateRelease: released " << n << " objects");
+ debugs(20, DBG_IMPORTANT, "storeLateRelease: released " << n << " objects");
return;
}
e->unlock();
- n++;
+ ++n;
}
eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
return 1;
}
- if (mem_obj->method == METHOD_HEAD) {
+ if (mem_obj->method == Http::METHOD_HEAD) {
debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
return 1;
}
- if (reply->sline.status == HTTP_NOT_MODIFIED)
+ if (reply->sline.status() == Http::scNotModified)
return 1;
- if (reply->sline.status == HTTP_NO_CONTENT)
+ if (reply->sline.status() == Http::scNoContent)
return 1;
diff = reply->hdr_sz + reply->content_length - objectLen();
if (!Config.onoff.memory_cache_first && swap_status == SWAPOUT_DONE && refcount == 1)
return 0;
- if (Config.memShared && IamWorkerProcess()) {
- const int64_t expectedSize = mem_obj->expectedReplySize();
- // objects of unknown size are not allowed into memory cache, for now
- if (expectedSize < 0 ||
- expectedSize > static_cast<int64_t>(Config.Store.maxInMemObjSize))
- return 0;
- }
-
return 1;
}
// are we using a shared memory cache?
if (Config.memShared && IamWorkerProcess()) {
- assert(new_status != IN_MEMORY); // we do not call this otherwise
+ // enumerate calling cases if shared memory is enabled
+ assert(new_status != IN_MEMORY || EBIT_TEST(flags, ENTRY_SPECIAL));
// This method was designed to update replacement policy, not to
// actually purge something from the memory cache (TODO: rename?).
// Shared memory cache does not have a policy that needs updates.
debugs(20, 4, "StoreEntry::setMemStatus: inserted mem node " << mem_obj->url << " key: " << getMD5Text());
}
- hot_obj_count++; // TODO: maintain for the shared hot cache as well
+ ++hot_obj_count; // TODO: maintain for the shared hot cache as well
} else {
if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
debugs(20, 4, "StoreEntry::setMemStatus: special entry " << mem_obj->url);
debugs(20, 4, "StoreEntry::setMemStatus: removed mem node " << mem_obj->url);
}
- hot_obj_count--;
+ --hot_obj_count;
}
mem_status = new_status;
void
StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl)
{
+ debugs(20, 3, "A mem_obj create attempted using : " << aUrl);
+
if (mem_obj)
return;
int i;
/* find the number of currently known repl types */
- for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
+ for (i = 0; storerepl_list && storerepl_list[i].typestr; ++i) {
if (strcmp(storerepl_list[i].typestr, type) == 0) {
- debugs(20, 1, "WARNING: Trying to load store replacement policy " << type << " twice.");
+ debugs(20, DBG_IMPORTANT, "WARNING: Trying to load store replacement policy " << type << " twice.");
return;
}
}
{
storerepl_entry_t *r;
- for (r = storerepl_list; r && r->typestr; r++) {
+ for (r = storerepl_list; r && r->typestr; ++r) {
if (strcmp(r->typestr, settings->type) == 0)
return r->create(settings->args);
}
- debugs(20, 1, "ERROR: Unknown policy " << settings->type);
- debugs(20, 1, "ERROR: Be sure to have set cache_replacement_policy");
- debugs(20, 1, "ERROR: and memory_replacement_policy in squid.conf!");
+ debugs(20, DBG_IMPORTANT, "ERROR: Unknown policy " << settings->type);
+ debugs(20, DBG_IMPORTANT, "ERROR: Be sure to have set cache_replacement_policy");
+ debugs(20, DBG_IMPORTANT, "ERROR: and memory_replacement_policy in squid.conf!");
fatalf("ERROR: Unknown policy %s\n", settings->type);
return NULL; /* NOTREACHED */
}
#endif
-
/*
* Replace a store entry with
* a new reply. This eats the reply.
debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
if (!mem_obj) {
- debugs(20, 0, "Attempt to replace object with no in-memory representation");
+ debugs(20, DBG_CRITICAL, "Attempt to replace object with no in-memory representation");
return;
}
startWriting();
}
-
void
StoreEntry::startWriting()
{
packerClean(&p);
}
-
char const *
StoreEntry::getSerialisedMetaData()
{
return result;
}
-bool
-StoreEntry::swapoutPossible()
-{
- if (!Config.cacheSwap.n_configured)
- return false;
-
- /* should we swap something out to disk? */
- debugs(20, 7, "storeSwapOut: " << url());
- debugs(20, 7, "storeSwapOut: store_status = " << storeStatusStr[store_status]);
-
- assert(mem_obj);
- MemObject::SwapOut::Decision &decision = mem_obj->swapout.decision;
-
- // if we decided that swapout is not possible, do not repeat same checks
- if (decision == MemObject::SwapOut::swImpossible) {
- debugs(20, 3, "storeSwapOut: already rejected");
- return false;
- }
-
- // this flag may change so we must check it even if we already said "yes"
- if (EBIT_TEST(flags, ENTRY_ABORTED)) {
- assert(EBIT_TEST(flags, RELEASE_REQUEST));
- // StoreEntry::abort() already closed the swap out file, if any
- decision = MemObject::SwapOut::swImpossible;
- return false;
- }
-
- // if we decided that swapout is possible, do not repeat same checks
- if (decision == MemObject::SwapOut::swPossible) {
- debugs(20, 3, "storeSwapOut: already allowed");
- return true;
- }
-
- // if we are swapping out already, do not repeat same checks
- if (swap_status != SWAPOUT_NONE) {
- debugs(20, 3, "storeSwapOut: already started");
- decision = MemObject::SwapOut::swPossible;
- return true;
- }
-
- if (!checkCachable()) {
- debugs(20, 3, "storeSwapOut: not cachable");
- decision = MemObject::SwapOut::swImpossible;
- return false;
- }
-
- if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
- debugs(20, 3, "storeSwapOut: " << url() << " SPECIAL");
- decision = MemObject::SwapOut::swImpossible;
- return false;
- }
-
- // check cache_dir max-size limit if all cache_dirs have it
- if (store_maxobjsize >= 0) {
- // TODO: add estimated store metadata size to be conservative
-
- // use guaranteed maximum if it is known
- const int64_t expectedEnd = mem_obj->expectedReplySize();
- debugs(20, 7, "storeSwapOut: expectedEnd = " << expectedEnd);
- if (expectedEnd > store_maxobjsize) {
- debugs(20, 3, "storeSwapOut: will not fit: " << expectedEnd <<
- " > " << store_maxobjsize);
- decision = MemObject::SwapOut::swImpossible;
- return false; // known to outgrow the limit eventually
- }
-
- // use current minimum (always known)
- const int64_t currentEnd = mem_obj->endOffset();
- if (currentEnd > store_maxobjsize) {
- debugs(20, 3, "storeSwapOut: does not fit: " << currentEnd <<
- " > " << store_maxobjsize);
- decision = MemObject::SwapOut::swImpossible;
- return false; // already does not fit and may only get bigger
- }
-
- // prevent default swPossible answer for yet unknown length
- if (expectedEnd < 0) {
- debugs(20, 3, "storeSwapOut: wait for more info: " <<
- store_maxobjsize);
- return false; // may fit later, but will be rejected now
- }
- }
-
- decision = MemObject::SwapOut::swPossible;
- return true;
-}
-
void
-StoreEntry::trimMemory()
+StoreEntry::trimMemory(const bool preserveSwappable)
{
/*
* DPW 2007-05-09
if (mem_status == IN_MEMORY)
return;
- if (!swapOutAble()) {
+ if (EBIT_TEST(flags, ENTRY_SPECIAL))
+ return; // cannot trim because we do not load them again
+
+ if (!preserveSwappable) {
if (mem_obj->policyLowestOffsetToKeep(0) == 0) {
/* Nothing to do */
return;
}
}
+bool
+StoreEntry::hasEtag(ETag &etag) const
+{
+ if (const HttpReply *reply = getReply()) {
+ etag = reply->header.getETag(HDR_ETAG);
+ if (etag.str)
+ return true;
+ }
+ return false;
+}
+
bool
StoreEntry::hasIfMatchEtag(const HttpRequest &request) const
{
{
const String reqETags = request.header.getList(HDR_IF_NONE_MATCH);
// weak comparison is allowed only for HEAD or full-body GET requests
- const bool allowWeakMatch = !request.flags.range &&
- (request.method == METHOD_GET || request.method == METHOD_HEAD);
+ const bool allowWeakMatch = !request.flags.isRanged &&
+ (request.method == Http::METHOD_GET || request.method == Http::METHOD_HEAD);
return hasOneOfEtags(reqETags, allowWeakMatch);
}
projects / thirdparty / squid.git / blobdiff