-
/*
- * DEBUG: section 20 Storage Manager
- * AUTHOR: Harvest Derived
- *
- * SQUID Web Proxy Cache http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- * Squid is the result of efforts by numerous individuals from
- * the Internet community; see the CONTRIBUTORS file for full
- * details. Many organizations have provided support for Squid's
- * development; see the SPONSORS file for full details. Squid is
- * Copyrighted (C) 2001 by the Regents of the University of
- * California; see the COPYRIGHT file for full details. Squid
- * incorporates software developed and/or copyrighted by other
- * sources; see the CREDITS file for full details.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
*
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
*/
+/* DEBUG: section 20 Storage Manager */
+
#include "squid.h"
#include "CacheDigest.h"
#include "CacheManager.h"
#include "comm/Connection.h"
+#include "comm/Read.h"
#include "ETag.h"
#include "event.h"
#include "fde.h"
#include "RequestFlags.h"
#include "SquidConfig.h"
#include "SquidTime.h"
-#include "Stack.h"
#include "StatCounters.h"
#include "stmem.h"
#include "Store.h"
#if USE_DELAY_POOLS
#include "DelayPools.h"
#endif
-#if HAVE_LIMITS_H
-#include <limits.h>
-#endif
+
+/** StoreEntry uses explicit new/delete operators, which set pool chunk size to 2MB
+ * XXX: convert to MEMPROXY_CLASS() API
+ */
+#include "mem/Pool.h"
+
+#include <climits>
+#include <stack>
#define REBUILD_TIMESTAMP_DELTA_MAX 2
/*
* local variables
*/
-static Stack<StoreEntry*> LateReleaseStack;
+static std::stack<StoreEntry*> LateReleaseStack;
MemAllocator *StoreEntry::pool = NULL;
StorePointer Store::CurrentRoot = NULL;
void
Store::Stats(StoreEntry * output)
{
- assert (output);
+ assert(output);
Root().stat(*output);
}
{}
void
-Store::unlink (StoreEntry &anEntry)
+Store::unlink(StoreEntry &)
{
fatal("Store::unlink on invalid Store\n");
}
void *
StoreEntry::operator new (size_t bytecount)
{
- assert (bytecount == sizeof (StoreEntry));
+ assert(bytecount == sizeof (StoreEntry));
if (!pool) {
pool = memPoolCreate ("StoreEntry", bytecount);
}
bool
-StoreEntry::checkDeferRead(int fd) const
+StoreEntry::checkDeferRead(int) const
{
return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
}
void
-StoreEntry::setNoDelay (bool const newValue)
+StoreEntry::setNoDelay(bool const newValue)
{
if (mem_obj)
mem_obj->setNoDelay(newValue);
}
StoreEntry::StoreEntry() :
- mem_obj(NULL),
- timestamp(-1),
- lastref(-1),
- expires(-1),
- lastmod(-1),
- swap_file_sz(0),
- refcount(0),
- flags(0),
- swap_filen(-1),
- swap_dirn(-1),
- mem_status(NOT_IN_MEMORY),
- ping_status(PING_NONE),
- store_status(STORE_PENDING),
- swap_status(SWAPOUT_NONE),
- lock_count(0)
-{
- debugs(20, 3, HERE << "new StoreEntry " << this);
+ mem_obj(NULL),
+ timestamp(-1),
+ lastref(-1),
+ expires(-1),
+ lastmod(-1),
+ swap_file_sz(0),
+ refcount(0),
+ flags(0),
+ swap_filen(-1),
+ swap_dirn(-1),
+ mem_status(NOT_IN_MEMORY),
+ ping_status(PING_NONE),
+ store_status(STORE_PENDING),
+ swap_status(SWAPOUT_NONE),
+ lock_count(0)
+{
+ debugs(20, 5, "StoreEntry constructed, this=" << this);
}
StoreEntry::~StoreEntry()
{
+ debugs(20, 5, "StoreEntry destructed, this=" << this);
}
#if USE_ADAPTATION
}
void
-StoreEntry::touch() {
+StoreEntry::touch()
+{
lastref = squid_curtime;
Store::Root().reference(*this);
}
{
debugs(20, 3, (context ? context : "somebody") <<
" unlocking key " << getMD5Text() << ' ' << *this);
+ assert(lock_count > 0);
--lock_count;
if (lock_count)
int private_key;
int too_many_open_files;
int too_many_open_fds;
+ int missing_parts;
} no;
struct {
return 0;
if (STORE_OK == store_status)
- if (mem_obj->object_sz < 0 ||
+ if (mem_obj->object_sz >= 0 &&
mem_obj->object_sz < Config.Store.minObjectSize)
return 1;
if (getReply()->content_length > -1)
return 0;
}
-// TODO: remove checks already performed by swapoutPossible()
+bool
+StoreEntry::checkTooBig() const
+{
+ if (mem_obj->endOffset() > store_maxobjsize)
+ return true;
+
+ if (getReply()->content_length < 0)
+ return false;
+
+ return (getReply()->content_length > store_maxobjsize);
+}
+
// TODO: move "too many open..." checks outside -- we are called too early/late
-int
+bool
StoreEntry::checkCachable()
{
+ // XXX: This method is used for both memory and disk caches, but some
+ // checks are specific to disk caches. Move them to mayStartSwapOut().
+
+ // XXX: This method may be called several times, sometimes with different
+ // outcomes, making store_check_cachable_hist counters misleading.
+
+ // check this first to optimize handling of repeated calls for uncachables
+ if (EBIT_TEST(flags, RELEASE_REQUEST)) {
+ debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
+ ++store_check_cachable_hist.no.not_entry_cachable; // TODO: rename?
+ return 0; // avoid rerequesting release below
+ }
+
#if CACHE_ALL_METHODS
if (mem_obj->method != Http::METHOD_GET) {
if (store_status == STORE_OK && EBIT_TEST(flags, ENTRY_BAD_LENGTH)) {
debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
++store_check_cachable_hist.no.wrong_content_length;
- } else if (EBIT_TEST(flags, RELEASE_REQUEST)) {
- debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
- ++store_check_cachable_hist.no.not_entry_cachable; // TODO: rename?
} else if (EBIT_TEST(flags, ENTRY_NEGCACHED)) {
debugs(20, 3, "StoreEntry::checkCachable: NO: negative cached");
++store_check_cachable_hist.no.negative_cached;
return 0; /* avoid release call below */
- } else if ((getReply()->content_length > 0 &&
- getReply()->content_length > store_maxobjsize) ||
- mem_obj->endOffset() > store_maxobjsize) {
+ } else if (!mem_obj || !getReply()) {
+ // XXX: In bug 4131, we forgetHit() without mem_obj, so we need
+ // this segfault protection, but how can we get such a HIT?
+ debugs(20, 2, "StoreEntry::checkCachable: NO: missing parts: " << *this);
+ ++store_check_cachable_hist.no.missing_parts;
+ } else if (checkTooBig()) {
debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
++store_check_cachable_hist.no.too_big;
} else if (checkTooSmall()) {
store_check_cachable_hist.no.wrong_content_length);
storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
store_check_cachable_hist.no.negative_cached);
+ storeAppendPrintf(sentry, "no.missing_parts\t%d\n",
+ store_check_cachable_hist.no.missing_parts);
storeAppendPrintf(sentry, "no.too_big\t%d\n",
store_check_cachable_hist.no.too_big);
storeAppendPrintf(sentry, "no.too_small\t%d\n",
/*
* Someone wants to abort this transfer. Set the reason in the
- * request structure, call the server-side callback and mark the
+ * request structure, call the callback and mark the
* entry for releasing
*/
void
* it becomes active will self register
*/
void
-Store::Maintain(void *notused)
+Store::Maintain(void *)
{
Store::Root().maintain();
// lock the entry until rebuilding is done
lock("storeLateRelease");
setReleaseFlag();
- LateReleaseStack.push_back(this);
+ LateReleaseStack.push(this);
} else {
destroyStoreEntry(static_cast<hash_link *>(this));
// "this" is no longer valid
}
static void
-storeLateRelease(void *unused)
+storeLateRelease(void *)
{
StoreEntry *e;
- int i;
static int n = 0;
if (StoreController::store_dirs_rebuilding) {
return;
}
- for (i = 0; i < 10; ++i) {
- e = LateReleaseStack.count ? LateReleaseStack.pop() : NULL;
-
- if (e == NULL) {
- /* done! */
+ // TODO: this works but looks unelegant.
+ for (int i = 0; i < 10; ++i) {
+ if (LateReleaseStack.empty()) {
debugs(20, DBG_IMPORTANT, "storeLateRelease: released " << n << " objects");
return;
+ } else {
+ e = LateReleaseStack.top();
+ LateReleaseStack.pop();
}
e->unlock("storeLateRelease");
storeRegisterWithCacheManager();
}
+/// computes maximum size of a cachable object
+/// larger objects are rejected by all (disk and memory) cache stores
+static int64_t
+storeCalcMaxObjSize()
+{
+ int64_t ms = 0; // nothing can be cached without at least one store consent
+
+ // global maximum is at least the disk store maximum
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ assert (Config.cacheSwap.swapDirs[i].getRaw());
+ const int64_t storeMax = dynamic_cast<SwapDir *>(Config.cacheSwap.swapDirs[i].getRaw())->maxObjectSize();
+ if (ms < storeMax)
+ ms = storeMax;
+ }
+
+ // global maximum is at least the memory store maximum
+ // TODO: move this into a memory cache class when we have one
+ const int64_t memMax = static_cast<int64_t>(min(Config.Store.maxInMemObjSize, Config.memMaxSize));
+ if (ms < memMax)
+ ms = memMax;
+
+ return ms;
+}
+
void
storeConfigure(void)
{
store_swap_low = (long) (((float) Store::Root().maxSize() *
(float) Config.Swap.lowWaterMark) / (float) 100);
store_pages_max = Config.memMaxSize / sizeof(mem_node);
+
+ store_maxobjsize = storeCalcMaxObjSize();
}
bool
-StoreEntry::memoryCachable() const
+StoreEntry::memoryCachable()
{
+ if (!checkCachable())
+ return 0;
+
if (mem_obj == NULL)
return 0;
if (!mem_obj) // not backed by a memory cache and not collapsed
return 0;
- if (mem_obj->memCache.index >= 0) // backed by a shared memory cache
- return 0;
-
// StoreEntry::storeClientType() assumes DISK_CLIENT here, but there is no
- // disk cache backing so we should not rely on the store cache at all. This
- // is wrong for range requests that could feed off nibbled memory (XXX).
- if (mem_obj->inmem_lo) // in local memory cache, but got nibbled at
+ // disk cache backing that store_client constructor will assert. XXX: This
+ // is wrong for range requests (that could feed off nibbled memory) and for
+ // entries backed by the shared memory cache (that could, in theory, get
+ // nibbled bytes from that cache, but there is no such "memoryIn" code).
+ if (mem_obj->inmem_lo) // in memory cache, but got nibbled at
return 0;
+ // The following check is correct but useless at this position. TODO: Move
+ // it up when the shared memory cache can either replenish locally nibbled
+ // bytes or, better, does not use local RAM copy at all.
+ // if (mem_obj->memCache.index >= 0) // backed by a shared memory cache
+ // return 1;
+
return 1;
}
const char *
StoreEntry::url() const
{
- if (this == NULL)
- return "[null_entry]";
- else if (mem_obj == NULL)
+ if (mem_obj == NULL)
return "[null_mem_obj]";
else
return mem_obj->storeId();
#endif
+void
+StoreEntry::storeErrorResponse(HttpReply *reply)
+{
+ lock("StoreEntry::storeErrorResponse");
+ buffer();
+ replaceHttpReply(reply);
+ flush();
+ complete();
+ negativeCache();
+ releaseRequest();
+ unlock("StoreEntry::storeErrorResponse");
+}
+
/*
* Replace a store entry with
* a new reply. This eats the reply.
return result;
}
+/**
+ * Abandon the transient entry our worker has created if neither the shared
+ * memory cache nor the disk cache wants to store it. Collapsed requests, if
+ * any, should notice and use Plan B instead of getting stuck waiting for us
+ * to start swapping the entry out.
+ */
+void
+StoreEntry::transientsAbandonmentCheck()
+{
+ if (mem_obj && !mem_obj->smpCollapsed && // this worker is responsible
+ mem_obj->xitTable.index >= 0 && // other workers may be interested
+ mem_obj->memCache.index < 0 && // rejected by the shared memory cache
+ mem_obj->swapout.decision == MemObject::SwapOut::swImpossible) {
+ debugs(20, 7, "cannot be shared: " << *this);
+ if (!shutting_down) // Store::Root() is FATALly missing during shutdown
+ Store::Root().transientsAbandon(*this);
+ }
+}
+
+void
+StoreEntry::memOutDecision(const bool)
+{
+ transientsAbandonmentCheck();
+}
+
+void
+StoreEntry::swapOutDecision(const MemObject::SwapOut::Decision &decision)
+{
+ // Abandon our transient entry if neither shared memory nor disk wants it.
+ assert(mem_obj);
+ mem_obj->swapout.decision = decision;
+ transientsAbandonmentCheck();
+}
+
void
StoreEntry::trimMemory(const bool preserveSwappable)
{
// print only non-default status values, using unique letters
if (e.mem_status != NOT_IN_MEMORY ||
- e.store_status != STORE_PENDING ||
- e.swap_status != SWAPOUT_NONE ||
- e.ping_status != PING_NONE) {
+ e.store_status != STORE_PENDING ||
+ e.swap_status != SWAPOUT_NONE ||
+ e.ping_status != PING_NONE) {
if (e.mem_status != NOT_IN_MEMORY) os << 'm';
if (e.store_status != STORE_PENDING) os << 's';
if (e.swap_status != SWAPOUT_NONE) os << 'w' << e.swap_status;
return NULL;
}
-#if !_USE_INLINE_
-#include "Store.cci"
-#endif
projects / thirdparty / squid.git / blobdiff