/*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
*
* Squid software is distributed under GPLv2+ license and includes
* contributions from numerous individuals and organizations.
/* DEBUG: section 13 High Level Memory Pool Management */
#include "squid.h"
-#include "acl/AclDenyInfoList.h"
-#include "acl/AclNameList.h"
-#include "CacheDigest.h"
+#include "base/PackableStream.h"
#include "ClientInfo.h"
-#include "disk.h"
#include "dlink.h"
#include "event.h"
+#include "fs_io.h"
#include "icmp/net_db.h"
#include "md5.h"
#include "mem/forward.h"
+#include "mem/Meter.h"
#include "mem/Pool.h"
#include "MemBuf.h"
-#include "memMeter.h"
#include "mgr/Registration.h"
#include "SquidConfig.h"
-#include "SquidList.h"
#include "SquidTime.h"
#include "Store.h"
-#include "StoreEntryStream.h"
#include <iomanip>
-#include <ostream>
/* forward declarations */
static void memFree2K(void *);
static void memFree32K(void *);
static void memFree64K(void *);
-/* module globals */
-const size_t squidSystemPageSize=getpagesize();
-
/* local prototypes */
static void memStringStats(std::ostream &);
/* module locals */
-static MemAllocator *MemPools[MEM_MAX];
static double xm_time = 0;
static double xm_deltat = 0;
-/* all pools are ready to be used */
-static bool MemIsInitialized = false;
-
/* string pools */
#define mem_str_pool_count 6
-// 4 bytes bigger than the biggest string pool size
-// which is in turn calculated from SmallestStringBeforeMemIsInitialized
-static const size_t SmallestStringBeforeMemIsInitialized = 1024*16+4;
-
-static const struct {
+struct PoolMeta {
const char *name;
size_t obj_size;
-}
+};
+
+static Mem::Meter StrCountMeter;
+static Mem::Meter StrVolumeMeter;
+
+static Mem::Meter HugeBufCountMeter;
+static Mem::Meter HugeBufVolumeMeter;
-StrPoolsAttrs[mem_str_pool_count] = {
-
- {
- "Short Strings", MemAllocator::RoundedSize(36),
- }, /* to fit rfc1123 and similar */
- {
- "Medium Strings", MemAllocator::RoundedSize(128),
- }, /* to fit most urls */
- {
- "Long Strings", MemAllocator::RoundedSize(512),
- },
- {
- "1KB Strings", MemAllocator::RoundedSize(1024),
- },
- {
- "4KB Strings", MemAllocator::RoundedSize(4*1024),
- },
- {
- "16KB Strings",
- MemAllocator::RoundedSize(SmallestStringBeforeMemIsInitialized-4)
+/* local routines */
+
+// XXX: refactor objects using these pools to use MEMPROXY classes instead
+// then remove this function entirely
+static MemAllocator *&
+GetPool(size_t type)
+{
+ static MemAllocator *pools[MEM_MAX];
+ static bool initialized = false;
+
+ if (!initialized) {
+ memset(pools, '\0', sizeof(pools));
+ initialized = true;
+ // Mem::Init() makes use of GetPool(type) to initialize
+ // the actual pools. So must come after the flag is true
+ Mem::Init();
}
-};
-static struct {
- MemAllocator *pool;
+ return pools[type];
}
-StrPools[mem_str_pool_count];
-static MemMeter StrCountMeter;
-static MemMeter StrVolumeMeter;
+static MemAllocator &
+GetStrPool(size_t type)
+{
+ static MemAllocator *strPools[mem_str_pool_count];
+ static bool initialized = false;
-static MemMeter HugeBufCountMeter;
-static MemMeter HugeBufVolumeMeter;
+ static const PoolMeta PoolAttrs[mem_str_pool_count] = {
+ {"Short Strings", MemAllocator::RoundedSize(36)}, /* to fit rfc1123 and similar */
+ {"Medium Strings", MemAllocator::RoundedSize(128)}, /* to fit most urls */
+ {"Long Strings", MemAllocator::RoundedSize(512)},
+ {"1KB Strings", MemAllocator::RoundedSize(1024)},
+ {"4KB Strings", MemAllocator::RoundedSize(4*1024)},
+ {"16KB Strings", MemAllocator::RoundedSize(16*1024)}
+ };
-/* local routines */
+ if (!initialized) {
+ memset(strPools, '\0', sizeof(strPools));
+
+ /** Lastly init the string pools. */
+ for (int i = 0; i < mem_str_pool_count; ++i) {
+ strPools[i] = memPoolCreate(PoolAttrs[i].name, PoolAttrs[i].obj_size);
+ strPools[i]->zeroBlocks(false);
+
+ if (strPools[i]->objectSize() != PoolAttrs[i].obj_size)
+ debugs(13, DBG_IMPORTANT, "NOTICE: " << PoolAttrs[i].name <<
+ " is " << strPools[i]->objectSize() <<
+ " bytes instead of requested " <<
+ PoolAttrs[i].obj_size << " bytes");
+ }
+
+ initialized = true;
+ }
+
+ return *strPools[type];
+}
+
+/* Find the best fit string pool type */
+static mem_type
+memFindStringSizeType(size_t net_size, bool fuzzy)
+{
+ mem_type type = MEM_NONE;
+ for (unsigned int i = 0; i < mem_str_pool_count; ++i) {
+ auto &pool = GetStrPool(i);
+ if (fuzzy && net_size < pool.objectSize()) {
+ type = static_cast<mem_type>(i);
+ break;
+ } else if (net_size == pool.objectSize()) {
+ type = static_cast<mem_type>(i);
+ break;
+ }
+ }
+
+ return type;
+}
static void
memStringStats(std::ostream &stream)
/* table body */
for (i = 0; i < mem_str_pool_count; ++i) {
- const MemAllocator *pool = StrPools[i].pool;
- const int plevel = pool->getMeter().inuse.level;
- stream << std::setw(20) << std::left << pool->objectType();
- stream << std::right << "\t " << xpercentInt(plevel, StrCountMeter.level);
- stream << "\t " << xpercentInt(plevel * pool->objectSize(), StrVolumeMeter.level) << "\n";
+ const auto &pool = GetStrPool(i);
+ const auto plevel = pool.getMeter().inuse.currentLevel();
+ stream << std::setw(20) << std::left << pool.objectType();
+ stream << std::right << "\t " << xpercentInt(plevel, StrCountMeter.currentLevel());
+ stream << "\t " << xpercentInt(plevel * pool.objectSize(), StrVolumeMeter.currentLevel()) << "\n";
pooled_count += plevel;
- pooled_volume += plevel * pool->objectSize();
+ pooled_volume += plevel * pool.objectSize();
}
/* malloc strings */
stream << std::setw(20) << std::left << "Other Strings";
-
stream << std::right << "\t ";
-
- stream << xpercentInt(StrCountMeter.level - pooled_count, StrCountMeter.level) << "\t ";
-
- stream << xpercentInt(StrVolumeMeter.level - pooled_volume, StrVolumeMeter.level) << "\n\n";
+ stream << xpercentInt(StrCountMeter.currentLevel() - pooled_count, StrCountMeter.currentLevel()) << "\t ";
+ stream << xpercentInt(StrVolumeMeter.currentLevel() - pooled_volume, StrVolumeMeter.currentLevel()) << "\n\n";
}
static void
memBufStats(std::ostream & stream)
{
stream << "Large buffers: " <<
- HugeBufCountMeter.level << " (" <<
- HugeBufVolumeMeter.level / 1024 << " KB)\n";
+ HugeBufCountMeter.currentLevel() << " (" <<
+ HugeBufVolumeMeter.currentLevel() / 1024 << " KB)\n";
}
void
Mem::Stats(StoreEntry * sentry)
{
- StoreEntryStream stream(sentry);
+ PackableStream stream(*sentry);
Report(stream);
memStringStats(stream);
memBufStats(stream);
{
assert(name && size);
- if (MemPools[type] != NULL)
+ if (GetPool(type) != NULL)
return;
- MemPools[type] = memPoolCreate(name, size);
- MemPools[type]->zeroBlocks(doZero);
+ GetPool(type) = memPoolCreate(name, size);
+ GetPool(type)->zeroBlocks(doZero);
}
/* find appropriate pool and use it (pools always init buffer with 0s) */
void *
memAllocate(mem_type type)
{
- assert(MemPools[type]);
- return MemPools[type]->alloc();
+ assert(GetPool(type));
+ return GetPool(type)->alloc();
}
/* give memory back to the pool */
void
memFree(void *p, int type)
{
- assert(MemPools[type]);
- MemPools[type]->freeOne(p);
+ assert(GetPool(type));
+ GetPool(type)->freeOne(p);
}
/* allocate a variable size buffer using best-fit string pool */
void *
memAllocString(size_t net_size, size_t * gross_size)
{
- MemAllocator *pool = NULL;
assert(gross_size);
- // if pools are not yet ready, make sure that
- // the requested size is not poolable so that the right deallocator
- // will be used
- if (!MemIsInitialized && net_size < SmallestStringBeforeMemIsInitialized)
- net_size = SmallestStringBeforeMemIsInitialized;
-
- unsigned int i;
- for (i = 0; i < mem_str_pool_count; ++i) {
- if (net_size <= StrPoolsAttrs[i].obj_size) {
- pool = StrPools[i].pool;
- break;
- }
+ auto type = memFindStringSizeType(net_size, true);
+ if (type != MEM_NONE) {
+ auto &pool = GetStrPool(type);
+ *gross_size = pool.objectSize();
+ assert(*gross_size >= net_size);
+ ++StrCountMeter;
+ StrVolumeMeter += *gross_size;
+ return pool.alloc();
}
- *gross_size = pool ? StrPoolsAttrs[i].obj_size : net_size;
- assert(*gross_size >= net_size);
- // may forget [de]allocations until MemIsInitialized
- memMeterInc(StrCountMeter);
- memMeterAdd(StrVolumeMeter, *gross_size);
- return pool ? pool->alloc() : xcalloc(1, net_size);
+ *gross_size = net_size;
+ ++StrCountMeter;
+ StrVolumeMeter += *gross_size;
+ return xcalloc(1, net_size);
}
size_t
size_t result = 0;
for (int counter = 0; counter < mem_str_pool_count; ++counter)
- result += memPoolInUseCount(StrPools[counter].pool);
+ result += GetStrPool(counter).inUseCount();
return result;
}
void
memFreeString(size_t size, void *buf)
{
- MemAllocator *pool = NULL;
assert(buf);
- if (MemIsInitialized) {
- for (unsigned int i = 0; i < mem_str_pool_count; ++i) {
- if (size <= StrPoolsAttrs[i].obj_size) {
- assert(size == StrPoolsAttrs[i].obj_size);
- pool = StrPools[i].pool;
- break;
- }
- }
- }
+ auto type = memFindStringSizeType(size, false);
+ if (type != MEM_NONE)
+ GetStrPool(type).freeOne(buf);
+ else
+ xfree(buf);
- // may forget [de]allocations until MemIsInitialized
- memMeterDec(StrCountMeter);
- memMeterDel(StrVolumeMeter, size);
- pool ? pool->freeOne(buf) : xfree(buf);
+ --StrCountMeter;
+ StrVolumeMeter -= size;
}
/* Find the best fit MEM_X_BUF type */
if (type != MEM_NONE)
return memAllocate(type);
else {
- memMeterInc(HugeBufCountMeter);
- memMeterAdd(HugeBufVolumeMeter, *gross_size);
+ ++HugeBufCountMeter;
+ HugeBufVolumeMeter += *gross_size;
return xcalloc(1, net_size);
}
}
memFree(buf, type);
else {
xfree(buf);
- memMeterDec(HugeBufCountMeter);
- memMeterDel(HugeBufVolumeMeter, size);
+ --HugeBufCountMeter;
+ HugeBufVolumeMeter -= size;
}
}
MemPools::GetInstance().setIdleLimit(new_pool_limit);
}
-/* XXX make these classes do their own memory management */
-#include "HttpHdrContRange.h"
-
void
Mem::Init(void)
{
- int i;
+ /* all pools are ready to be used */
+ static bool MemIsInitialized = false;
+ if (MemIsInitialized)
+ return;
/** \par
* NOTE: Mem::Init() is called before the config file is parsed
* on stderr.
*/
- /** \par
- * Set all pointers to null. */
- memset(MemPools, '\0', sizeof(MemPools));
/**
* Then initialize all pools.
* \par
memDataInit(MEM_16K_BUF, "16K Buffer", 16384, 10, false);
memDataInit(MEM_32K_BUF, "32K Buffer", 32768, 10, false);
memDataInit(MEM_64K_BUF, "64K Buffer", 65536, 10, false);
- memDataInit(MEM_ACL_DENY_INFO_LIST, "AclDenyInfoList",
- sizeof(AclDenyInfoList), 0);
- memDataInit(MEM_ACL_NAME_LIST, "acl_name_list", sizeof(AclNameList), 0);
-#if USE_CACHE_DIGESTS
-
- memDataInit(MEM_CACHE_DIGEST, "CacheDigest", sizeof(CacheDigest), 0);
-#endif
-
- memDataInit(MEM_LINK_LIST, "link_list", sizeof(link_list), 10);
- memDataInit(MEM_DLINK_NODE, "dlink_node", sizeof(dlink_node), 10);
memDataInit(MEM_DREAD_CTRL, "dread_ctrl", sizeof(dread_ctrl), 0);
memDataInit(MEM_DWRITE_Q, "dwrite_q", sizeof(dwrite_q), 0);
- memDataInit(MEM_HTTP_HDR_CONTENT_RANGE, "HttpHdrContRange", sizeof(HttpHdrContRange), 0);
- memDataInit(MEM_NETDBENTRY, "netdbEntry", sizeof(netdbEntry), 0);
- memDataInit(MEM_NET_DB_NAME, "net_db_name", sizeof(net_db_name), 0);
- memDataInit(MEM_CLIENT_INFO, "ClientInfo", sizeof(ClientInfo), 0);
memDataInit(MEM_MD5_DIGEST, "MD5 digest", SQUID_MD5_DIGEST_LENGTH, 0);
- MemPools[MEM_MD5_DIGEST]->setChunkSize(512 * 1024);
-
- /** Lastly init the string pools. */
- for (i = 0; i < mem_str_pool_count; ++i) {
- StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size);
- StrPools[i].pool->zeroBlocks(false);
-
- if (StrPools[i].pool->objectSize() != StrPoolsAttrs[i].obj_size)
- debugs(13, DBG_IMPORTANT, "Notice: " << StrPoolsAttrs[i].name << " is " << StrPools[i].pool->objectSize() << " bytes instead of requested " << StrPoolsAttrs[i].obj_size << " bytes");
- }
+ GetPool(MEM_MD5_DIGEST)->setChunkSize(512 * 1024);
MemIsInitialized = true;
{
mem_type t = MEM_NONE;
- while (++t < MEM_DONTFREE) {
+ while (++t < MEM_MAX) {
/*
* If you hit this assertion, then you forgot to add a
* memDataInit() line for type 't'.
- * Or placed the pool type in the wrong section of the enum list.
*/
- assert(MemPools[t]);
+ assert(GetPool(t));
}
}
int
memInUse(mem_type type)
{
- return memPoolInUseCount(MemPools[type]);
+ return GetPool(type)->inUseCount();
}
/* ick */
return memFree64K;
default:
- memMeterDec(HugeBufCountMeter);
- memMeterDel(HugeBufVolumeMeter, size);
+ --HugeBufCountMeter;
+ HugeBufVolumeMeter -= size;
return cxx_xfree;
}
}
}
/*
* Fragmentation calculation:
- * needed = inuse.level / chunk_capacity
+ * needed = inuse.currentLevel() / chunk_capacity
* excess = used - needed
* fragmentation = excess / needed * 100%
*
*/
/* allocated */
stream << mp_st->items_alloc << delim;
- stream << toKB(mp_st->obj_size * pm->alloc.level) << delim;
- stream << toKB(mp_st->obj_size * pm->alloc.hwater_level) << delim;
- stream << std::setprecision(2) << ((squid_curtime - pm->alloc.hwater_stamp) / 3600.) << delim;
- stream << std::setprecision(3) << xpercent(mp_st->obj_size * pm->alloc.level, AllMeter->alloc.level) << delim;
+ stream << toKB(mp_st->obj_size * pm->alloc.currentLevel()) << delim;
+ stream << toKB(mp_st->obj_size * pm->alloc.peak()) << delim;
+ stream << std::setprecision(2) << ((squid_curtime - pm->alloc.peakTime()) / 3600.) << delim;
+ stream << std::setprecision(3) << xpercent(mp_st->obj_size * pm->alloc.currentLevel(), AllMeter->alloc.currentLevel()) << delim;
/* in use */
stream << mp_st->items_inuse << delim;
- stream << toKB(mp_st->obj_size * pm->inuse.level) << delim;
- stream << toKB(mp_st->obj_size * pm->inuse.hwater_level) << delim;
- stream << std::setprecision(2) << ((squid_curtime - pm->inuse.hwater_stamp) / 3600.) << delim;
- stream << std::setprecision(3) << xpercent(pm->inuse.level, pm->alloc.level) << delim;
+ stream << toKB(mp_st->obj_size * pm->inuse.currentLevel()) << delim;
+ stream << toKB(mp_st->obj_size * pm->inuse.peak()) << delim;
+ stream << std::setprecision(2) << ((squid_curtime - pm->inuse.peakTime()) / 3600.) << delim;
+ stream << std::setprecision(3) << xpercent(pm->inuse.currentLevel(), pm->alloc.currentLevel()) << delim;
/* idle */
stream << mp_st->items_idle << delim;
- stream << toKB(mp_st->obj_size * pm->idle.level) << delim;
- stream << toKB(mp_st->obj_size * pm->idle.hwater_level) << delim;
+ stream << toKB(mp_st->obj_size * pm->idle.currentLevel()) << delim;
+ stream << toKB(mp_st->obj_size * pm->idle.peak()) << delim;
/* saved */
stream << (int)pm->gb_saved.count << delim;
stream << std::setprecision(3) << xpercent(pm->gb_saved.count, AllMeter->gb_allocated.count) << delim;
// use this to sort on %Total Allocated
//
- double pa = (double) A->obj_size * A->meter->alloc.level;
- double pb = (double) B->obj_size * B->meter->alloc.level;
+ double pa = (double) A->obj_size * A->meter->alloc.currentLevel();
+ double pb = (double) B->obj_size * B->meter->alloc.currentLevel();
if (pa > pb)
return -1;
#if 0
// use this to sort on In Use high(hrs)
//
- if (A->meter->inuse.hwater_stamp > B->meter->inuse.hwater_stamp)
+ if (A->meter->inuse.peakTime() > B->meter->inuse.peakTime())
return -1;
- if (B->meter->inuse.hwater_stamp > A->meter->inuse.hwater_stamp)
+ if (B->meter->inuse.peakTime() > A->meter->inuse.peakTime())
return 1;
#endif
stream << "Cumulative allocated volume: "<< double_to_str(buf, 64, mp_total.TheMeter->gb_allocated.bytes) << "\n";
/* overhead */
stream << "Current overhead: " << mp_total.tot_overhead << " bytes (" <<
- std::setprecision(3) << xpercent(mp_total.tot_overhead, mp_total.TheMeter->inuse.level) << "%)\n";
+ std::setprecision(3) << xpercent(mp_total.tot_overhead, mp_total.TheMeter->inuse.currentLevel()) << "%)\n";
/* limits */
if (mp_total.mem_idle_limit >= 0)
stream << "Idle pool limit: " << std::setprecision(2) << toMB(mp_total.mem_idle_limit) << " MB\n";
projects / thirdparty / squid.git / blobdiff