src/mem.cc

   1
   2 /*
   3  * $Id: mem.cc,v 1.53 2001/01/07 23:36:39 hno Exp $
   4  *
   5  * DEBUG: section 13    High Level Memory Pool Management
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  the Regents of the University of California.  Please see the
  16  *  COPYRIGHT file for full details.  Squid incorporates software
  17  *  developed and/or copyrighted by other sources.  Please see the
  18  *  CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37
  38 /* module globals */
  39
  40 static MemPool *MemPools[MEM_MAX];
  41
  42 /* string pools */
  43 #define mem_str_pool_count 3
  44 static const struct {
  45     const char *name;
  46     size_t obj_size;
  47 } StrPoolsAttrs[mem_str_pool_count] = {
  48
  49     {
  50         "Short Strings", 36,
  51     },                          /* to fit rfc1123 and similar */
  52     {
  53         "Medium Strings", 128,
  54     },                          /* to fit most urls */
  55     {
  56         "Long Strings", 512
  57     }                           /* other */
  58 };
  59 static struct {
  60     MemPool *pool;
  61 } StrPools[mem_str_pool_count];
  62 static MemMeter StrCountMeter;
  63 static MemMeter StrVolumeMeter;
  64
  65
  66 /* local routines */
  67
  68 static void
  69 memStringStats(StoreEntry * sentry)
  70 {
  71     const char *pfmt = "%-20s\t %d\t %d\n";
  72     int i;
  73     int pooled_count = 0;
  74     size_t pooled_volume = 0;
  75     /* heading */
  76     storeAppendPrintf(sentry,
  77         "String Pool\t Impact\t\t\n"
  78         " \t (%%strings)\t (%%volume)\n");
  79     /* table body */
  80     for (i = 0; i < mem_str_pool_count; i++) {
  81         const MemPool *pool = StrPools[i].pool;
  82         const int plevel = pool->meter.inuse.level;
  83         storeAppendPrintf(sentry, pfmt,
  84             pool->label,
  85             xpercentInt(plevel, StrCountMeter.level),
  86             xpercentInt(plevel * pool->obj_size, StrVolumeMeter.level));
  87         pooled_count += plevel;
  88         pooled_volume += plevel * pool->obj_size;
  89     }
  90     /* malloc strings */
  91     storeAppendPrintf(sentry, pfmt,
  92         "Other Strings",
  93         xpercentInt(StrCountMeter.level - pooled_count, StrCountMeter.level),
  94         xpercentInt(StrVolumeMeter.level - pooled_volume, StrVolumeMeter.level));
  95 }
  96
  97 static void
  98 memStats(StoreEntry * sentry)
  99 {
 100     storeBuffer(sentry);
 101     memReport(sentry);
 102     memStringStats(sentry);
 103     storeBufferFlush(sentry);
 104 }
 105
 106
 107 /*
 108  * public routines
 109  */
 110
 111 /*
 112  * we have a limit on _total_ amount of idle memory so we ignore
 113  * max_pages for now
 114  */
 115 void
 116 memDataInit(mem_type type, const char *name, size_t size, int max_pages_notused)
 117 {
 118     assert(name && size);
 119     MemPools[type] = memPoolCreate(name, size);
 120 }
 121
 122
 123 /* find appropriate pool and use it (pools always init buffer with 0s) */
 124 void *
 125 memAllocate(mem_type type)
 126 {
 127     return memPoolAlloc(MemPools[type]);
 128 }
 129
 130 /* give memory back to the pool */
 131 void
 132 memFree(void *p, int type)
 133 {
 134     memPoolFree(MemPools[type], p);
 135 }
 136
 137 /* allocate a variable size buffer using best-fit pool */
 138 void *
 139 memAllocBuf(size_t net_size, size_t * gross_size)
 140 {
 141     int i;
 142     MemPool *pool = NULL;
 143     assert(gross_size);
 144     for (i = 0; i < mem_str_pool_count; i++) {
 145         if (net_size <= StrPoolsAttrs[i].obj_size) {
 146             pool = StrPools[i].pool;
 147             break;
 148         }
 149     }
 150     *gross_size = pool ? pool->obj_size : net_size;
 151     assert(*gross_size >= net_size);
 152     memMeterInc(StrCountMeter);
 153     memMeterAdd(StrVolumeMeter, *gross_size);
 154     return pool ? memPoolAlloc(pool) : xcalloc(1, net_size);
 155 }
 156
 157 /* free buffer allocated with memAllocBuf() */
 158 void
 159 memFreeBuf(size_t size, void *buf)
 160 {
 161     int i;
 162     MemPool *pool = NULL;
 163     assert(size && buf);
 164     for (i = 0; i < mem_str_pool_count; i++) {
 165         if (size <= StrPoolsAttrs[i].obj_size) {
 166             assert(size == StrPoolsAttrs[i].obj_size);
 167             pool = StrPools[i].pool;
 168             break;
 169         }
 170     }
 171     memMeterDec(StrCountMeter);
 172     memMeterDel(StrVolumeMeter, size);
 173     pool ? memPoolFree(pool, buf) : xfree(buf);
 174 }
 175
 176 void
 177 memInit(void)
 178 {
 179     int i;
 180     memInitModule();
 181     /* set all pointers to null */
 182     memset(MemPools, '\0', sizeof(MemPools));
 183     /*
 184      * it does not hurt much to have a lot of pools since sizeof(MemPool) is
 185      * small; someday we will figure out what to do with all the entries here
 186      * that are never used or used only once; perhaps we should simply use
 187      * malloc() for those? @?@
 188      */
 189     memDataInit(MEM_2K_BUF, "2K Buffer", 2048, 10);
 190     memDataInit(MEM_4K_BUF, "4K Buffer", 4096, 10);
 191     memDataInit(MEM_8K_BUF, "8K Buffer", 8192, 10);
 192     memDataInit(MEM_16K_BUF, "16K Buffer", 16384, 10);
 193     memDataInit(MEM_32K_BUF, "32K Buffer", 32768, 10);
 194     memDataInit(MEM_64K_BUF, "64K Buffer", 65536, 10);
 195     memDataInit(MEM_CLIENT_SOCK_BUF, "Client Socket Buffer", CLIENT_SOCK_SZ, 0);
 196     memDataInit(MEM_ACCESSLOGENTRY, "AccessLogEntry",
 197         sizeof(AccessLogEntry), 10);
 198     memDataInit(MEM_ACL, "acl", sizeof(acl), 0);
 199     memDataInit(MEM_ACLCHECK_T, "aclCheck_t", sizeof(aclCheck_t), 0);
 200     memDataInit(MEM_ACL_ACCESS, "acl_access", sizeof(acl_access), 0);
 201     memDataInit(MEM_ACL_DENY_INFO_LIST, "acl_deny_info_list",
 202         sizeof(acl_deny_info_list), 0);
 203     memDataInit(MEM_ACL_IP_DATA, "acl_ip_data", sizeof(acl_ip_data), 0);
 204     memDataInit(MEM_ACL_LIST, "acl_list", sizeof(acl_list), 0);
 205     memDataInit(MEM_ACL_NAME_LIST, "acl_name_list", sizeof(acl_name_list), 0);
 206     memDataInit(MEM_ACL_TIME_DATA, "acl_time_data", sizeof(acl_time_data), 0);
 207     memDataInit(MEM_AUTH_USER_T, "auth_user_t",
 208         sizeof(auth_user_t), 0);
 209     memDataInit(MEM_AUTH_USER_HASH, "auth_user_hash_pointer",
 210         sizeof(auth_user_hash_pointer), 0);
 211     memDataInit(MEM_ACL_PROXY_AUTH_MATCH, "acl_proxy_auth_match_cache",
 212         sizeof(acl_proxy_auth_match_cache), 0);
 213     memDataInit(MEM_ACL_USER_DATA, "acl_user_data",
 214         sizeof(acl_user_data), 0);
 215     memDataInit(MEM_CACHEMGR_PASSWD, "cachemgr_passwd",
 216         sizeof(cachemgr_passwd), 0);
 217 #if USE_CACHE_DIGESTS
 218     memDataInit(MEM_CACHE_DIGEST, "CacheDigest", sizeof(CacheDigest), 0);
 219 #endif
 220     memDataInit(MEM_CLIENTHTTPREQUEST, "clientHttpRequest",
 221         sizeof(clientHttpRequest), 0);
 222     memDataInit(MEM_CLOSE_HANDLER, "close_handler", sizeof(close_handler), 0);
 223     memDataInit(MEM_COMMWRITESTATEDATA, "CommWriteStateData",
 224         sizeof(CommWriteStateData), 0);
 225     memDataInit(MEM_CONNSTATEDATA, "ConnStateData", sizeof(ConnStateData), 0);
 226 #if USE_CACHE_DIGESTS
 227     memDataInit(MEM_DIGEST_FETCH_STATE, "DigestFetchState", sizeof(DigestFetchState), 0);
 228 #endif
 229     memDataInit(MEM_LINK_LIST, "link_list", sizeof(link_list), 10);
 230     memDataInit(MEM_DLINK_LIST, "dlink_list", sizeof(dlink_list), 10);
 231     memDataInit(MEM_DLINK_NODE, "dlink_node", sizeof(dlink_node), 10);
 232     memDataInit(MEM_DNSSERVER_T, "dnsserver_t", sizeof(dnsserver_t), 0);
 233     memDataInit(MEM_DNSSTATDATA, "dnsStatData", sizeof(dnsStatData), 0);
 234     memDataInit(MEM_DOMAIN_PING, "domain_ping", sizeof(domain_ping), 0);
 235     memDataInit(MEM_DOMAIN_TYPE, "domain_type", sizeof(domain_type), 0);
 236     memDataInit(MEM_DREAD_CTRL, "dread_ctrl", sizeof(dread_ctrl), 0);
 237     memDataInit(MEM_DWRITE_Q, "dwrite_q", sizeof(dwrite_q), 0);
 238     memDataInit(MEM_ERRORSTATE, "ErrorState", sizeof(ErrorState), 0);
 239     memDataInit(MEM_FILEMAP, "fileMap", sizeof(fileMap), 0);
 240     memDataInit(MEM_FWD_STATE, "FwdState", sizeof(FwdState), 0);
 241     memDataInit(MEM_FWD_SERVER, "FwdServer", sizeof(FwdServer), 0);
 242     memDataInit(MEM_HASH_LINK, "hash_link", sizeof(hash_link), 0);
 243     memDataInit(MEM_HASH_TABLE, "hash_table", sizeof(hash_table), 0);
 244     memDataInit(MEM_HIERARCHYLOGENTRY, "HierarchyLogEntry",
 245         sizeof(HierarchyLogEntry), 0);
 246     memDataInit(MEM_HTTP_STATE_DATA, "HttpStateData", sizeof(HttpStateData), 0);
 247     memDataInit(MEM_HTTP_REPLY, "HttpReply", sizeof(HttpReply), 0);
 248     memDataInit(MEM_HTTP_HDR_ENTRY, "HttpHeaderEntry", sizeof(HttpHeaderEntry), 0);
 249     memDataInit(MEM_HTTP_HDR_CC, "HttpHdrCc", sizeof(HttpHdrCc), 0);
 250     memDataInit(MEM_HTTP_HDR_RANGE_SPEC, "HttpHdrRangeSpec", sizeof(HttpHdrRangeSpec), 0);
 251     memDataInit(MEM_HTTP_HDR_RANGE, "HttpHdrRange", sizeof(HttpHdrRange), 0);
 252     memDataInit(MEM_HTTP_HDR_CONTENT_RANGE, "HttpHdrContRange", sizeof(HttpHdrContRange), 0);
 253     memDataInit(MEM_ICPUDPDATA, "icpUdpData", sizeof(icpUdpData), 0);
 254     memDataInit(MEM_ICP_COMMON_T, "icp_common_t", sizeof(icp_common_t), 0);
 255     memDataInit(MEM_ICP_PING_DATA, "ping_data", sizeof(ping_data), 0);
 256     memDataInit(MEM_INTLIST, "intlist", sizeof(intlist), 0);
 257     memDataInit(MEM_IOSTATS, "iostats", sizeof(iostats), 0);
 258     memDataInit(MEM_MEMOBJECT, "MemObject", sizeof(MemObject),
 259         Squid_MaxFD >> 3);
 260     memDataInit(MEM_MEM_NODE, "mem_node", sizeof(mem_node), 0);
 261     memDataInit(MEM_NETDBENTRY, "netdbEntry", sizeof(netdbEntry), 0);
 262     memDataInit(MEM_NET_DB_NAME, "net_db_name", sizeof(net_db_name), 0);
 263     memDataInit(MEM_NET_DB_PEER, "net_db_peer", sizeof(net_db_peer), 0);
 264     memDataInit(MEM_PEER, "peer", sizeof(peer), 0);
 265 #if USE_CACHE_DIGESTS
 266     memDataInit(MEM_PEER_DIGEST, "PeerDigest", sizeof(PeerDigest), 0);
 267     memDataInit(MEM_DIGEST_FETCH_STATE, "DigestFetchState", sizeof(DigestFetchState), 0);
 268 #endif
 269 #if USE_ICMP
 270     memDataInit(MEM_PINGERECHODATA, "pingerEchoData",
 271         sizeof(pingerEchoData), 0);
 272     memDataInit(MEM_PINGERREPLYDATA, "pingerReplyData",
 273         sizeof(pingerReplyData), 0);
 274 #endif
 275     memDataInit(MEM_PS_STATE, "ps_state", sizeof(ps_state), 0);
 276     memDataInit(MEM_REFRESH_T, "refresh_t", sizeof(refresh_t), 0);
 277     memDataInit(MEM_RELIST, "relist", sizeof(relist), 0);
 278     memDataInit(MEM_REQUEST_T, "request_t", sizeof(request_t),
 279         Squid_MaxFD >> 3);
 280     memDataInit(MEM_SQUIDCONFIG, "SquidConfig", sizeof(SquidConfig), 0);
 281     memDataInit(MEM_SQUIDCONFIG2, "SquidConfig2", sizeof(SquidConfig2), 0);
 282     memDataInit(MEM_STATCOUNTERS, "StatCounters", sizeof(StatCounters), 0);
 283     memDataInit(MEM_STMEM_BUF, "Store Mem Buffer", SM_PAGE_SIZE,
 284         Config.memMaxSize / SM_PAGE_SIZE);
 285     memDataInit(MEM_STOREENTRY, "StoreEntry", sizeof(StoreEntry), 0);
 286     memDataInit(MEM_STORE_CLIENT, "store_client", sizeof(store_client), 0);
 287     memDataInit(MEM_SWAPDIR, "SwapDir", sizeof(SwapDir), 0);
 288     memDataInit(MEM_USHORTLIST, "ushort_list", sizeof(ushortlist), 0);
 289     memDataInit(MEM_WORDLIST, "wordlist", sizeof(wordlist), 0);
 290     memDataInit(MEM_CLIENT_INFO, "ClientInfo", sizeof(ClientInfo), 0);
 291     memDataInit(MEM_MD5_DIGEST, "MD5 digest", MD5_DIGEST_CHARS, 0);
 292     memDataInit(MEM_HELPER, "helper", sizeof(helper), 0);
 293     memDataInit(MEM_HELPER_STATEFUL, "stateful_helper", sizeof(statefulhelper), 0);
 294     memDataInit(MEM_HELPER_REQUEST, "helper_request",
 295         sizeof(helper_request), 0);
 296     memDataInit(MEM_HELPER_STATEFUL_REQUEST, "helper_stateful_request",
 297         sizeof(helper_stateful_request), 0);
 298     memDataInit(MEM_HELPER_SERVER, "helper_server",
 299         sizeof(helper_server), 0);
 300     memDataInit(MEM_HELPER_STATEFUL_SERVER, "helper_stateful_server",
 301         sizeof(helper_stateful_server), 0);
 302     memDataInit(MEM_STORE_IO, "storeIOState", sizeof(storeIOState), 0);
 303     memDataInit(MEM_TLV, "storeSwapTLV", sizeof(tlv), 0);
 304     memDataInit(MEM_GEN_CBDATA, "generic_cbdata", sizeof(generic_cbdata), 0);
 305     memDataInit(MEM_PUMP_STATE_DATA, "PumpStateData", sizeof(PumpStateData), 0);
 306     memDataInit(MEM_CLIENT_REQ_BUF, "clientRequestBuffer", CLIENT_REQ_BUF_SZ, 0);
 307     memDataInit(MEM_SWAP_LOG_DATA, "storeSwapLogData", sizeof(storeSwapLogData), 0);
 308
 309     /* init string pools */
 310     for (i = 0; i < mem_str_pool_count; i++) {
 311         StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size);
 312     }
 313     cachemgrRegister("mem",
 314         "Memory Utilization",
 315         memStats, 0, 1);
 316 }
 317
 318 /*
 319  * Test that all entries are initialized
 320  */
 321 void
 322 memCheckInit(void)
 323 {
 324     mem_type t;
 325     for (t = MEM_NONE, t++; t < MEM_MAX; t++) {
 326         if (MEM_DONTFREE == t)
 327             continue;
 328         /*
 329          * If you hit this assertion, then you forgot to add a
 330          * memDataInit() line for type 't'.
 331          */
 332         assert(MemPools[t]);
 333     }
 334 }
 335
 336 void
 337 memClean(void)
 338 {
 339     memCleanModule();
 340 }
 341
 342 int
 343 memInUse(mem_type type)
 344 {
 345     return memPoolInUseCount(MemPools[type]);
 346 }
 347
 348 /* ick */
 349
 350 void
 351 memFree2K(void *p)
 352 {
 353     memFree(p, MEM_2K_BUF);
 354 }
 355
 356 void
 357 memFree4K(void *p)
 358 {
 359     memFree(p, MEM_4K_BUF);
 360 }
 361
 362 void
 363 memFree8K(void *p)
 364 {
 365     memFree(p, MEM_8K_BUF);
 366 }
 367
 368 void
 369 memFree16K(void *p)
 370 {
 371     memFree(p, MEM_16K_BUF);
 372 }
 373
 374 void
 375 memFree32K(void *p)
 376 {
 377     memFree(p, MEM_32K_BUF);
 378 }
 379
 380 void
 381 memFree64K(void *p)
 382 {
 383     memFree(p, MEM_64K_BUF);
 384 }