From: rousskov <> Date: Tue, 3 Mar 1998 07:30:54 +0000 (+0000) Subject: - Added "mem_pools_limit" configuration option. Semantics of X-Git-Tag: SQUID_3_0_PRE1~3948 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7021844c42212902ee9a10ba7097201a6d66a1c5;p=thirdparty%2Fsquid.git - Added "mem_pools_limit" configuration option. Semantics of "mem_pools" option has also changed a bit to reflect new memory management policy. - Reorganized memory pools. Squid now allocates memory in big chunks and distributes that memory among "frequently allocated" objects. - memAllocate() has now only one parameter. Objects are always reset with 0s. - HttpHeader.c: fixed warnings generated by some compilers on member to union conversions. Tested using DEC cc only. - cachmgr.c now tries to interpret strings containing '\t' as table rows and format them using html table attributes. --- diff --git a/ChangeLog b/ChangeLog index 1e440acd42..c64d1c2660 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,15 @@ + - Added "mem_pools_limit" configuration option. Semantics of + "mem_pools" option has also changed a bit to reflect new memory + management policy. + - Reorganized memory pools. Squid now allocates memory in big chunks + and distributes that memory among "frequently allocated" objects. + There is a configurable limit on the total amount of memory to be + allocated for static pools. All requests that exceed that amount are + satisfied using malloc library. Support for variable size objects + (mostly strings) will be enabled soon. + - memAllocate() has now only one parameter. Objects are always reset + with 0s. (We actually never used that parameter before; it was + always set to "clear"). - Added Squid "signature" to all ERR_ pages. The signature is hard- -coded and is added on-the-fly. The signature may use %-escapes. Added interface to add more hard-coded responses if needed (see diff --git a/include/Stack.h b/include/Stack.h index d0769b62d5..213ae1439c 100644 --- a/include/Stack.h +++ b/include/Stack.h @@ -1,5 +1,5 @@ /* - * $Id: Stack.h,v 1.3 1998/02/25 07:43:03 rousskov Exp $ + * $Id: Stack.h,v 1.4 1998/03/03 00:30:55 rousskov Exp $ * * AUTHOR: Alex Rousskov * @@ -35,22 +35,21 @@ struct _Stack { /* public, read only */ size_t capacity; - int is_full; /* true if the stack is full */ - - u_num32 push_count; /* always grows, might overflow, use for stats only */ - u_num32 pop_count; /* always grows, might overflow, use for stats only */ /* protected, do not use these, use interface functions instead */ size_t count; - void **buf; + void **items; }; typedef struct _Stack Stack; -extern Stack *stackCreate(size_t capacity); +extern Stack *stackCreate(); +extern void stackInit(Stack * s); +extern void stackClean(Stack * s); extern void stackDestroy(Stack *s); extern void *stackPop(Stack *s); extern void stackPush(Stack *s, void *obj); +extern void stackPrePush(Stack * s, int push_count); #endif /* ndef _STACK_H_ */ diff --git a/include/util.h b/include/util.h index d27dce2309..8c5905d9b5 100644 --- a/include/util.h +++ b/include/util.h @@ -1,5 +1,5 @@ /* - * $Id: util.h,v 1.42 1998/02/26 22:16:26 kostas Exp $ + * $Id: util.h,v 1.43 1998/03/03 00:30:56 rousskov Exp $ * * AUTHOR: Harvest Derived * @@ -190,4 +190,8 @@ extern void stringAppend(String *, const char *, size_t); extern void stringFree(String *); #define stringLength(S) (S)->off +extern double xpercent(double part, double whole); +extern double xdiv(double nom, double denom); + + #endif /* ndef _UTIL_H_ */ diff --git a/lib/Makefile.in b/lib/Makefile.in index a6c216e653..592ccf0e40 100644 --- a/lib/Makefile.in +++ b/lib/Makefile.in @@ -1,5 +1,5 @@ # -# $Id: Makefile.in,v 1.33 1998/02/21 00:56:37 rousskov Exp $ +# $Id: Makefile.in,v 1.34 1998/03/03 00:30:56 rousskov Exp $ # prefix = @prefix@ top_srcdir = @top_srcdir@ @@ -37,7 +37,6 @@ UTILOBJS = rfc1123.o \ radix.o \ String.o \ stub_memaccount.o \ - MemPool.o \ Stack.o \ $(LIBOBJS) REGEXOBJS = GNUregex.o diff --git a/lib/Stack.c b/lib/Stack.c index b8b2e67c90..b828f467f5 100644 --- a/lib/Stack.c +++ b/lib/Stack.c @@ -1,5 +1,5 @@ /* - * $Id: Stack.c,v 1.3 1998/02/26 17:49:54 wessels Exp $ + * $Id: Stack.c,v 1.4 1998/03/03 00:30:57 rousskov Exp $ * * AUTHOR: Alex Rousskov * @@ -28,91 +28,98 @@ */ /* - * Stack is a (void*) stack with fixed capacity with limited accounting. - * Errors are handled with asserts. + * Stack is a (void*) stack with unlimited capacity and limited accounting. */ -/* - * To-do: - * - stack that grows as needed if a given delta is non zero - */ - - -#if 0 - -Synopsis(void) -{ - - /* - * creating a stack that can hold up to objCnt pointers. - * If objCnt is zero, the stack is always full (disabled) - */ - Stack *s1 = stackCreate(objCnt); - Stack *s2 = stackCreate(objCnt * 2); - - /* - * pop/push works as expected; it is OK to push a null pointer - */ - if (!s2->is_full && s1->count) - stackPush(s2, stackPop(s1)); - - /* destroying a stack */ - stackDestroy(s1); -} - -#endif /* Synopsis */ - #include "config.h" #if HAVE_ASSERT_H #include #endif +#if HAVE_STRING_H +#include +#endif #include "util.h" #include "Stack.h" -/* performance hack instead of non-ANSI inline function */ -#define stackIsFull(s) (s->count >= s->capacity) +static void stackGrow(Stack * s, int min_capacity); Stack * -stackCreate(size_t capacity) +stackCreate() { - Stack *s = xcalloc(1, sizeof(Stack)); - s->buf = capacity > 0 ? xcalloc(capacity, sizeof(void *)) : NULL; - s->capacity = capacity; - s->count = 0; - s->is_full = stackIsFull(s); - /* other members are set to 0 in calloc */ + Stack *s = xmalloc(sizeof(Stack)); + stackInit(s); return s; } void -stackDestroy(Stack * s) +stackInit(Stack * s) +{ + assert(s); + memset(s, 0, sizeof(Stack)); +} + +void +stackClean(Stack * s) { assert(s); /* could also warn if some objects are left */ - if (s->buf) - xfree(s->buf); + xfree(s->items); + s->items = NULL; +} + +void +stackDestroy(Stack * s) +{ + assert(s); + stackClean(s); xfree(s); } void * stackPop(Stack * s) { - void *popped; assert(s); assert(s->count); - popped = s->buf[--s->count]; - s->is_full = stackIsFull(s); - s->pop_count++; /* might overflow eventually, but ok */ - return popped; + return s->items[--s->count]; } void stackPush(Stack * s, void *obj) { assert(s); - assert(!s->is_full); - s->buf[s->count++] = obj; - s->is_full = stackIsFull(s); - s->push_count++; /* might overflow eventually, but ok */ + if (s->count >= s->capacity) + stackGrow(s, s->count+1); + s->items[s->count++] = obj; +} + +/* if you are going to push a known and large number of items, call this first */ +void +stackPrePush(Stack * s, int push_count) +{ + assert(s); + if (s->count + push_count > s->capacity) + stackGrow(s, s->count + push_count); +} + +/* grows internal buffer to satisfy required minimal capacity */ +static void +stackGrow(Stack * s, int min_capacity) +{ + static const int min_delta = 16; + int delta; + assert(s->capacity < min_capacity); + delta = min_capacity; + /* make delta a multiple of min_delta */ + delta += min_delta-1; + delta /= min_delta; + delta *= min_delta; + /* actual grow */ + assert(delta > 0); + s->capacity += delta; + s->items = s->items ? + xrealloc(s->items, s->capacity * sizeof(void*)) : + xmalloc(s->capacity * sizeof(void*)); + /* reset, just in case */ + memset(s->items+s->count, 0, (s->capacity-s->count) * sizeof(void*)); } diff --git a/lib/util.c b/lib/util.c index f03d901a9e..58d2887843 100644 --- a/lib/util.c +++ b/lib/util.c @@ -1,6 +1,6 @@ /* - * $Id: util.c,v 1.52 1998/02/28 16:17:28 kostas Exp $ + * $Id: util.c,v 1.53 1998/03/03 00:30:57 rousskov Exp $ * * DEBUG: * AUTHOR: Harvest Derived @@ -730,3 +730,17 @@ xcountws(const char *str) } return count; } + +/* somewhat safer calculation of %s */ +double +xpercent(double part, double whole) +{ + return xdiv(100 * part, whole); +} + +/* somewhat safer division */ +double +xdiv(double nom, double denom) +{ + return (denom != 0.0) ? nom / denom : -1.0; +} diff --git a/snmplib/mib.c b/snmplib/mib.c index 04c0988a32..f82749c95c 100644 --- a/snmplib/mib.c +++ b/snmplib/mib.c @@ -531,6 +531,7 @@ set_functions(subtree) void init_mib(char *file) { + snmplib_debug(0, "init MIB(%p): file: '%s'\n", Mib, file ? file : ""); if (Mib != NULL) return; diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc index 57f79521fb..a8c49ff8bc 100644 --- a/src/HttpHeader.cc +++ b/src/HttpHeader.cc @@ -1,5 +1,5 @@ /* - * $Id: HttpHeader.cc,v 1.10 1998/02/26 18:00:31 wessels Exp $ + * $Id: HttpHeader.cc,v 1.11 1998/03/03 00:30:59 rousskov Exp $ * * DEBUG: section 55 HTTP Header * AUTHOR: Alex Rousskov @@ -29,7 +29,6 @@ */ #include "squid.h" -#include "MemPool.h" /* * On naming conventions: @@ -300,16 +299,18 @@ static void freeShortString(char *str); static int strListGetItem(const char *str, char del, const char **item, int *ilen, const char **pos); static const char *getStringPrefix(const char *str); -static double xpercent(double part, double whole); -static double xdiv(double nom, double denom); - - -/* delete this when everybody remembers that ':' is not a part of a name */ -#define conversion_period_name_check(name) assert(!strchr((name), ':')) /* handy to determine the #elements in a static array */ #define countof(arr) (sizeof(arr)/sizeof(*arr)) +/* + * some compilers do not want to convert a type into a union which that type + * belongs to + */ +field_store intField(int n) { field_store f; f.v_int = n; return f; } +field_store timeField(time_t t) { field_store f; f.v_time = t; return f; } +field_store ptrField(void *p) { field_store f; f.v_pchar = (char*)p; return f; } + /* * Module initialization routines */ @@ -328,7 +329,7 @@ httpHeaderInitModule() ReplyHeadersMask = httpHeaderCalcMask((const int *) ReplyHeaders, countof(ReplyHeaders)); RequestHeadersMask = httpHeaderCalcMask((const int *) RequestHeaders, countof(RequestHeaders)); /* create a pool of short strings @?@ we never destroy it! */ - shortStrings = memPoolCreate(shortStrPoolCount, shortStrPoolCount / 10, shortStrSize, "shortStr"); + shortStrings = memPoolCreate("'short http hdr strs'", shortStrSize); /* init header stats */ for (i = 0; i < HttpHeaderStatCount; i++) httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label); @@ -336,6 +337,15 @@ httpHeaderInitModule() "HTTP Header Statistics", httpHeaderStoreReport, 0); } +void +httpHeaderCleanModule() +{ + if (shortStrings) { + memPoolDestroy(shortStrings); + shortStrings = NULL; + } +} + static void httpHeaderInitAttrTable(field_attrs_t * table, int count) { @@ -825,7 +835,7 @@ httpHeaderAddExt(HttpHeader * hdr, const char *name, const char *value) HttpHeaderEntry e; debug(55, 8) ("%p adds ext entry '%s:%s'\n", hdr, name, value); - httpHeaderEntryInit(&e, HDR_OTHER, ext); + httpHeaderEntryInit(&e, HDR_OTHER, ptrField(ext)); httpHeaderAddNewEntry(hdr, &e); } @@ -986,7 +996,7 @@ httpHeaderEntryParseInit(HttpHeaderEntry * e, const char *field_start, const cha Headers[id].stat.parsCount++; if (id == HDR_OTHER) { /* hm.. it is an extension field indeed */ - httpHeaderEntryInit(e, id, f); + httpHeaderEntryInit(e, id, ptrField(f)); return 1; } /* ok, we got something interesting, parse it further */ @@ -1009,7 +1019,7 @@ httpHeaderEntryParseExtFieldInit(HttpHeaderEntry * e, int id, const HttpHeaderEx switch (id) { case HDR_PROXY_KEEPALIVE: /* we treat Proxy-Connection as "keep alive" only if it says so */ - httpHeaderEntryInit(e, id, (int) !strcasecmp(f->value, "Keep-Alive")); + httpHeaderEntryInit(e, id, intField(!strcasecmp(f->value, "Keep-Alive"))); break; default: /* if we got here, it is something that can be parsed based on value type */ @@ -1201,16 +1211,12 @@ httpHeaderEntryIsValid(const HttpHeaderEntry * e) return e->field.v_int >= 0; case ftPChar: return e->field.v_pchar != NULL; - break; case ftDate_1123: return e->field.v_time >= 0; - break; case ftPSCC: return e->field.v_pscc != NULL; - break; case ftPExtField: return e->field.v_pefield != NULL; - break; default: assert(0); /* query for invalid/unknown type */ } @@ -1244,23 +1250,18 @@ httpHeaderFieldDup(field_type type, field_store value) /* type based duplication */ switch (type) { case ftInt: - return value.v_int; - case ftPChar: - return dupShortStr(value.v_pchar); - break; case ftDate_1123: - return value.v_time; - break; + return value; + case ftPChar: + return ptrField(dupShortStr(value.v_pchar)); case ftPSCC: - return httpSccDup(value.v_pscc); - break; + return ptrField(httpSccDup(value.v_pscc)); case ftPExtField: - return httpHeaderExtFieldDup(value.v_pefield); - break; + return ptrField(httpHeaderExtFieldDup(value.v_pefield)); default: assert(0); /* dup of invalid/unknown type */ } - return NULL; /* not reached */ + return ptrField(NULL); /* not reached */ } /* @@ -1272,17 +1273,18 @@ httpHeaderFieldBadValue(field_type type) { switch (type) { case ftInt: + return intField(-1); case ftDate_1123: - return -1; + return timeField(-1); case ftPChar: case ftPSCC: case ftPExtField: - return NULL; + return ptrField(NULL); case ftInvalid: default: assert(0); /* query for invalid/unknown type */ } - return NULL; /* not reached */ + return ptrField(NULL); /* not reached */ } /* @@ -1292,7 +1294,7 @@ httpHeaderFieldBadValue(field_type type) static HttpScc * httpSccCreate() { - HttpScc *scc = memAllocate(MEM_HTTP_SCC, 1); + HttpScc *scc = memAllocate(MEM_HTTP_SCC); scc->max_age = -1; return scc; } @@ -1484,7 +1486,7 @@ httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, const int valid_id = id >= 0 && id < HDR_ENUM_END; const char *name = valid_id ? Headers[id].name : "INVALID"; if (count || valid_id) - storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2lf\n", + storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n", id, name, count, xdiv(count, HeaderParsedCount)); } @@ -1495,7 +1497,7 @@ httpHeaderCCStatDumper(StoreEntry * sentry, int idx, double val, double size, in const int valid_id = id >= 0 && id < SCC_ENUM_END; const char *name = valid_id ? SccAttrs[id].name : "INVALID"; if (count || valid_id) - storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2lf\n", + storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n", id, name, count, xdiv(count, CcPasredCount)); } @@ -1504,7 +1506,7 @@ static void httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count) { if (count) - storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2lf\n", + storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n", idx, ((int) (val + size)), count, xpercent(count, HeaderEntryParsedCount)); } @@ -1515,15 +1517,15 @@ httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e) assert(hs && e); storeAppendPrintf(e, "\n

Header Stats: %s

\n", hs->label); - storeAppendPrintf(e, "\t

Field type distribution

\n"); + storeAppendPrintf(e, "

Field type distribution

\n"); storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n", "id", "name", "count", "#/header"); statHistDump(&hs->fieldTypeDistr, e, httpHeaderFieldStatDumper); - storeAppendPrintf(e, "\t

Cache-control directives distribution

\n"); + storeAppendPrintf(e, "

Cache-control directives distribution

\n"); storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n", "id", "name", "count", "#/cc_field"); statHistDump(&hs->ccTypeDistr, e, httpHeaderCCStatDumper); - storeAppendPrintf(e, "\t

Number of fields per header distribution (init size: %d)

\n", + storeAppendPrintf(e, "

Number of fields per header distribution (init size: %d)

\n", INIT_FIELDS_PER_HEADER); storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n", "id", "#flds", "count", "%total"); @@ -1533,10 +1535,11 @@ httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e) static void shortStringStatDump(StoreEntry * e) { - storeAppendPrintf(e, "

Short String Stats

\n

%s\n

\n", - memPoolReport(shortStrings)); + storeAppendPrintf(e, "

Short String Stats

\n

"); + memPoolReport(shortStrings, e); + storeAppendPrintf(e, "\n

\n"); storeAppendPrintf(e, "

Long String Stats

\n"); - storeAppendPrintf(e, "\talive: %3d (%5.1lf KB) high-water: %3d (%5.1lf KB)\n", + storeAppendPrintf(e, "alive: %3d (%5.1f KB) high-water: %3d (%5.1f KB)\n", longStrAliveCount, (double) longStrAliveSize / 1024., longStrHighWaterCount, (double) longStrHighWaterSize / 1024.); } @@ -1560,7 +1563,7 @@ httpHeaderStoreReport(StoreEntry * e) "id", "name", "#alive", "%err", "%repeat"); for (ht = 0; ht < HDR_ENUM_END; ht++) { field_attrs_t *f = Headers + ht; - storeAppendPrintf(e, "%2d\t %-20s\t %5d\t %6.3lf\t %6.3lf\n", + storeAppendPrintf(e, "%2d\t %-20s\t %5d\t %6.3f\t %6.3f\n", f->id, f->name, f->stat.aliveCount, xpercent(f->stat.errCount, f->stat.parsCount), xpercent(f->stat.repCount, f->stat.parsCount)); @@ -1611,8 +1614,7 @@ allocShortBuf(size_t sz) { char *buf = NULL; assert(shortStrings); - /* tmp_debug(here) ("allocating short buffer of size %d (max: %d)\n", sz, shortStrings->obj_size); @?@ */ - if (sz > shortStrings->obj_size) { + if (sz > shortStrSize) { buf = xmalloc(sz); longStrAliveCount++; longStrAliveSize += sz; @@ -1621,7 +1623,7 @@ allocShortBuf(size_t sz) if (longStrHighWaterSize < longStrAliveSize) longStrHighWaterSize = longStrAliveSize; } else - buf = memPoolGetObj(shortStrings); + buf = memPoolAlloc(shortStrings); return buf; } @@ -1631,15 +1633,15 @@ freeShortString(char *str) assert(shortStrings); if (str) { const size_t sz = strlen(str) + 1; - debug(55, 9) ("freeing short str of size %d (max: %d) '%s' (%p)\n", sz, shortStrings->obj_size, str, str); - if (sz > shortStrings->obj_size) { - debug(55, 9) ("LONG short string[%d>%d]: %s\n", sz, shortStrings->obj_size, str); + debug(55, 9) ("freeing short str of size %d (max: %d) '%s' (%p)\n", sz, shortStrSize, str, str); + if (sz > shortStrSize) { + debug(55, 9) ("LONG short string[%d>%d]: %s\n", sz, shortStrSize, str); assert(longStrAliveCount); xfree(str); longStrAliveCount--; longStrAliveSize -= sz; } else - memPoolPutObj(shortStrings, str); + memPoolFree(shortStrings, str); } } @@ -1693,17 +1695,3 @@ getStringPrefix(const char *str) xstrncpy(buf, str, SHORT_PREFIX_SIZE); return buf; } - -/* safe percent calculation */ -static double -xpercent(double part, double whole) -{ - return xdiv(100 * part, whole); -} - -/* safe division */ -static double -xdiv(double nom, double denom) -{ - return (denom != 0.0) ? nom / denom : -1; -} diff --git a/src/HttpReply.cc b/src/HttpReply.cc index 5efc00ba87..d31125e518 100644 --- a/src/HttpReply.cc +++ b/src/HttpReply.cc @@ -1,6 +1,6 @@ /* - * $Id: HttpReply.cc,v 1.6 1998/02/26 18:00:31 wessels Exp $ + * $Id: HttpReply.cc,v 1.7 1998/03/03 00:30:59 rousskov Exp $ * * DEBUG: section 58 HTTP Reply (Response) * AUTHOR: Alex Rousskov @@ -48,7 +48,7 @@ static int httpReplyIsolateHeaders(const char **parse_start, const char **blk_st HttpReply * httpReplyCreate() { - HttpReply *rep = memAllocate(MEM_HTTPREPLY, 1); + HttpReply *rep = memAllocate(MEM_HTTPREPLY); tmp_debug(here) ("creating rep: %p\n", rep); httpReplyInit(rep); return rep; @@ -110,7 +110,7 @@ httpReplyParse(HttpReply * rep, const char *buf) * in store. Currently we have to xstrncpy the buffer becuase store.c may * feed a non 0-terminated buffer to us @?@. */ - char *headers = memAllocate(MEM_4K_BUF, 1); + char *headers = memAllocate(MEM_4K_BUF); int success; /* reset current state, because we are not used in incremental fashion */ httpReplyReset(rep); diff --git a/src/Makefile.in b/src/Makefile.in index 49264584d4..52fcbf9e8f 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -1,7 +1,7 @@ # # Makefile for the Squid Object Cache server # -# $Id: Makefile.in,v 1.125 1998/02/25 09:53:54 rousskov Exp $ +# $Id: Makefile.in,v 1.126 1998/03/03 00:31:00 rousskov Exp $ # # Uncomment and customize the following to suit your needs: # @@ -101,6 +101,7 @@ OBJS = \ ipcache.o \ main.o \ mem.o \ + Mem.o \ MemBuf.o \ mime.o \ multicast.o \ diff --git a/src/asn.cc b/src/asn.cc index 5ccd41c223..0e70aec1eb 100644 --- a/src/asn.cc +++ b/src/asn.cc @@ -1,5 +1,5 @@ /* - * $Id: asn.cc,v 1.21 1998/02/13 20:37:10 wessels Exp $ + * $Id: asn.cc,v 1.22 1998/03/03 00:31:00 rousskov Exp $ * * DEBUG: section 53 AS Number handling * AUTHOR: Duane Wessels, Kostas Anagnostakis @@ -197,7 +197,7 @@ asnCacheStart(int as) storeClientListAdd(e, asState); } asState->entry = e; - storeClientCopy(e, 0, 0, 4096, memAllocate(MEM_4K_BUF, 1), asHandleReply, asState); + storeClientCopy(e, 0, 0, 4096, memAllocate(MEM_4K_BUF), asHandleReply, asState); } static void @@ -397,7 +397,7 @@ whoisReadReply(int fd, void *data) { whoisState *p = data; StoreEntry *entry = p->entry; - char *buf = memAllocate(MEM_4K_BUF, 1); + char *buf = memAllocate(MEM_4K_BUF); int len; len = read(fd, buf, 4096); diff --git a/src/cache_cf.cc b/src/cache_cf.cc index bf14b1d89a..541ae61f3d 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -1,6 +1,6 @@ /* - * $Id: cache_cf.cc,v 1.254 1998/02/26 18:00:37 wessels Exp $ + * $Id: cache_cf.cc,v 1.255 1998/03/03 00:31:01 rousskov Exp $ * * DEBUG: section 3 Configuration File Parsing * AUTHOR: Harvest Derived @@ -192,6 +192,8 @@ configDoConfigure(void) { LOCAL_ARRAY(char, buf, BUFSIZ); memset(&Config2, '\0', sizeof(SquidConfig2)); + /* init memory as early as possible */ + memConfigure(); /* Sanity checks */ if (Config.cacheSwap.swapDirs == NULL) fatal("No cache_dir's specified in config file"); diff --git a/src/cachemgr.cc b/src/cachemgr.cc index 91ce34244c..9d2b465217 100644 --- a/src/cachemgr.cc +++ b/src/cachemgr.cc @@ -1,6 +1,6 @@ /* - * $Id: cachemgr.cc,v 1.72 1998/02/26 18:00:38 wessels Exp $ + * $Id: cachemgr.cc,v 1.73 1998/03/03 00:31:02 rousskov Exp $ * * DEBUG: section 0 CGI Cache Manager * AUTHOR: Duane Wessels @@ -182,6 +182,13 @@ safe_str(const char *str) return str ? str : ""; } +/* relaxed number format */ +static int +is_number(const char *str) +{ + return strspn(str, "\t -+01234567890./\n") == strlen(str); +} + static char * xstrtok(char **str, char del) { @@ -293,37 +300,80 @@ munge_menu_line(const char *buf, cachemgr_request * req) const char *d; const char *p; char *a_url; - static char html[1024]; + char *buf_copy; + static char html[2*1024]; if (strlen(buf) < 1) return buf; if (*buf != ' ') return buf; - x = xstrdup(buf); + buf_copy = x = xstrdup(buf); a = xstrtok(&x, '\t'); d = xstrtok(&x, '\t'); p = xstrtok(&x, '\t'); a_url = xstrdup(menu_url(req, a)); /* no reason to give a url for a disabled action */ if (!strcmp(p, "disabled")) - snprintf(html, 1024, "
  • %s (disabled).\n", d, a_url); + snprintf(html, sizeof(html), "
  • %s (disabled).\n", d, a_url); else /* disable a hidden action (requires a password, but password is not in squid.conf) */ if (!strcmp(p, "hidden")) - snprintf(html, 1024, "
  • %s (hidden).\n", d, a_url); + snprintf(html, sizeof(html), "
  • %s (hidden).\n", d, a_url); else /* disable link if authentication is required and we have no password */ if (!strcmp(p, "protected") && !req->passwd) - snprintf(html, 1024, "
  • %s (requires authentication).\n", + snprintf(html, sizeof(html), "
  • %s (requires authentication).\n", d, menu_url(req, "authenticate"), a_url); else /* highlight protected but probably available entries */ if (!strcmp(p, "protected")) - snprintf(html, 1024, "
  • %s\n", + snprintf(html, sizeof(html), "
  • %s\n", a_url, d); /* public entry or unknown type of protection */ else - snprintf(html, 1024, "
  • %s\n", a_url, d); + snprintf(html, sizeof(html), "
  • %s\n", a_url, d); xfree(a_url); + xfree(buf_copy); + return html; +} + +static const char * +munge_other_line(const char *buf, cachemgr_request * req) +{ + static const char* ttags[] = { "td", "th" }; + static char html[4096]; + static table_line_num = 0; + int is_header = 0; + const char *ttag; + char *cell; + char *buf_copy; + char *x; + int l = 0; + /* does it look like a table? */ + if (!strchr(buf, '\t') || *buf == '\t') { + /* nope, just text */ + snprintf(html, sizeof(html), "%s%s", + table_line_num ? "\n
    " : "", buf);
    +	table_line_num = 0;
    +	return html;
    +    }
    +    if (!table_line_num)
    +	l += snprintf(html+l, sizeof(html)-l, "
    \n"); + is_header = !table_line_num && !strchr(buf, ':') && !is_number(buf); + ttag = ttags[is_header]; + /* record starts */ + l += snprintf(html+l, sizeof(html)-l, ""); + /* substitute '\t' */ + buf_copy = x = xstrdup(buf); + while (x && (cell = xstrtok(&x, '\t'))) { + l += snprintf(html+l, sizeof(html)-l, "<%s align=\"%s\">%s", + ttag, + is_header ? "center" : is_number(cell) ? "right" : "left", + cell, ttag); + } + xfree(buf_copy); + /* record ends */ + l += snprintf(html+l, sizeof(html)-l, "\n"); + table_line_num++; return html; } @@ -398,7 +448,7 @@ read_reply(int s, cachemgr_request * req) if (parse_menu) fputs(munge_menu_line(buf, req), stdout); else - fputs(buf, stdout); + fputs(munge_other_line(buf, req), stdout); break; case isForward: /* forward: no modifications allowed */ @@ -633,7 +683,7 @@ make_pub_auth(cachemgr_request * req) static char buf[1024]; safe_free(req->pub_auth); debug(3) fprintf(stderr, "cmgr: encoding for pub...\n"); - if (!req->passwd && !strlen(req->passwd)) + if (!req->passwd || !strlen(req->passwd)) return; /* host | time | user | passwd */ snprintf(buf, sizeof(buf), "%s|%d|%s|%s", diff --git a/src/cf.data.pre b/src/cf.data.pre index f93d4fc17e..313fb2f8b5 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1622,11 +1622,44 @@ LOC: Config.onoff.mem_pools DOC_START If set, Squid will keep pools of allocated (but unused) memory available for future use. If memory is a premium on your - system, disable this. + system and you believe your malloc library outperforms Squid + routines, disable this. memory_pools on DOC_END +NAME: memory_pools_limit +COMMENT: (bytes) +TYPE: b_size_t +DEFAULT: none +LOC: Config.onoff.mem_pools +DOC_START + Used only with memory_pools on. + + If set to a non-zero value, Squid will allocate at most the specified + limit for memory pools. All requests for memory that exceed this limit + will be handled by your malloc library. Squid does not allocate all + pools at once, but does it on-demand. Thus, it is safe to set + memory_pools_limit to a reasonably high value even if your + configuration will use less memory. High value of memory_pools_limit + allows Squid to start allocation with larger memory chunks which is + more efficient. + + If not set or set to zero, Squid will allocate memory for pools by + small chunks. There will be no limit on the amount of memory it can + allocate. No-limit mode is less efficient because Squid has more + memory chunks to maintain. However, this mode is useful when it is the + first time you run your configuration with mem_pools "on" and do not + want to guess the limit value. Squid logs the actual amount of memory + used in cache log. Next time you run Squid, set memory_pools_limit + close to the value reported in the log. + + To disable memory allocation optimization, do not set + memory_pools_limit to 0. Set memory_pools to "off" instead. + +memory_pools_limit 50 MB +DOC_END + NAME: forwarded_for COMMENT: on|off TYPE: onoff diff --git a/src/client_side.cc b/src/client_side.cc index b4eb51ccb7..1c61a46fea 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.218 1998/02/26 18:00:40 wessels Exp $ + * $Id: client_side.cc,v 1.219 1998/03/03 00:31:04 rousskov Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -188,14 +188,14 @@ clientCreateStoreEntry(clientHttpRequest * h, method_t m, int flags) * so make a fake one. */ if (h->request == NULL) { - r = memAllocate(MEM_REQUEST_T, 1); + r = memAllocate(MEM_REQUEST_T); r->method = m; r->protocol = PROTO_NONE; h->request = requestLink(r); } e = storeCreateEntry(h->uri, h->log_uri, flags, m); storeClientListAdd(e, h); - storeClientCopy(e, 0, 0, 4096, memAllocate(MEM_4K_BUF, 1), clientSendMoreData, h); + storeClientCopy(e, 0, 0, 4096, memAllocate(MEM_4K_BUF), clientSendMoreData, h); return e; } @@ -319,7 +319,7 @@ clientProcessExpired(void *data) http->out.offset, http->out.offset, 4096, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientHandleIMSReply, http); } @@ -381,7 +381,7 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) http->out.offset + size, http->out.offset, 4096, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientHandleIMSReply, http); return; @@ -434,7 +434,7 @@ clientHandleIMSReply(void *data, char *buf, ssize_t size) http->out.offset, http->out.offset, 4096, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientCacheHit, http); } @@ -885,8 +885,8 @@ clientBuildReplyHeader(clientHttpRequest * http, debug(33, 3) ("clientBuildReplyHeader: DIDN'T FIND END-OF-HEADERS\n"); return 0; } - xbuf = memAllocate(MEM_4K_BUF, 1); - ybuf = memAllocate(MEM_4K_BUF, 1); + xbuf = memAllocate(MEM_4K_BUF); + ybuf = memAllocate(MEM_4K_BUF); for (t = hdr_in; t < end; t += strcspn(t, crlf), t += strspn(t, crlf)) { hdr_len = t - hdr_in; l = strcspn(t, crlf) + 1; @@ -1010,7 +1010,7 @@ clientSendMoreData(void *data, char *buf, ssize_t size) C = *(buf + size); } *(buf + size) = '\0'; - newbuf = memAllocate(MEM_8K_BUF, 1); + newbuf = memAllocate(MEM_8K_BUF); hdrlen = 0; l = clientBuildReplyHeader(http, buf, &hdrlen, newbuf, 8192); @@ -1114,7 +1114,7 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da http->out.offset, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientSendMoreData, http); } @@ -1142,7 +1142,7 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da http->out.offset, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientSendMoreData, http); } @@ -1199,7 +1199,7 @@ clientGetHeadersForIMS(void *data, char *buf, ssize_t size) http->out.offset + size, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientGetHeadersForIMS, http); return; @@ -1235,7 +1235,7 @@ clientGetHeadersForIMS(void *data, char *buf, ssize_t size) http->out.offset, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientSendMoreData, http); return; @@ -1403,7 +1403,7 @@ clientProcessRequest(clientHttpRequest * http) http->out.offset, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientCacheHit, http); break; @@ -1414,7 +1414,7 @@ clientProcessRequest(clientHttpRequest * http) http->out.offset, http->out.offset, SM_PAGE_SIZE, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), clientGetHeadersForIMS, http); break; diff --git a/src/ftp.cc b/src/ftp.cc index 3ab5e78db8..5608e779cf 100644 --- a/src/ftp.cc +++ b/src/ftp.cc @@ -1,6 +1,6 @@ /* - * $Id: ftp.cc,v 1.198 1998/02/25 16:57:19 wessels Exp $ + * $Id: ftp.cc,v 1.199 1998/03/03 00:31:05 rousskov Exp $ * * DEBUG: section 9 File Transfer Protocol (FTP) * AUTHOR: Harvest Derived @@ -704,7 +704,7 @@ ftpParseListing(FtpStateData * ftpState, int len) debug(9, 3) ("ftpParseListing: didn't find end for %s\n", storeUrl(e)); return; } - line = memAllocate(MEM_4K_BUF, 1); + line = memAllocate(MEM_4K_BUF); end++; /* XXX there is an ABR bug here. We need to make sure buf is * NULL terminated */ @@ -1024,7 +1024,7 @@ ftpConnectDone(int fd, int status, void *data) comm_close(ftpState->ctrl.fd); } else { ftpState->state = BEGIN; - ftpState->ctrl.buf = memAllocate(MEM_4K_BUF, 1); + ftpState->ctrl.buf = memAllocate(MEM_4K_BUF); ftpState->ctrl.freefunc = memFree4K; ftpState->ctrl.size = 4096; ftpState->ctrl.offset = 0; diff --git a/src/gopher.cc b/src/gopher.cc index a688955c4e..c54f54a7ab 100644 --- a/src/gopher.cc +++ b/src/gopher.cc @@ -1,6 +1,6 @@ /* - * $Id: gopher.cc,v 1.119 1998/02/24 21:17:04 wessels Exp $ + * $Id: gopher.cc,v 1.120 1998/03/03 00:31:06 rousskov Exp $ * * DEBUG: section 10 Gopher * AUTHOR: Harvest Derived @@ -680,7 +680,7 @@ gopherReadReply(int fd, void *data) } /* check if we want to defer reading */ clen = entry->mem_obj->inmem_hi; - buf = memAllocate(MEM_4K_BUF, 1); + buf = memAllocate(MEM_4K_BUF); errno = 0; /* leave one space for \0 in gopherToHTML */ len = read(fd, buf, TEMP_BUF_SIZE - 1); @@ -809,7 +809,7 @@ gopherSendRequest(int fd, void *data) { GopherStateData *gopherState = data; LOCAL_ARRAY(char, query, MAX_URL); - char *buf = memAllocate(MEM_4K_BUF, 1); + char *buf = memAllocate(MEM_4K_BUF); char *t; if (gopherState->type_id == GOPHER_CSO) { sscanf(gopherState->request, "?%s", query); @@ -931,7 +931,7 @@ CreateGopherStateData(void) { GopherStateData *gd = xcalloc(1, sizeof(GopherStateData)); cbdataAdd(gd, MEM_NONE); - gd->buf = memAllocate(MEM_4K_BUF, 1); + gd->buf = memAllocate(MEM_4K_BUF); return (gd); } diff --git a/src/http.cc b/src/http.cc index 9bba73a3ed..bd2dcdd4c3 100644 --- a/src/http.cc +++ b/src/http.cc @@ -1,6 +1,6 @@ /* - * $Id: http.cc,v 1.242 1998/02/25 09:53:58 rousskov Exp $ + * $Id: http.cc,v 1.243 1998/03/03 00:31:07 rousskov Exp $ * * DEBUG: section 11 Hypertext Transfer Protocol (HTTP) * AUTHOR: Harvest Derived @@ -303,7 +303,7 @@ httpCacheNegatively(StoreEntry * entry) void httpParseReplyHeaders(const char *buf, struct _http_reply *reply) { - char *headers = memAllocate(MEM_4K_BUF, 1); + char *headers = memAllocate(MEM_4K_BUF); char *line; char *end; char *s = NULL; @@ -327,7 +327,7 @@ httpParseReplyHeaders(const char *buf, struct _http_reply *reply) return; } reply->hdr_sz = end - headers; - line = memAllocate(MEM_4K_BUF, 1); + line = memAllocate(MEM_4K_BUF); for (s = headers; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { l = strcspn(s, crlf) + 1; if (l > 4096) @@ -526,7 +526,7 @@ httpProcessReplyHeader(HttpStateData * httpState, const char *buf, int size) debug(11, 3) ("httpProcessReplyHeader: key '%s'\n", storeKeyText(entry->key)); if (httpState->reply_hdr == NULL) - httpState->reply_hdr = memAllocate(MEM_8K_BUF, 1); + httpState->reply_hdr = memAllocate(MEM_8K_BUF); if (httpState->reply_hdr_state == 0) { hdr_len = strlen(httpState->reply_hdr); room = 8191 - hdr_len; @@ -796,9 +796,9 @@ httpBuildRequestHeader(request_t * request, { LOCAL_ARRAY(char, ybuf, YBUF_SZ); LOCAL_ARRAY(char, no_forward, 1024); - char *xbuf = memAllocate(MEM_4K_BUF, 1); - char *viabuf = memAllocate(MEM_4K_BUF, 1); - char *fwdbuf = memAllocate(MEM_4K_BUF, 1); + char *xbuf = memAllocate(MEM_4K_BUF); + char *viabuf = memAllocate(MEM_4K_BUF); + char *fwdbuf = memAllocate(MEM_4K_BUF); char *t = NULL; char *s = NULL; char *end = NULL; @@ -946,7 +946,7 @@ httpSendRequest(int fd, void *data) return; } if (buflen < DISK_PAGE_SIZE) { - buf = memAllocate(MEM_8K_BUF, 1); + buf = memAllocate(MEM_8K_BUF); buftype = BUF_TYPE_8K; buflen = DISK_PAGE_SIZE; } else { @@ -1021,7 +1021,7 @@ httpBuildState(int fd, StoreEntry * entry, request_t * orig_request, peer * e) httpState->entry = entry; httpState->fd = fd; if (e) { - request = memAllocate(MEM_REQUEST_T, 1); + request = memAllocate(MEM_REQUEST_T); request->method = orig_request->method; xstrncpy(request->host, e->host, SQUIDHOSTNAMELEN); request->port = e->http_port; diff --git a/src/icmp.cc b/src/icmp.cc index 74cb82483e..b10792114a 100644 --- a/src/icmp.cc +++ b/src/icmp.cc @@ -1,6 +1,6 @@ /* - * $Id: icmp.cc,v 1.55 1998/02/03 04:21:16 wessels Exp $ + * $Id: icmp.cc,v 1.56 1998/03/03 00:31:08 rousskov Exp $ * * DEBUG: section 37 ICMP Routines * AUTHOR: Duane Wessels @@ -203,7 +203,7 @@ icmpSourcePing(struct in_addr to, const icp_common_t * header, const char *url) debug(37, 3) ("icmpSourcePing: '%s'\n", url); if ((ulen = strlen(url)) > MAX_URL) return; - payload = memAllocate(MEM_8K_BUF, 1); + payload = memAllocate(MEM_8K_BUF); len = sizeof(icp_common_t); xmemcpy(payload, header, len); strcpy(payload + len, url); diff --git a/src/main.cc b/src/main.cc index 2243acb55c..afd7b995eb 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1,6 +1,6 @@ /* - * $Id: main.cc,v 1.230 1998/02/27 09:07:24 kostas Exp $ + * $Id: main.cc,v 1.231 1998/03/03 00:31:08 rousskov Exp $ * * DEBUG: section 1 Startup and Main Loop * AUTHOR: Harvest Derived @@ -745,6 +745,7 @@ normal_shutdown(void) ipcacheFreeMemory(); fqdncacheFreeMemory(); asnFreeMemory(); + httpHeaderCleanModule(); #endif #if WHY_DO_THIS file_close(0); @@ -753,6 +754,7 @@ normal_shutdown(void) #endif fdDumpOpen(); fdFreeMemory(); + memClean(); #if XMALLOC_TRACE { extern int xmalloc_total; diff --git a/src/mem.cc b/src/mem.cc index b1ea7000b2..a121aa785c 100644 --- a/src/mem.cc +++ b/src/mem.cc @@ -1,8 +1,8 @@ /* - * $Id: mem.cc,v 1.7 1998/02/26 18:00:46 wessels Exp $ + * $Id: mem.cc,v 1.8 1998/03/03 00:31:09 rousskov Exp $ * - * DEBUG: section 13 Memory Pool Management + * DEBUG: section 13 High Level Memory Pool Management * AUTHOR: Harvest Derived * * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ @@ -31,122 +31,72 @@ #include "squid.h" -#define stackSize(S) ((S)->top - (S)->base) +/* module globals */ -typedef struct { - void **base; - void **top; - int max_size; -} Stack; - -typedef struct { - char *name; - int n_allocated; - size_t size; - int n_used; - Stack Stack; -} memData; - -static memData MemData[MEM_MAX]; - -static void *stackPop(Stack * s); -static int stackFull(Stack * s); -static int stackEmpty(Stack * s); -static void stackPush(Stack * s, void *p); -static void memDataInit(mem_type, const char *, size_t, int); -static OBJH memStats; - -static int -stackEmpty(Stack * s) -{ - return s->top == s->base; -} - -static int -stackFull(Stack * s) -{ - return (stackSize(s) == s->max_size); -} - -static void * -stackPop(Stack * s) -{ - void *p; - assert(s->top != s->base); - s->top--; - p = *s->top; - *s->top = NULL; - return p; -} +static MemPool *MemPools[MEM_MAX]; +/* all pools share common memory chunks so it is probably better to ignore max_pages */ static void -stackPush(Stack * s, void *p) +memDataInit(mem_type type, const char *name, size_t size, int max_pages_notused) { - if (stackSize(s) == s->max_size) { - xfree(p); - } else { - *s->top = p; - s->top++; - } + assert(name && size); + MemPools[type] = memPoolCreate(name, size); } static void -memDataInit(mem_type type, const char *name, size_t size, int max_pages) +memStats(StoreEntry * sentry) { - memData *m = &MemData[type]; - m->size = size; - m->name = xstrdup(name); -#if !PURIFY - if (Config.onoff.mem_pools) { - m->Stack.max_size = max_pages; - m->Stack.base = xcalloc(max_pages, sizeof(void **)); - m->Stack.top = m->Stack.base; + mem_type t; + storeBuffer(sentry); + storeAppendPrintf(sentry, "%-20s\t %s\t %s\t %s\t %s\t %s\t %s\t %s\t %s\t %s\t %s\n", + "Pool", "Obj Size", + "Capacity (#)", "Capacity (KB)", "Used (KB)", "HWater (KB)", + "Util (%)", "Grow Count", + "Malloc (#)", "Malloc (KB)", "MHWater (KB)"); + for (t = MEM_NONE + 1; t < MEM_MAX; t++) { + const MemPool *pool = MemPools[t]; + if (!memPoolWasNeverUsed(pool)) + memPoolReport(pool, sentry); } -#endif + storeAppendPrintf(sentry, "\n"); + /* memStringStats(sentry); */ + memReportTotals(sentry); + storeBufferFlush(sentry); } + + /* * PUBLIC ROUTINES */ +/* find appropriate pool and use it (pools always init buffer with 0s) */ void * -memAllocate(mem_type type, int clear) +memAllocate(mem_type type) { - void *p = NULL; - memData *m = &MemData[type]; - if (!stackEmpty(&m->Stack)) { - p = stackPop(&m->Stack); - assert(p != NULL); - } else { - p = xmalloc(m->size); - m->n_allocated++; - } - m->n_used++; - if (clear) - memset(p, '\0', m->size); - return p; + return memPoolAlloc(MemPools[type]); } +/* find appropriate pool and use it */ void memFree(mem_type type, void *p) { - memData *m = &MemData[type]; - assert(p != NULL); - m->n_used--; - if (stackFull(&m->Stack)) { - xfree(p); - m->n_allocated--; - } else { - stackPush(&m->Stack, p); - } + memPoolFree(MemPools[type], p); } void memInit(void) { mem_type t; - memData *m; - memset(MemData, '\0', MEM_MAX * sizeof(memData)); + memInitModule(); + /* set all pointers to null */ + memset(MemPools, '\0', sizeof(MemPools)); + /* + * it does not hurt much to have a lot of pools since sizeof(MemPool) is + * small; someday we will figure out what to do with all the entries here + * that are never used or used only once; perhaps we should simply use + * malloc() for those? @?@ + */ memDataInit(MEM_4K_BUF, "4K Buffer", 4096, 10); memDataInit(MEM_8K_BUF, "8K Buffer", 8192, 10); memDataInit(MEM_ACCESSLOGENTRY, "AccessLogEntry", @@ -227,13 +177,13 @@ memInit(void) memDataInit(MEM_SWAPDIR, "SwapDir", sizeof(SwapDir), 0); memDataInit(MEM_USHORTLIST, "ushort_list", sizeof(ushortlist), 0); memDataInit(MEM_WORDLIST, "wordlist", sizeof(wordlist), 0); + /* test that all entries are initialized */ for (t = MEM_NONE + 1; t < MEM_MAX; t++) { - m = &MemData[t]; /* * If you hit this assertion, then you forgot to add a * memDataInit() line for type 't' above. */ - assert(m->size); + assert(MemPools[t]); } cachemgrRegister("mem", "Memory Utilization", @@ -241,64 +191,28 @@ memInit(void) } void -memFreeMemory(void) +memClean() { mem_type t; - memData *m; - void *p; + int dirty_count = 0; for (t = MEM_NONE + 1; t < MEM_MAX; t++) { - m = &MemData[t]; - while (!stackEmpty(&m->Stack)) { - p = stackPop(&m->Stack); - xfree(p); + MemPool *pool = MemPools[t]; + if (memPoolIsUsedNow(pool)) { + memPoolDescribe(pool); + dirty_count++; } - xfree(m->Stack.base); } + if (dirty_count) + debug(13, 2) ("memClean: %d pools are left dirty\n", dirty_count); + else + memCleanModule(); /* will free chunks and stuff */ } -static void -memStats(StoreEntry * sentry) -{ - mem_type t; - memData *m; - size_t in_use = 0; - size_t not_in_use = 0; - size_t allocated = 0; - storeBuffer(sentry); - storeAppendPrintf(sentry, "%25s %6s %15s %15s\n", - "NAME", - "SIZE", - "NOT-USED", - "ALLOCATED"); - for (t = MEM_NONE + 1; t < MEM_MAX; t++) { - m = &MemData[t]; - if (m->n_allocated == 0) - continue; - storeAppendPrintf(sentry, "%25.25s %6d %6d %5d KB %6d %5d KB\n", - m->name, - m->size, - stackSize(&m->Stack), - m->size * stackSize(&m->Stack) >> 10, - m->n_allocated, - m->size * m->n_allocated >> 10); - in_use += m->size * m->n_used; - not_in_use += m->size * stackSize(&m->Stack); - allocated += m->size * m->n_allocated; - } - storeAppendPrintf(sentry, "\n"); - storeAppendPrintf(sentry, "Total Memory In Use %6d KB\n", - (int) in_use >> 10); - storeAppendPrintf(sentry, "Total Memory Not In Use %6d KB\n", - (int) not_in_use >> 10); - storeAppendPrintf(sentry, "Total Memory Allocated %6d KB\n", - (int) allocated >> 10); - storeBufferFlush(sentry); -} int memInUse(mem_type type) { - return MemData[type].n_used; + return memPoolUsedCount(MemPools[type]); } /* ick */ diff --git a/src/mime.cc b/src/mime.cc index ce9d59ac23..aca5309a1a 100644 --- a/src/mime.cc +++ b/src/mime.cc @@ -1,6 +1,6 @@ /* - * $Id: mime.cc,v 1.52 1998/02/26 18:00:47 wessels Exp $ + * $Id: mime.cc,v 1.53 1998/03/03 00:31:10 rousskov Exp $ * * DEBUG: section 25 MIME Parsing * AUTHOR: Harvest Derived @@ -454,7 +454,7 @@ mimeLoadIconFile(const char *icon) assert(e != NULL); e->mem_obj->request = requestLink(urlParse(METHOD_GET, url)); #if 0 /* use new interface */ - buf = memAllocate(MEM_4K_BUF, 1); + buf = memAllocate(MEM_4K_BUF); l = 0; l += snprintf(buf + l, SM_PAGE_SIZE - l, "HTTP/1.0 200 OK\r\n"); l += snprintf(buf + l, SM_PAGE_SIZE - l, "Date: %s\r\n", mkrfc1123(squid_curtime)); @@ -472,7 +472,7 @@ mimeLoadIconFile(const char *icon) type, (int) sb.st_size, sb.st_mtime, squid_curtime + 86400); httpReplySwapOut(e->mem_obj->reply, e); /* read the file into the buffer and append it to store */ - buf = memAllocate(MEM_4K_BUF, 1); + buf = memAllocate(MEM_4K_BUF); #endif while ((n = read(fd, buf, 4096)) > 0) storeAppend(e, buf, n); diff --git a/src/net_db.cc b/src/net_db.cc index 3a532fab53..79a4fd3506 100644 --- a/src/net_db.cc +++ b/src/net_db.cc @@ -1,6 +1,6 @@ /* - * $Id: net_db.cc,v 1.71 1998/02/26 22:16:29 kostas Exp $ + * $Id: net_db.cc,v 1.72 1998/03/03 00:31:10 rousskov Exp $ * * DEBUG: section 37 Network Measurement Database * AUTHOR: Duane Wessels @@ -82,7 +82,7 @@ netdbHashDelete(const char *key) static void netdbHostInsert(netdbEntry * n, const char *hostname) { - net_db_name *x = memAllocate(MEM_NET_DB_NAME, 1); + net_db_name *x = memAllocate(MEM_NET_DB_NAME); x->name = xstrdup(hostname); x->next = n->hosts; n->hosts = x; @@ -197,7 +197,7 @@ netdbAdd(struct in_addr addr) if (memInUse(MEM_NETDBENTRY) > Config.Netdb.high) netdbPurgeLRU(); if ((n = netdbLookupAddr(addr)) == NULL) { - n = memAllocate(MEM_NETDBENTRY, 1); + n = memAllocate(MEM_NETDBENTRY); netdbHashInsert(n, addr); } return n; @@ -393,7 +393,7 @@ static void netdbReloadState(void) { LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); - char *buf = memAllocate(MEM_4K_BUF, 1); + char *buf = memAllocate(MEM_4K_BUF); char *t; FILE *fp; netdbEntry *n; @@ -436,7 +436,7 @@ netdbReloadState(void) if ((t = strtok(NULL, w_space)) == NULL) continue; N.last_use_time = (time_t) atoi(t); - n = memAllocate(MEM_NETDBENTRY, 1); + n = memAllocate(MEM_NETDBENTRY); xmemcpy(n, &N, sizeof(netdbEntry)); netdbHashInsert(n, addr); while ((t = strtok(NULL, w_space)) != NULL) { @@ -636,6 +636,9 @@ netdbDump(StoreEntry * sentry) p->rtt, p->hops); } + /* put a new line if no peers */ + if (!n->n_peers) + storeAppendPrintf(sentry, "\n"); } xfree(list); #else diff --git a/src/protos.h b/src/protos.h index 6ffaba66d1..9ae9fb99b0 100644 --- a/src/protos.h +++ b/src/protos.h @@ -259,6 +259,7 @@ extern void httpBodyPackInto(const HttpBody * body, Packer * p); /* Http Header */ extern void httpHeaderInitModule(); +extern void httpHeaderCleanModule(); /* create/init/clean/destroy */ extern HttpHeader *httpHeaderCreate(); extern void httpHeaderInit(HttpHeader * hdr); @@ -506,14 +507,35 @@ void statHistEnumInit(StatHist * H, int last_enum); extern void memInit(void); +extern void memClean(); +extern void memInitModule(); +extern void memCleanModule(); +extern void memConfigure(); +#if 0 /* not used */ extern void memFreeMemory(void); -extern void *memAllocate(mem_type, int); +#endif +extern void *memAllocate(mem_type); extern void memFree(mem_type, void *); extern void memFree4K(void *); extern void memFree8K(void *); extern void memFreeDISK(void *); extern int memInUse(mem_type); +extern DynPool *dynPoolCreate(); +extern void dynPoolDestroy(DynPool *pool); +extern void *dynPoolAlloc(DynPool *pool, size_t size); +extern void dynPoolFree(DynPool *pool, void *obj, size_t size); +extern MemPool *memPoolCreate(const char *label, size_t obj_size); +extern void memPoolDestroy(MemPool *pool); +extern void *memPoolAlloc(MemPool *pool); +extern void memPoolFree(MemPool *pool, void *obj); +extern int memPoolWasNeverUsed(const MemPool *pool); +extern int memPoolIsUsedNow(const MemPool *pool); +extern int memPoolUsedCount(const MemPool *pool); +extern void memPoolDescribe(const MemPool *pool); +extern void memPoolReport(const MemPool *pool, StoreEntry *e); +extern void memReportTotals(StoreEntry *e); + extern int stmemFreeDataUpto(mem_hdr *, int); extern void stmemAppend(mem_hdr *, const char *, int); extern ssize_t stmemCopy(const mem_hdr *, off_t, char *, size_t); diff --git a/src/redirect.cc b/src/redirect.cc index 2134d036a0..401badc233 100644 --- a/src/redirect.cc +++ b/src/redirect.cc @@ -1,6 +1,6 @@ /* - * $Id: redirect.cc,v 1.56 1998/02/19 23:09:59 wessels Exp $ + * $Id: redirect.cc,v 1.57 1998/03/03 00:31:12 rousskov Exp $ * * DEBUG: section 29 Redirector * AUTHOR: Duane Wessels @@ -222,7 +222,7 @@ redirectDispatch(redirector_t * redirect, redirectStateData * r) redirect->dispatch_time = current_time; if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL) fqdn = dash_str; - buf = memAllocate(MEM_8K_BUF, 1); + buf = memAllocate(MEM_8K_BUF); snprintf(buf, 8192, "%s %s/%s %s %s\n", r->orig_url, inet_ntoa(r->client_addr), @@ -329,7 +329,7 @@ redirectOpenServers(void) EBIT_SET(redirect_child_table[k]->flags, HELPER_ALIVE); redirect_child_table[k]->index = k; redirect_child_table[k]->fd = redirectsocket; - redirect_child_table[k]->inbuf = memAllocate(MEM_8K_BUF, 1); + redirect_child_table[k]->inbuf = memAllocate(MEM_8K_BUF); redirect_child_table[k]->size = 8192; redirect_child_table[k]->offset = 0; if ((s = strrchr(prg, '/'))) diff --git a/src/stmem.cc b/src/stmem.cc index 02cf079152..7c0495f445 100644 --- a/src/stmem.cc +++ b/src/stmem.cc @@ -1,6 +1,6 @@ /* - * $Id: stmem.cc,v 1.58 1998/02/10 22:17:55 wessels Exp $ + * $Id: stmem.cc,v 1.59 1998/03/03 00:31:13 rousskov Exp $ * * DEBUG: section 19 Store Memory Primitives * AUTHOR: Harvest Derived @@ -188,7 +188,7 @@ stmemAppend(mem_hdr * mem, const char *data, int len) p = xcalloc(1, sizeof(mem_node)); p->next = NULL; p->len = len_to_copy; - p->data = memAllocate(MEM_STMEM_BUF, 1); + p->data = memAllocate(MEM_STMEM_BUF); store_mem_size += SM_PAGE_SIZE; xmemcpy(p->data, data, len_to_copy); if (!mem->head) { diff --git a/src/store.cc b/src/store.cc index 9ff1eee90b..d33a0efe9e 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.387 1998/02/26 18:00:55 wessels Exp $ + * $Id: store.cc,v 1.388 1998/03/03 00:31:14 rousskov Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -175,7 +175,7 @@ static int store_maintain_buckets; static MemObject * new_MemObject(const char *url, const char *log_url) { - MemObject *mem = memAllocate(MEM_MEMOBJECT, 1); + MemObject *mem = memAllocate(MEM_MEMOBJECT); mem->reply = httpReplyCreate(); mem->url = xstrdup(url); mem->log_url = xstrdup(log_url); @@ -190,7 +190,7 @@ StoreEntry * new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) { StoreEntry *e = NULL; - e = memAllocate(MEM_STOREENTRY, 1); + e = memAllocate(MEM_STOREENTRY); if (mem_obj_flag) e->mem_obj = new_MemObject(url, log_url); debug(20, 3) ("new_StoreEntry: returning %p\n", e); @@ -428,7 +428,7 @@ storeCreateEntry(const char *url, const char *log_url, int flags, method_t metho storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_NONE; e->swap_file_number = -1; - mem->data = memAllocate(MEM_MEM_HDR, 1); + mem->data = memAllocate(MEM_MEM_HDR); e->refcount = 0; e->lastref = squid_curtime; e->timestamp = 0; /* set in storeTimestampsSet() */ diff --git a/src/store_client.cc b/src/store_client.cc index 450ad3a2f7..fb2a1e607c 100644 --- a/src/store_client.cc +++ b/src/store_client.cc @@ -47,7 +47,7 @@ storeClientListAdd(StoreEntry * e, void *data) if (storeClientListSearch(mem, data) != NULL) return; mem->nclients++; - sc = memAllocate(MEM_STORE_CLIENT, 1); + sc = memAllocate(MEM_STORE_CLIENT); cbdataAdd(sc, MEM_STORE_CLIENT); /* sc is callback_data for file_read */ sc->callback_data = data; sc->seen_offset = 0; diff --git a/src/store_swapout.cc b/src/store_swapout.cc index 1815a5c12e..a0a9d2d77c 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -164,7 +164,7 @@ storeCheckSwapOut(StoreEntry * e) assert(mem->swapout.fd > -1); if (swapout_size > STORE_SWAP_BUF) swapout_size = STORE_SWAP_BUF; - swap_buf = memAllocate(MEM_DISK_BUF, 1); + swap_buf = memAllocate(MEM_DISK_BUF); swap_buf_len = stmemCopy(mem->data, mem->swapout.queue_offset, swap_buf, diff --git a/src/structs.h b/src/structs.h index a5616025d4..65514c8434 100644 --- a/src/structs.h +++ b/src/structs.h @@ -332,6 +332,9 @@ struct _SquidConfig { time_t timeout; int maxtries; } retry; + struct { + size_t limit; + } MemPools; }; struct _SquidConfig2 { diff --git a/src/typedefs.h b/src/typedefs.h index e150e95a7b..3fdf0158c8 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -63,6 +63,7 @@ typedef struct _ipcache_addrs ipcache_addrs; typedef struct _ipcache_entry ipcache_entry; typedef struct _domain_ping domain_ping; typedef struct _domain_type domain_type; +typedef struct _DynPool DynPool; typedef struct _Packer Packer; typedef struct _peer peer; typedef struct _net_db_name net_db_name; @@ -81,6 +82,7 @@ typedef struct _mem_node mem_node; typedef struct _mem_hdr mem_hdr; typedef struct _store_client store_client; typedef struct _MemObject MemObject; +typedef struct _MemPool MemPool; typedef struct _StoreEntry StoreEntry; typedef struct _SwapDir SwapDir; typedef struct _request_t request_t; diff --git a/src/url.cc b/src/url.cc index 48b08bba54..600c54f4d3 100644 --- a/src/url.cc +++ b/src/url.cc @@ -1,6 +1,6 @@ /* - * $Id: url.cc,v 1.81 1998/02/26 18:00:59 wessels Exp $ + * $Id: url.cc,v 1.82 1998/03/03 00:31:17 rousskov Exp $ * * DEBUG: section 23 URL Parsing * AUTHOR: Duane Wessels @@ -267,7 +267,7 @@ urlParse(method_t method, char *url) return NULL; } #endif - request = memAllocate(MEM_REQUEST_T, 1); + request = memAllocate(MEM_REQUEST_T); request->method = method; request->protocol = protocol; xstrncpy(request->host, host, SQUIDHOSTNAMELEN); @@ -284,7 +284,7 @@ urnParse(method_t method, char *urn) { request_t *request = NULL; debug(50, 5) ("urnParse: %s\n", urn); - request = memAllocate(MEM_REQUEST_T, 1); + request = memAllocate(MEM_REQUEST_T); request->method = method; request->protocol = PROTO_URN; xstrncpy(request->urlpath, &urn[4], MAX_URL); diff --git a/src/urn.cc b/src/urn.cc index c3890734e7..a22c762d0a 100644 --- a/src/urn.cc +++ b/src/urn.cc @@ -139,7 +139,7 @@ urnStart(request_t * r, StoreEntry * e) 0, 0, 4096, - memAllocate(MEM_4K_BUF, 1), + memAllocate(MEM_4K_BUF), urnHandleReply, urnState); }