From: rousskov <> Date: Tue, 14 Apr 1998 21:16:23 +0000 (+0000) Subject: - digest size is now calculated using both maximum cache size X-Git-Tag: SQUID_3_0_PRE1~3504 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=304b267e28e9a4fe51ee341a306e267b083b9641;p=thirdparty%2Fsquid.git - digest size is now calculated using both maximum cache size and currently used size: the capacity is set to 1.5*#store_entries and then fitted inside [10%,100%] of the max number of entries in the store hash size = FudgeFactor * capacity; - added debug to report who is requesting digests from us current level is 1 we probably want to maintain a list (like the "bad peers sending IPC" list) rather then reporting it via cache.log - maybe-fix to determine whether we should call StoreComplete() in peer_digest.c (reused old entryXor not?). Looks ugly though.X --- diff --git a/src/CacheDigest.cc b/src/CacheDigest.cc index 9e0b0ac33d..43fd747960 100644 --- a/src/CacheDigest.cc +++ b/src/CacheDigest.cc @@ -1,6 +1,6 @@ /* - * $Id: CacheDigest.cc,v 1.11 1998/04/07 23:23:36 rousskov Exp $ + * $Id: CacheDigest.cc,v 1.12 1998/04/14 15:16:23 rousskov Exp $ * * DEBUG: section 70 Cache Digest * AUTHOR: Alex Rousskov @@ -41,6 +41,7 @@ typedef struct { /* local functions */ static void cacheDigestHashKey(const CacheDigest *cd, const cache_key *key); +static size_t cacheDigestCalcMaskSize(int cap); /* configuration params */ static const int BitsPerEntry = 4; @@ -52,7 +53,7 @@ static u_num32 hashed_keys[4]; CacheDigest * cacheDigestCreate(int capacity) { - const size_t mask_size = (size_t) (capacity * BitsPerEntry + 7) / 8; + const size_t mask_size = cacheDigestCalcMaskSize(capacity); CacheDigest *cd = cacheDigestSizedCreate(mask_size, capacity); return cd; } @@ -97,6 +98,17 @@ cacheDigestClear(CacheDigest * cd) memset(cd->mask, 0, cd->mask_size); } +void +cacheDigestChangeCap(CacheDigest * cd, int new_cap) +{ + assert(cd); + /* have to clear because capacity changes hash functions */ + cacheDigestClear(cd); + cd->capacity = new_cap; + cd->mask_size = cacheDigestCalcMaskSize(new_cap); + cd->mask = xrealloc(cd->mask, cd->mask_size); +} + /* returns true if the key belongs to the digest */ int cacheDigestTest(const CacheDigest * cd, const cache_key * key) @@ -238,6 +250,12 @@ cacheDigestReport(CacheDigest *cd, const char *label, StoreEntry * e) ); } +static size_t +cacheDigestCalcMaskSize(int cap) +{ + return (size_t) (cap * BitsPerEntry + 7) / 8; +} + static void cacheDigestHashKey(const CacheDigest *cd, const cache_key *key) { diff --git a/src/client_side.cc b/src/client_side.cc index 88348c0771..a54b844b5f 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.276 1998/04/12 06:13:56 rousskov Exp $ + * $Id: client_side.cc,v 1.277 1998/04/14 15:16:24 rousskov Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -587,6 +587,8 @@ clientUpdateCounters(clientHttpRequest * http) if (http->internal && strStr(http->request->urlpath, StoreDigestUrlPath)) { kb_incr(&Counter.cd.kbytes_sent, http->out.size); Counter.cd.msgs_sent++; + debug(33, 1) ("Client %s requested local cache digest (%d bytes)\n", + inet_ntoa(http->request->client_addr), http->out.size); } /* @?@ split this ugly if-monster */ if (/* we used ICP or CD for peer selecton */ @@ -600,6 +602,7 @@ clientUpdateCounters(clientHttpRequest * http) EBIT_TEST(http->request->flags, REQ_CACHABLE) && /* paranoid: we have a reply pointer */ (reply = storeEntryReply(http->entry))) { + const char *x_cache_fld = httpHeaderGetLastStr(&reply->header, HDR_X_CACHE); const int real_hit = x_cache_fld && !strncmp(x_cache_fld, "HIT", 3); const int guess_hit = LOOKUP_HIT == H->cd_lookup; @@ -612,7 +615,7 @@ clientUpdateCounters(clientHttpRequest * http) cacheDigestGuessStatsUpdate(&peer->digest.stats.guess, real_hit, guess_hit); } else { - /* temporary paranoid debug */ + /* temporary paranoid debug @?@ */ static int max_count = 200; if (max_count > 0) { debug(33,1) ("clientUpdateCounters: lost peer %s for %s! (%d)\n", diff --git a/src/peer_digest.cc b/src/peer_digest.cc index 313fb985bf..f17644283b 100644 --- a/src/peer_digest.cc +++ b/src/peer_digest.cc @@ -1,6 +1,6 @@ /* - * $Id: peer_digest.cc,v 1.13 1998/04/12 06:10:06 rousskov Exp $ + * $Id: peer_digest.cc,v 1.14 1998/04/14 15:16:25 rousskov Exp $ * * DEBUG: section 72 Peer Digest Routines * AUTHOR: Alex Rousskov @@ -58,7 +58,7 @@ static int peerDigestSetCBlock(peer *p, const char *buf); #define StoreDigestCBlockSize sizeof(StoreDigestCBlock) /* min interval for requesting digests from the same peer */ -static const time_t PeerDigestRequestMinGap = 10 * 60; /* seconds */ +static const time_t PeerDigestRequestMinGap = 15 * 60; /* seconds */ /* min interval for requesting digests at start */ static const time_t GlobalDigestRequestMinGap = 1 * 60; /* seconds */ @@ -526,7 +526,8 @@ peerDigestFetchFinish(DigestFetchState *fetch, char *buf, const char *err_msg) /* release buggy entry */ storeReleaseRequest(fetch->entry); } else { - if (fetch->entry->store_status == STORE_OK) { + /* ugly condition, but how? @?@ @?@ */ + if (fetch->entry->mem_obj->reply->sline.status == HTTP_NOT_MODIFIED) { debug(72, 2) ("re-used old digest from %s\n", peer->host); } else { debug(72, 2) ("received valid digest from %s\n", peer->host); @@ -538,8 +539,8 @@ peerDigestFetchFinish(DigestFetchState *fetch, char *buf, const char *err_msg) peerDigestDelay(peer, 0, max_delay(peerDigestExpiresDelay(peer, fetch->entry), 0)); } - /* note: outgoing numbers are not precise! @?@ */ /* update global stats */ + /* note: outgoing numbers are not precise! @?@ */ kb_incr(&Counter.cd.kbytes_sent, req->headers_sz); kb_incr(&Counter.cd.kbytes_recv, (size_t)b_read); Counter.cd.msgs_sent++; diff --git a/src/protos.h b/src/protos.h index 6f37b8c88f..1a188b27e0 100644 --- a/src/protos.h +++ b/src/protos.h @@ -913,6 +913,7 @@ extern CacheDigest *cacheDigestSizedCreate(size_t size, int capacity); extern void cacheDigestDestroy(CacheDigest * cd); extern CacheDigest *cacheDigestClone(const CacheDigest * cd); extern void cacheDigestClear(CacheDigest * cd); +extern void cacheDigestChangeCap(CacheDigest * cd, int new_cap); extern int cacheDigestTest(const CacheDigest * cd, const cache_key * key); extern void cacheDigestAdd(CacheDigest * cd, const cache_key * key); extern void cacheDigestDel(CacheDigest * cd, const cache_key * key); diff --git a/src/store_digest.cc b/src/store_digest.cc index 6373dcd714..ba05fa4b58 100644 --- a/src/store_digest.cc +++ b/src/store_digest.cc @@ -1,5 +1,5 @@ /* - * $Id: store_digest.cc,v 1.6 1998/04/08 22:52:38 rousskov Exp $ + * $Id: store_digest.cc,v 1.7 1998/04/14 15:16:26 rousskov Exp $ * * DEBUG: section 71 Store Digest Manager * AUTHOR: Alex Rousskov @@ -55,6 +55,8 @@ static const time_t StoreDigestRewritePeriod = 60 * 60; static const int StoreDigestSwapOutChunkSize = SM_PAGE_SIZE; /* portion (0,1] of a hash table to be rescanned at a time */ static const double StoreDigestRebuildChunkPercent = 0.10; +/* Fudge Factor for sizing the digest */ +static const double StoreDigestFudgeFactor = 1.5; /* local vars */ static StoreDigestState sd_state; @@ -67,22 +69,19 @@ static void storeDigestRewrite(); static void storeDigestRewriteFinish(StoreEntry * e); static void storeDigestSwapOutStep(StoreEntry * e); static void storeDigestCBlockSwapOut(StoreEntry * e); +static int storeDigestCalcCap(); +static int storeDigestResize(); void storeDigestInit() { - /* - * To-Do: Bloom proved that the optimal filter utilization is 50% (half of - * the bits are off). However, we do not have a formula to calculate the - * number of _entries_ we want to pre-allocate for. - * Use 1.5*max#entries because 2*max#entries gives about 40% utilization. - */ #if SQUID_MAINTAIN_CACHE_DIGEST - const int cap = (int) (1.5 * Config.Swap.maxSize / Config.Store.avgObjectSize); + const int cap = storeDigestCalcCap(); store_digest = cacheDigestCreate(cap); - debug(71, 1) ("Using %d byte cache digest; rebuild/rewrite every %d/%d sec\n", - store_digest->mask_size, StoreDigestRebuildPeriod, StoreDigestRewritePeriod); + debug(71, 1) ("Using %d byte cache digest (%d entries); rebuild/rewrite every %d/%d sec\n", + store_digest->mask_size, store_digest->capacity, + StoreDigestRebuildPeriod, StoreDigestRewritePeriod); #else store_digest = NULL; debug(71, 1) ("Local cache digest is 'off'\n"); @@ -138,8 +137,9 @@ storeDigestRebuild(void *datanotused) } sd_state.rebuild_lock = 1; sd_state.rebuild_offset = 0; - /* not clean()! */ - cacheDigestClear(store_digest); + /* resize or clear */ + if (!storeDigestResize()) + cacheDigestClear(store_digest); /* not clean()! */ debug(71, 2) ("storeDigestRebuild: start rebuild #%d\n", sd_state.rebuild_count + 1); storeDigestRebuildStep(NULL); } @@ -286,6 +286,48 @@ storeDigestCBlockSwapOut(StoreEntry * e) storeAppend(e, (char*) &sd_state.cblock, sizeof(sd_state.cblock)); } +/* calculates digest capacity */ +static int +storeDigestCalcCap() +{ + /* + * To-Do: Bloom proved that the optimal filter utilization is 50% (half of + * the bits are off). However, we do not have a formula to calculate the + * number of _entries_ we want to pre-allocate for. + */ + const int hi_cap = store_table->size; + const int lo_cap = hi_cap/5; + int cap = (int) (1.5 * memInUse(MEM_STOREENTRY)); + if (cap < lo_cap) + cap = lo_cap; + if (cap > hi_cap) + cap = hi_cap; + cap = 1 + (int) (StoreDigestFudgeFactor * cap); + return cap; +} + +/* returns true if we actually resized the digest */ +static int +storeDigestResize() +{ + const int cap = storeDigestCalcCap(); + int diff; + assert(store_digest); + diff = abs(cap - store_digest->capacity); + debug(71, 2) ("storeDigestResize: %d -> %d; change: %d (%d%%)\n", + store_digest->capacity, cap, diff, + xpercentInt(diff, store_digest->capacity)); + /* avoid minor adjustments */ + if (diff <= store_digest->capacity/10) { + debug(71, 1) ("storeDigestResize: small change, will not resize.\n"); + return 0; /* at most 10% change */ + } else { + debug(71, 1) ("storeDigestResize: big change, resizing.\n"); + cacheDigestChangeCap(store_digest, cap); + return 1; + } +} + void storeDigestReport(StoreEntry * e) {