/*
- * $Id: store_digest.cc,v 1.8 1998/04/22 05:54:28 rousskov Exp $
+ * $Id: store_digest.cc,v 1.9 1998/04/22 16:22:58 rousskov Exp $
*
* DEBUG: section 71 Store Digest Manager
* AUTHOR: Alex Rousskov
*
*/
+
+/*
+ * TODO: We probably do not track all the cases when
+ * storeDigestNoteStoreReady() must be called; this may prevent
+ * storeDigestRebuild/write schedule to be activated
+ */
+
#include "squid.h"
/* local types */
StoreDigestCBlock cblock;
int rebuild_lock; /* bucket number */
StoreEntry *rewrite_lock; /* store entry with the digest */
- const char *other_lock; /* used buy external modules to pause rebuilds and rewrites */
int rebuild_offset;
int rewrite_offset;
int rebuild_count;
int rewrite_count;
} StoreDigestState;
+typedef struct {
+ int del_count; /* #store entries deleted from store_digest */
+ int del_lost_count; /* #store entries not found in store_digest on delete */
+ int add_count; /* #store entries accepted to store_digest */
+ int add_coll_count; /* #accepted entries that collided with existing ones */
+ int rej_count; /* #store entries not accepted to store_digest */
+ int rej_coll_count; /* #not accepted entries that collided with existing ones */
+} StoreDigestStats;
+
/*
* local constants (many of these are good candidates for SquidConfig
*/
-/* how often we want to rebuild the digest, seconds */
+/* #bits per entry in store digest */
+static const int StoreDigestBitsPerEntry = 6;
+/* how often we want to rebuild the digest, in seconds */
static const time_t StoreDigestRebuildPeriod = 60 * 60;
-/* how often we want to rewrite the digest, seconds */
-static const time_t StoreDigestRewritePeriod = 60 * 60;
+/* how often we want to rewrite the digest after rebuild, in seconds */
+static const int StoreDigestRewritePeriod = 60 * 60;
/* how many bytes to swap out at a time */
static const int StoreDigestSwapOutChunkSize = SM_PAGE_SIZE;
/* portion (0,1] of a hash table to be rescanned at a time */
static const double StoreDigestRebuildChunkPercent = 0.10;
-/* Fudge Factor for sizing the digest */
-static const double StoreDigestFudgeFactor = 1.5;
/* local vars */
static StoreDigestState sd_state;
+static StoreDigestStats sd_stats;
/* local prototypes */
-static void storeDigestRebuild(void *datanotused);
+static void storeDigestRebuildStart(void *datanotused);
+static void storeDigestRebuildResume();
static void storeDigestRebuildFinish();
static void storeDigestRebuildStep(void *datanotused);
-static void storeDigestRewrite();
+static void storeDigestRewriteStart();
+static void storeDigestRewriteResume();
static void storeDigestRewriteFinish(StoreEntry * e);
static void storeDigestSwapOutStep(StoreEntry * e);
static void storeDigestCBlockSwapOut(StoreEntry * e);
{
#if SQUID_MAINTAIN_CACHE_DIGEST
const int cap = storeDigestCalcCap();
- store_digest = cacheDigestCreate(cap);
- debug(71, 1) ("Using %d byte cache digest (%d entries); rebuild/rewrite every %d/%d sec\n",
- store_digest->mask_size, store_digest->capacity,
+ store_digest = cacheDigestCreate(cap, StoreDigestBitsPerEntry);
+ debug(71, 1) ("Local cache digest enabled; rebuild/rewrite every %d/%d sec\n",
StoreDigestRebuildPeriod, StoreDigestRewritePeriod);
#else
store_digest = NULL;
storeDigestReport, 0);
}
+/* called when store_rebuild completes */
void
-storeDigestScheduleRebuild()
+storeDigestNoteStoreReady()
{
- eventAdd("storeDigestRebuild", storeDigestRebuild, NULL, StoreDigestRebuildPeriod);
+ storeDigestRebuildStart(NULL);
+ storeDigestRewriteStart(NULL);
}
-/* externally initiated rewrite (inits store entry and pauses) */
void
-storeDigestRewriteStart() {
- eventAdd("storeDigestRewrite", storeDigestRewrite, NULL, 0);
-}
-
-#if OLD_CODE
-/* externally initiated rewrite (inits store entry and pauses) */
-void
-storeDigestRewriteStart(const char *initiator) {
- assert(initiator);
- assert(!sd_state.other_lock);
- sd_state.other_lock = initiator;
- storeDigestRewrite(NULL);
+storeDigestAdd(const StoreEntry * entry)
+{
+ int good_entry = 0;
+ assert(entry && store_digest);
+ debug(71,6) ("storeDigestAdd: checking entry, key: %s\n",
+ storeKeyText(entry->key));
+ /* only public entries are digested */
+ if (!EBIT_TEST(entry->flag, KEY_PRIVATE)) {
+ const time_t expires = refreshWhen(entry);
+ debug(71,6) ("storeDigestAdd: entry expires in %d secs\n",
+ (int)(expires - squid_curtime));
+ /* if expires too soon, ignore */
+ /* Note: We should use the time of the next rebuild, not cur_time @?@ */
+ if (expires <= squid_curtime + StoreDigestRebuildPeriod) {
+ debug(71,6) ("storeDigestAdd: entry expires too early, ignoring\n");
+ } else {
+ good_entry = 1;
+ }
+ }
+ if (good_entry) {
+ sd_stats.add_count++;
+ if (cacheDigestTest(store_digest, entry->key))
+ sd_stats.add_coll_count++;
+ cacheDigestAdd(store_digest, entry->key);
+ debug(71,6) ("storeDigestAdd: added entry, key: %s\n",
+ storeKeyText(entry->key));
+ } else {
+ sd_stats.rej_count++;
+ if (cacheDigestTest(store_digest, entry->key))
+ sd_stats.rej_coll_count++;
+ }
}
-/* continue externally initiated rewrite */
void
-storeDigestRewriteContinue(const char *initiator)
+storeDigestDel(const StoreEntry * entry)
{
- assert(initiator);
- assert(!strcmp(sd_state.other_lock, initiator));
- assert(sd_state.rewrite_lock);
- sd_state.other_lock = NULL;
- storeDigestSwapOutStep(sd_state.rewrite_lock);
+ assert(entry && store_digest);
+ debug(71,6) ("storeDigestDel: checking entry, key: %s\n",
+ storeKeyText(entry->key));
+ if (!EBIT_TEST(entry->flag, KEY_PRIVATE)) {
+ if (!cacheDigestTest(store_digest, entry->key)) {
+ sd_stats.del_lost_count++;
+ debug(71,6) ("storeDigestDel: lost entry, key: %s url: %s\n",
+ storeKeyText(entry->key), storeUrl(entry));
+ } else {
+ sd_stats.del_count++;
+ cacheDigestDel(store_digest, entry->key);
+ debug(71,6) ("storeDigestDel: deled entry, key: %s\n",
+ storeKeyText(entry->key));
+ }
+ }
}
-#endif /* OLD_CODE */
/* rebuilds digest from scratch */
static void
-storeDigestRebuild(void *datanotused)
+storeDigestRebuildStart(void *datanotused)
{
assert(store_digest);
/* prevent overlapping if rebuild schedule is too tight */
if (sd_state.rebuild_lock) {
- debug(71, 1) ("storeDigestRebuild: overlap detected, consider increasing rebuild period\n");
+ debug(71, 1) ("storeDigestRebuildStart: overlap detected, consider increasing rebuild period\n");
return;
}
sd_state.rebuild_lock = 1;
+ debug(71, 2) ("storeDigestRebuildStart: rebuild #%d\n", sd_state.rebuild_count + 1);
+ if (sd_state.rewrite_lock) {
+ debug(71, 2) ("storeDigestRebuildStart: waiting for Rewrite to finish.\n");
+ return;
+ }
+ storeDigestRebuildResume();
+}
+
+/* called be Rewrite to push Rebuild forward */
+static void
+storeDigestRebuildResume()
+{
+ assert(sd_state.rebuild_lock);
+ assert(!sd_state.rewrite_lock);
sd_state.rebuild_offset = 0;
/* resize or clear */
if (!storeDigestResize())
cacheDigestClear(store_digest); /* not clean()! */
- debug(71, 2) ("storeDigestRebuild: start rebuild #%d\n", sd_state.rebuild_count + 1);
- storeDigestRebuildStep(NULL);
+ memset(&sd_stats, 0, sizeof(sd_stats));
+ eventAdd("storeDigestRebuildStep", storeDigestRebuildStep, NULL, 0);
}
/* finishes swap out sequence for the digest; schedules next rebuild */
sd_state.rebuild_lock = 0;
sd_state.rebuild_count++;
debug(71, 2) ("storeDigestRebuildFinish: done.\n");
- storeDigestScheduleRebuild();
- /* resume pending write if any */
+ eventAdd("storeDigestRebuildStart", storeDigestRebuildStart, NULL, StoreDigestRebuildPeriod);
+ /* resume pending Rewrite if any */
if (sd_state.rewrite_lock)
- storeDigestSwapOutStep(sd_state.rewrite_lock);
+ storeDigestRewriteResume();
}
/* recalculate a few hash buckets per invocation; schedules next step */
while (bcount--) {
hash_link *link_ptr = hash_get_bucket(store_table, sd_state.rebuild_offset);
for (; link_ptr; link_ptr = link_ptr->next) {
- StoreEntry *e = (StoreEntry *) link_ptr;
- if (!EBIT_TEST(e->flag, KEY_PRIVATE))
- cacheDigestAdd(store_digest, e->key);
+ storeDigestAdd((StoreEntry *) link_ptr);
}
sd_state.rebuild_offset++;
}
/* starts swap out sequence for the digest */
static void
-storeDigestRewrite(void *datanotused)
+storeDigestRewriteStart(void *datanotused)
{
int flags;
- StoreEntry *e;
char *url;
+ StoreEntry *e;
assert(store_digest);
/* prevent overlapping if rewrite schedule is too tight */
url = urlInternal("", StoreDigestUrlPath);
flags = 0;
EBIT_SET(flags, REQ_CACHABLE);
- e = storeCreateEntry(url, url, flags, METHOD_GET);
- assert(e);
- sd_state.rewrite_lock = e;
+ sd_state.rewrite_lock = e = storeCreateEntry(url, url, flags, METHOD_GET);
+ assert(sd_state.rewrite_lock);
+ debug(71, 3) ("storeDigestRewrite: url: %s key: %s\n", url, storeKeyText(e->key));
+ e->mem_obj->request = requestLink(urlParse(METHOD_GET, url));
+ /* wait for rebuild (if any) to finish */
+ if (sd_state.rebuild_lock) {
+ debug(71, 2) ("storeDigestRewriteStart: waiting for rebuild to finish.\n");
+ return;
+ }
+ storeDigestRewriteResume();
+}
+
+static void
+storeDigestRewriteResume()
+{
+ StoreEntry *e = sd_state.rewrite_lock;
+
+ assert(sd_state.rewrite_lock);
+ assert(!sd_state.rebuild_lock);
sd_state.rewrite_offset = 0;
EBIT_SET(e->flag, ENTRY_SPECIAL);
/* setting public key will purge old digest entry if any */
storeSetPublicKey(e);
- debug(71, 3) ("storeDigestRewrite: url: %s key: %s\n", url, storeKeyText(e->key));
- /* we never unlink it! @?@ @?@ */
- e->mem_obj->request = requestLink(urlParse(METHOD_GET, url));
/* fake reply */
httpReplyReset(e->mem_obj->reply);
httpReplySetHeaders(e->mem_obj->reply, 1.0, 200, "Cache Digest OK",
"application/cache-digest", store_digest->mask_size + sizeof(sd_state.cblock),
squid_curtime, squid_curtime + StoreDigestRewritePeriod);
- debug(71, 3) ("storeDigestRewrite: reply.expires = %s\n", mkrfc1123(e->mem_obj->reply->expires));
+ debug(71, 3) ("storeDigestRewrite: entry expires on %s\n", mkrfc1123(e->mem_obj->reply->expires));
storeBuffer(e);
httpReplySwapOut(e->mem_obj->reply, e);
storeDigestCBlockSwapOut(e);
storeBufferFlush(e);
- if (sd_state.other_lock) {
- debug(71, 2) ("storeDigestRewrite: waiting for %s to finish.\n", sd_state.other_lock);
- return;
- }
- storeDigestSwapOutStep(e);
+ eventAdd("storeDigestSwapOutStep", (EVH *) storeDigestSwapOutStep, sd_state.rewrite_lock, 0);
}
/* finishes swap out sequence for the digest; schedules next rewrite */
storeTimestampsSet(e);
debug(71, 2) ("storeDigestRewriteFinish: digest expires on %s (%d)\n",
mkrfc1123(e->expires), e->expires);
+ /* is this the write order? @?@ */
+ requestUnlink(e->mem_obj->request);
+ e->mem_obj->request = NULL;
storeUnlockObject(e);
sd_state.rewrite_lock = e = NULL;
sd_state.rewrite_count++;
- eventAdd("storeDigestRewrite", storeDigestRewrite, NULL, StoreDigestRewritePeriod);
+ eventAdd("storeDigestRewriteStart", storeDigestRewriteStart, NULL, StoreDigestRewritePeriod);
+ /* resume pending Rebuild if any */
+ if (sd_state.rebuild_lock)
+ storeDigestRebuildResume();
}
/* swaps out one digest "chunk" per invocation; schedules next swap out */
{
int chunk_size = StoreDigestSwapOutChunkSize;
assert(e);
- assert(!sd_state.other_lock);
assert(e == sd_state.rewrite_lock);
- /* wait for rebuild (if any) to finish @?@ */
- if (sd_state.rebuild_lock) {
- debug(71, 2) ("storeDigestRewrite: waiting for rebuild to finish.\n");
- return;
- }
/* _add_ check that nothing bad happened while we were waiting @?@ @?@ */
if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
sd_state.cblock.count = htonl(store_digest->count);
sd_state.cblock.del_count = htonl(store_digest->del_count);
sd_state.cblock.mask_size = htonl(store_digest->mask_size);
+ sd_state.cblock.bits_per_entry = (unsigned char) StoreDigestBitsPerEntry;
+ sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount;
storeAppend(e, (char*) &sd_state.cblock, sizeof(sd_state.cblock));
}
* the bits are off). However, we do not have a formula to calculate the
* number of _entries_ we want to pre-allocate for.
*/
- const int hi_cap = store_table->size;
- const int lo_cap = hi_cap/5;
- int cap = (int) (1.5 * memInUse(MEM_STOREENTRY));
+ const int hi_cap = Config.Swap.maxSize / Config.Store.avgObjectSize;
+ const int lo_cap = 1 + store_swap_size / Config.Store.avgObjectSize;
+ const int e_count = memInUse(MEM_STOREENTRY);
+ int cap = e_count ? e_count : hi_cap;
+ debug(71, 2) ("storeDigestCalcCap: have: %d, want %d entries; limits: [%d, %d]\n",
+ e_count, cap, lo_cap, hi_cap);
if (cap < lo_cap)
cap = lo_cap;
- if (cap > hi_cap)
- cap = hi_cap;
- cap = 1 + (int) (StoreDigestFudgeFactor * cap);
- return hi_cap;
+ /* do not enforce hi_cap limit, average-based estimation may be wrong
+ *if (cap > hi_cap)
+ * cap = hi_cap;
+ */
+ return cap;
}
/* returns true if we actually resized the digest */
xpercentInt(diff, store_digest->capacity));
/* avoid minor adjustments */
if (diff <= store_digest->capacity/10) {
- debug(71, 1) ("storeDigestResize: small change, will not resize.\n");
- return 0; /* at most 10% change */
+ debug(71, 2) ("storeDigestResize: small change, will not resize.\n");
+ return 0;
} else {
- debug(71, 1) ("storeDigestResize: big change, resizing.\n");
+ debug(71, 2) ("storeDigestResize: big change, resizing.\n");
cacheDigestChangeCap(store_digest, cap);
return 1;
}
{
if (store_digest) {
cacheDigestReport(store_digest, "store", e);
+ storeAppendPrintf(e, "\t added: %d rejected: %d ( %.2f %%) del-ed: %d\n",
+ sd_stats.add_count,
+ sd_stats.rej_count,
+ xpercent(sd_stats.rej_count, sd_stats.rej_count + sd_stats.add_count),
+ sd_stats.del_count);
+ storeAppendPrintf(e, "\t collisions: on add: %.2f %% on rej: %.2f %%\n",
+ xpercent(sd_stats.add_coll_count, sd_stats.add_count),
+ xpercent(sd_stats.rej_coll_count, sd_stats.rej_count));
} else {
storeAppendPrintf(e, "store digest: disabled.\n");
}
projects / thirdparty / squid.git / commitdiff
