From: rousskov <>
Date: Fri, 3 Apr 1998 00:11:20 +0000 (+0000)
Subject: - Added internal support for cache digest (store_digest). The digest is
X-Git-Tag: SQUID_3_0_PRE1~3645
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8638fc6669169597202b7c816088c63feb46677f;p=thirdparty%2Fsquid.git

- Added internal support for cache digest (store_digest). The digest is
  maintained as a "mirror" of store_hash table. storeHashInsert/Delete forward
  public key objects to the store_digest object. Sending digests to other
  caches is not yet supported. Periodic digest rebuild is not yet supported.

- Rudimental reporting for store digest is done via cache manager.
---

diff --git a/src/CacheDigest.cc b/src/CacheDigest.cc
index f05aaa1f09..af963beee7 100644
--- a/src/CacheDigest.cc
+++ b/src/CacheDigest.cc
@@ -1,8 +1,8 @@
 
 /*
- * $Id: CacheDigest.cc,v 1.6 1998/04/01 07:14:05 rousskov Exp $
+ * $Id: CacheDigest.cc,v 1.7 1998/04/02 17:11:20 rousskov Exp $
  *
- * DEBUG: section ??    Cache Digest
+ * DEBUG: section 70    Cache Digest
  * AUTHOR: Alex Rousskov
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -126,6 +126,30 @@ cacheDigestUtil(const CacheDigest * cd, int *bit_cnt_p, int *on_cnt_p)
     return xpercent(on_count, bit_count);
 }
 
+void
+cacheDigestReport(CacheDigest *cd, const char *label, StoreEntry * e)
+{
+    int bit_count, on_count;
+    assert(cd && e);
+    cacheDigestUtil(cd, &bit_count, &on_count);
+    storeAppendPrintf(e, "%s digest: size: %d bytes.\n",
+	label ? label : "",
+	bit_count/8
+    );
+    storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
+	cd->count,
+	cd->capacity,
+	xpercentInt(cd->count, cd->capacity)
+    );
+    storeAppendPrintf(e, "\t deletion attempts: %d\n", 
+	cd->del_count
+    );
+    storeAppendPrintf(e, "\t bits: on: %d capacity: %d util: %d%%\n", 
+	on_count, bit_count,
+	xpercentInt(on_count, bit_count)
+    );
+}
+
 static void
 cacheDigestHashKey(int bit_count, const char *key)
 {
diff --git a/src/Makefile.in b/src/Makefile.in
index afc868140e..add1d6b153 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -1,7 +1,7 @@
 #
 #  Makefile for the Squid Object Cache server
 #
-#  $Id: Makefile.in,v 1.140 1998/03/30 20:42:40 rousskov Exp $
+#  $Id: Makefile.in,v 1.141 1998/04/02 17:11:21 rousskov Exp $
 #
 #  Uncomment and customize the following to suit your needs:
 #
@@ -72,6 +72,7 @@ OBJS	 	= \
 		asn.o \
 		@ASYNC_OBJS@ \
 		cache_cf.o \
+		CacheDigest.o \
 		cbdata.o \
 		client_db.o \
 		client_side.o \
@@ -132,6 +133,7 @@ OBJS	 	= \
 		store.o \
 		store_clean.o \
 		store_client.o \
+		store_digest.o \
 		store_dir.o \
 		store_key_md5.o \
 		store_log.o \
diff --git a/src/cache_cf.cc b/src/cache_cf.cc
index 88469944ea..c253a58cb9 100644
--- a/src/cache_cf.cc
+++ b/src/cache_cf.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: cache_cf.cc,v 1.269 1998/03/31 05:37:34 wessels Exp $
+ * $Id: cache_cf.cc,v 1.270 1998/04/02 17:11:22 rousskov Exp $
  *
  * DEBUG: section 3     Configuration File Parsing
  * AUTHOR: Harvest Derived
@@ -681,6 +681,8 @@ parse_peer(peer ** head)
 	    EBIT_SET(p->options, NEIGHBOR_PROXY_ONLY);
 	} else if (!strcasecmp(token, "no-query")) {
 	    EBIT_SET(p->options, NEIGHBOR_NO_QUERY);
+	} else if (!strcasecmp(token, "no-digest")) {
+	    EBIT_SET(p->options, NEIGHBOR_NO_DIGEST);
 	} else if (!strcasecmp(token, "multicast-responder")) {
 	    EBIT_SET(p->options, NEIGHBOR_MCAST_RESPONDER);
 	} else if (!strncasecmp(token, "weight=", 7)) {
diff --git a/src/enums.h b/src/enums.h
index fed737442d..4294742fff 100644
--- a/src/enums.h
+++ b/src/enums.h
@@ -435,6 +435,7 @@ enum {
 enum {
     NEIGHBOR_PROXY_ONLY,
     NEIGHBOR_NO_QUERY,
+    NEIGHBOR_NO_DIGEST,
     NEIGHBOR_DEFAULT_PARENT,
     NEIGHBOR_ROUNDROBIN,
     NEIGHBOR_MCAST_RESPONDER,
diff --git a/src/globals.h b/src/globals.h
index b73e19c42d..95e5983dd6 100644
--- a/src/globals.h
+++ b/src/globals.h
@@ -1,6 +1,6 @@
 
 /*
- * $Id: globals.h,v 1.48 1998/04/01 00:13:14 wessels Exp $
+ * $Id: globals.h,v 1.49 1998/04/02 17:11:23 rousskov Exp $
  */
 
 extern FILE *debug_log;		/* NULL */
@@ -99,3 +99,4 @@ extern dlink_list store_list;
 extern const String StringNull;	/* { 0, 0, NULL } */
 extern int hot_obj_count;	/* 0 */
 extern int _db_level;
+extern CacheDigest *store_digest; /* NULL */
diff --git a/src/neighbors.cc b/src/neighbors.cc
index 20de57e7f7..6d8ccf2aff 100644
--- a/src/neighbors.cc
+++ b/src/neighbors.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: neighbors.cc,v 1.188 1998/04/02 07:40:31 wessels Exp $
+ * $Id: neighbors.cc,v 1.189 1998/04/02 17:11:24 rousskov Exp $
  *
  * DEBUG: section 15    Neighbor Routines
  * AUTHOR: Harvest Derived
@@ -1006,6 +1006,8 @@ dump_peer_options(StoreEntry * sentry, peer * p)
 	storeAppendPrintf(sentry, " proxy-only");
     if (EBIT_TEST(p->options, NEIGHBOR_NO_QUERY))
 	storeAppendPrintf(sentry, " no-query");
+    if (EBIT_TEST(p->options, NEIGHBOR_NO_DIGEST))
+	storeAppendPrintf(sentry, " no-digest");
     if (EBIT_TEST(p->options, NEIGHBOR_DEFAULT_PARENT))
 	storeAppendPrintf(sentry, " default");
     if (EBIT_TEST(p->options, NEIGHBOR_ROUNDROBIN))
diff --git a/src/protos.h b/src/protos.h
index 1d491e4824..61634364bb 100644
--- a/src/protos.h
+++ b/src/protos.h
@@ -696,6 +696,12 @@ extern HASHCMP storeKeyHashCmp;
  */
 extern EVH storeDirClean;
 
+/* store_digest.c */
+extern void storeDigestInit();
+extern void storeDigestRebuild();
+extern void storeDigestReport();
+
+
 /*
  * store_dir.c
  */
@@ -900,6 +906,7 @@ extern int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
 extern void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
 extern void cacheDigestDel(CacheDigest * cd, const cache_key * key);
 extern double cacheDigestUtil(const CacheDigest * cd, int *bit_cnt_p, int *on_cnt_p);
+extern void cacheDigestReport(CacheDigest *cd, const char *label, StoreEntry * e);
 
 /*
  * prototypes for system functions missing from system includes
diff --git a/src/squid.h b/src/squid.h
index 406434be52..7f96a5c035 100644
--- a/src/squid.h
+++ b/src/squid.h
@@ -1,6 +1,6 @@
 
 /*
- * $Id: squid.h,v 1.165 1998/03/17 04:00:16 wessels Exp $
+ * $Id: squid.h,v 1.166 1998/04/02 17:11:25 rousskov Exp $
  *
  * AUTHOR: Duane Wessels
  *
@@ -355,4 +355,11 @@ struct rusage {
 #if SQUID_SNMP
 extern struct snmp_mib_tree *Mib;
 #endif
+
+/*
+ * maintain a digest of cache contents and send the digest to neighbors upon
+ * request; if disabled we still can request digests from other caches
+ */
+#define SQUID_MAINTAIN_CACHE_DIGEST 1
+
 #endif /* SQUID_H */
diff --git a/src/store.cc b/src/store.cc
index 4653dae41d..e4f8f48476 100644
--- a/src/store.cc
+++ b/src/store.cc
@@ -1,8 +1,8 @@
 
 /*
- * $Id: store.cc,v 1.399 1998/03/29 08:51:03 wessels Exp $
+ * $Id: store.cc,v 1.400 1998/04/02 17:11:26 rousskov Exp $
  *
- * DEBUG: section 20    Storeage Manager
+ * DEBUG: section 20    Storage Manager
  * AUTHOR: Harvest Derived
  *
  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
@@ -254,11 +254,15 @@ storeHashInsert(StoreEntry * e, const cache_key * key)
     e->key = storeKeyDup(key);
     hash_join(store_table, (hash_link *) e);
     dlinkAdd(e, &e->lru, &store_list);
+    if (store_digest && !EBIT_TEST(e->flag, KEY_PRIVATE))
+	cacheDigestAdd(store_digest, e->key);
 }
 
 static void
 storeHashDelete(StoreEntry * e)
 {
+    if (store_digest && !EBIT_TEST(e->flag, KEY_PRIVATE))
+	cacheDigestDel(store_digest, e->key);
     hash_remove_link(store_table, (hash_link *) e);
     dlinkDelete(&e->lru, &store_list);
     storeKeyFree(e->key);
@@ -370,8 +374,8 @@ storeSetPrivateKey(StoreEntry * e)
 	newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter(METHOD_NONE));
     }
     assert(hash_lookup(store_table, newkey) == NULL);
-    storeHashInsert(e, newkey);
     EBIT_SET(e->flag, KEY_PRIVATE);
+    storeHashInsert(e, newkey);
 }
 
 void
@@ -392,8 +396,8 @@ storeSetPublicKey(StoreEntry * e)
     }
     if (e->key)
 	storeHashDelete(e);
-    storeHashInsert(e, newkey);
     EBIT_CLR(e->flag, KEY_PRIVATE);
+    storeHashInsert(e, newkey);
     if (e->swap_file_number > -1)
 	storeDirSwapLog(e, SWAP_LOG_ADD);
 }
@@ -821,6 +825,7 @@ storeInit(void)
     storeInitHashValues();
     store_table = hash_create(storeKeyHashCmp,
 	store_hash_buckets, storeKeyHashHash);
+    storeDigestInit();
     storeLogOpen();
     if (storeVerifyCacheDirs() < 0) {
 	xstrncpy(tmp_error_buf,
@@ -931,6 +936,9 @@ storeFreeMemory(void)
     hashFreeItems(store_table, destroy_StoreEntry);
     hashFreeMemory(store_table);
     store_table = NULL;
+    if (store_digest)
+	cacheDigestDestroy(store_digest);
+    store_digest = NULL;
 }
 
 int
diff --git a/src/store_digest.cc b/src/store_digest.cc
new file mode 100644
index 0000000000..68ab82ff7c
--- /dev/null
+++ b/src/store_digest.cc
@@ -0,0 +1,68 @@
+/*
+ * $Id: store_digest.cc,v 1.1 1998/04/02 17:11:27 rousskov Exp $
+ *
+ * DEBUG: section 71    Store Digest Manager
+ * AUTHOR: Alex Rousskov
+ *
+ * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
+ * --------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from the
+ *  Internet community.  Development is led by Duane Wessels of the
+ *  National Laboratory for Applied Network Research and funded by
+ *  the National Science Foundation.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *  
+ */
+
+#include "squid.h"
+
+void
+storeDigestInit()
+{
+    /*
+     * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
+     * the bits are off). However, we do not have a formula to calculate the 
+     * number of _entries_ we want to pre-allocate for.
+     * Use 1.5*max#entries because 2*max#entries gives about 40% utilization.
+     */
+    const int cap = (int)(1.5 * Config.Swap.maxSize / Config.Store.avgObjectSize);
+#if SQUID_MAINTAIN_CACHE_DIGEST
+    store_digest = cacheDigestCreate(cap);
+#else
+    store_digest = NULL;
+#endif
+    cachemgrRegister("store_digest", "Store Digest",
+        storeDigestReport, 0);
+}
+
+/* rebuilds digest from scratch */
+void
+storeDigestRebuild()
+{
+    assert(store_digest);
+}
+
+void
+storeDigestReport(StoreEntry *e)
+{
+    if (store_digest) {
+	cacheDigestReport(store_digest, "store", e);
+    } else {
+	storeAppendPrintf(e, "store digest: disabled.\n");
+    }
+}
+
diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc
index 80360a31e4..695555d399 100644
--- a/src/store_rebuild.cc
+++ b/src/store_rebuild.cc
@@ -1,3 +1,33 @@
+/*
+ * $Id: store_rebuild.cc,v 1.27 1998/04/02 17:11:27 rousskov Exp $
+ *
+ * DEBUG: section 20    Store Rebuild Routines
+ * AUTHOR: Duane Wessels
+ *
+ * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
+ * --------------------------------------------------------
+ *
+ *  Squid is the result of efforts by numerous individuals from the
+ *  Internet community.  Development is led by Duane Wessels of the
+ *  National Laboratory for Applied Network Research and funded by
+ *  the National Science Foundation.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *  
+ */
+
 #include "squid.h"
 
 #define STORE_META_BUFSZ 4096
@@ -511,7 +541,6 @@ storeAddDiskRestore(const cache_key * key,
     /* if you call this you'd better be sure file_number is not 
      * already in use! */
     e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
-    storeHashInsert(e, key);
     e->store_status = STORE_OK;
     storeSetMemStatus(e, NOT_IN_MEMORY);
     e->swap_status = SWAPOUT_DONE;
@@ -531,6 +560,7 @@ storeAddDiskRestore(const cache_key * key,
     e->ping_status = PING_NONE;
     EBIT_CLR(e->flag, ENTRY_VALIDATED);
     storeDirMapBitSet(e->swap_file_number);
+    storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
     return e;
 }
 
diff --git a/src/test_cache_digest.cc b/src/test_cache_digest.cc
index b2e46c5c29..af43bb6eba 100644
--- a/src/test_cache_digest.cc
+++ b/src/test_cache_digest.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: test_cache_digest.cc,v 1.15 1998/04/01 18:02:19 rousskov Exp $
+ * $Id: test_cache_digest.cc,v 1.16 1998/04/02 17:11:28 rousskov Exp $
  *
  * AUTHOR: Alex Rousskov
  *
@@ -348,22 +348,6 @@ cacheReport(Cache * cache)
 	cache->name, cache->count, cache->req_count,
 	cache->bad_add_count, cache->bad_del_count);
 
-    if (cache->digest) {
-	int bit_count, on_count;
-	cacheDigestUtil(cache->digest, &bit_count, &on_count);
-	fprintf(stdout, "%s: digest entries: cnt: %d (-=%d) cap: %d util: %d%% size: %d b\n", 
-	    cache->name, 
-	    cache->digest->count, cache->digest->del_count,
-	    cache->digest->capacity,
-	    xpercentInt(cache->digest->count, cache->digest->capacity),
-	    bit_count/8
-	);
-	fprintf(stdout, "%s: digest bits: on: %d cap: %d util: %d%%\n", 
-	    cache->name,
-	    on_count, bit_count,
-	    xpercentInt(on_count, bit_count)
-	);
-    }
 }
 
 static void