- Adding more stuff to test_cache_digest

diff --git a/src/test_cache_digest.cc b/src/test_cache_digest.cc
index 81c2d2255fd3405fdbd6cecff597649533558dee..b1e5860a56fe5671248f282b3a9dad168a7218a0 100644 (file)
--- a/src/test_cache_digest.cc
+++ b/src/test_cache_digest.cc
@@ -1,5 +1,5 @@
 /*
- * $Id: test_cache_digest.cc,v 1.2 1998/03/30 21:16:10 rousskov Exp $
+ * $Id: test_cache_digest.cc,v 1.3 1998/03/31 00:03:55 rousskov Exp $
  *
  * AUTHOR: Alex Rousskov
  *
@@ -66,7 +66,32 @@ const char *RequestMethodStr[] =
 };
 
 
+static CacheIndex *Peer = NULL;
+
 static int cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file);
+static int cacheIndexScanAccessLog(CacheIndex *idx, const char *fname, FILE *file);
+
+/* copied from url.c */
+static method_t
+cacheIndexParseMethod(const char *s)
+{
+    if (strcasecmp(s, "GET") == 0) {
+        return METHOD_GET;
+    } else if (strcasecmp(s, "POST") == 0) {
+        return METHOD_POST;
+    } else if (strcasecmp(s, "PUT") == 0) {
+        return METHOD_PUT;
+    } else if (strcasecmp(s, "HEAD") == 0) {
+        return METHOD_HEAD;
+    } else if (strcasecmp(s, "CONNECT") == 0) {
+        return METHOD_CONNECT;
+    } else if (strcasecmp(s, "TRACE") == 0) {
+        return METHOD_TRACE;
+    } else if (strcasecmp(s, "PURGE") == 0) {
+        return METHOD_PURGE;
+    }
+    return METHOD_NONE;
+}
 
 
 static CacheEntry *
@@ -127,7 +152,7 @@ cacheIndexInitDigest(CacheIndex *idx)
     struct timeval t_start, t_end;
     assert(idx && !idx->digest);
     fprintf(stderr, "%s: init-ing digest with %d entries\n", idx->name, idx->count);
-    idx->digest = cacheDigestCreate(idx->count); /* no "fat" */
+    idx->digest = cacheDigestCreate(2*idx->count); /* 50% utilization */
     gettimeofday(&t_start, NULL);
     for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) {
        cacheDigestAdd(idx->digest, hashr->key);
@@ -284,6 +309,140 @@ cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file)
     return count;
 }
 
+/* Us */
+
+static int we_icp_query_count = 0;
+static int we_true_hit_count = 0;
+static int we_true_miss_count = 0;
+static int we_false_hit_count = 0;
+static int we_false_miss_count = 0;
+
+static void
+cacheIndexQueryPeer(CacheIndex *idx, const cache_key *key)
+{
+    const int peer_has_it = hash_lookup(Peer->hash, key) != NULL;
+    const int we_think_we_have_it = cacheDigestTest(Peer->digest, key);
+
+    we_icp_query_count++;
+    if (peer_has_it)
+       if (we_think_we_have_it)
+           we_true_hit_count++;
+       else
+           we_false_miss_count++;
+    else
+       if (we_think_we_have_it)
+           we_false_hit_count++;
+       else
+           we_true_miss_count++;
+}
+
+static void
+cacheIndexIcpReport(CacheIndex *idx)
+{
+    fprintf(stdout, "we: icp: %d\n", we_icp_query_count);
+    fprintf(stdout, "we: t-hit: %d (%d%%) t-miss: %d (%d%%) t-*: %d (%d%%)\n",
+       we_true_hit_count, xpercentInt(we_true_hit_count, we_icp_query_count),
+       we_true_miss_count, xpercentInt(we_true_miss_count, we_icp_query_count),
+       we_true_hit_count+we_true_miss_count, 
+           xpercentInt(we_true_hit_count+we_true_miss_count, we_icp_query_count)
+    );
+    fprintf(stdout, "we: f-hit: %d (%d%%) f-miss: %d (%d%%) f-*: %d (%d%%)\n",
+       we_false_hit_count, xpercentInt(we_false_hit_count, we_icp_query_count),
+       we_false_miss_count, xpercentInt(we_false_miss_count, we_icp_query_count),
+       we_false_hit_count+we_false_miss_count, 
+           xpercentInt(we_false_hit_count+we_false_miss_count, we_icp_query_count)
+    );
+}
+
+static int
+cacheIndexAddAccessLog(CacheIndex *idx, const char *fname)
+{
+    FILE *file;
+    int scanned_count = 0;
+    assert(!idx);
+    assert(fname && strlen(fname));
+
+    file = fopen(fname, "r");
+    if (!file) {
+       fprintf(stderr, "cannot open %s: %s\n", fname, strerror(errno));
+       return 0;
+    }
+    scanned_count = cacheIndexScanAccessLog(idx, fname, file);
+    fclose(file);
+    return scanned_count;
+}
+
+static int
+cacheIndexScanAccessLog(CacheIndex *idx, const char *fname, FILE *file)
+{
+    static char buf[4096];
+    int count = 0;
+    int scanned_count = 0;
+    int icp_count = 0;
+    assert(!idx);
+    fprintf(stderr, "%s scanning\n", fname);
+    while (fgets(buf, sizeof(buf), file)) {
+       char *url = strstr(buf, "://");
+       char *method;
+       int method_id = METHOD_NONE;
+       char *hier = url ? strstr(url, " - ") : NULL;
+       const cache_key *key = NULL;
+
+       scanned_count++;
+       if (!(scanned_count % 50000))
+           fprintf(stderr, "%s scanned %d K entries (%d bad)\n", 
+               fname, scanned_count/1000, scanned_count-count-1);
+       if (!url || !hier) {
+           fprintf(stderr, "%s:%d: strange access log entry '%s'\n", 
+               fname, scanned_count, buf);
+           continue;
+       }
+       method = url;
+       while (!isdigit(*method)) {
+           if (*method == ' ')
+               *method = '\0';
+           --method;
+       }
+       method += 2;
+       method_id = cacheIndexParseMethod(method);
+       if (method_id == METHOD_NONE) {
+           fprintf(stderr, "%s:%d: invalid method %s in '%s'\n", 
+               fname, scanned_count, method, buf);
+           continue;
+       }
+       while (*url) url--;
+       url++;
+       *hier = '\0';
+       hier += 3;
+       *strchr(hier, '/') = '\0';
+       /*fprintf(stdout, "%s:%d: %s %s %s\n",
+           fname, count, method, url, hier);*/
+       count++;
+       /* no ICP lookup for these status codes */
+       if (!strcmp(hier, "NONE") ||
+           !strcmp(hier, "DIRECT") ||
+           !strcmp(hier, "FIREWALL_IP_DIRECT") ||
+           !strcmp(hier, "LOCAL_IP_DIRECT") ||
+           !strcmp(hier, "NO_DIRECT_FAIL") ||
+           !strcmp(hier, "NO_PARENT_DIRECT") ||
+           !strcmp(hier, "SINGLE_PARENT") ||
+           !strcmp(hier, "PASSTHROUGH_PARENT") ||
+           !strcmp(hier, "SSL_PARENT_MISS") ||
+           !strcmp(hier, "DEFAULT_PARENT"))
+           continue;
+       key = storeKeyPublic(url, method_id);
+       /*fprintf(stdout, "%s:%d: %s %s %s %s\n",
+           fname, count, method, storeKeyText(key), url, hier);*/
+       cacheIndexQueryPeer(idx, key);
+       icp_count++;
+    }
+    fprintf(stderr, "%s: scanned %d access log entries; bad: %d\n",
+       fname, scanned_count, scanned_count-count);
+    fprintf(stderr, "%s: icp: %d (%d%%)\n",
+       fname, icp_count, xpercentInt(icp_count, count));
+    return count;
+}
+
 static int
 usage(const char *prg_name)
 {
@@ -301,14 +460,18 @@ main(int argc, char *argv[])
     if (argc < 3)
        return usage(argv[0]);
 
-    they = cacheIndexCreate("they");
+    they = Peer = cacheIndexCreate("they");
     for (i = 2; i < argc; ++i) {
        cacheIndexAddLog(they, argv[i]);
     }
     cacheIndexInitDigest(they);
     cacheIndexInitReport(they);
 
+    if (!cacheIndexAddAccessLog(NULL, argv[1]))
+       return 1;
+    cacheIndexIcpReport(NULL);
+
     cacheIndexDestroy(they);
 
-    return 1;
+    return 0;
 }