From: rousskov <> Date: Tue, 31 Mar 1998 07:03:55 +0000 (+0000) Subject: - Adding more stuff to test_cache_digest X-Git-Tag: SQUID_3_0_PRE1~3692 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=76e3f5c2d11436b883d0f9591f7b47e5bdb8d9f9;p=thirdparty%2Fsquid.git - Adding more stuff to test_cache_digest --- diff --git a/src/test_cache_digest.cc b/src/test_cache_digest.cc index 81c2d2255f..b1e5860a56 100644 --- a/src/test_cache_digest.cc +++ b/src/test_cache_digest.cc @@ -1,5 +1,5 @@ /* - * $Id: test_cache_digest.cc,v 1.2 1998/03/30 21:16:10 rousskov Exp $ + * $Id: test_cache_digest.cc,v 1.3 1998/03/31 00:03:55 rousskov Exp $ * * AUTHOR: Alex Rousskov * @@ -66,7 +66,32 @@ const char *RequestMethodStr[] = }; +static CacheIndex *Peer = NULL; + static int cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file); +static int cacheIndexScanAccessLog(CacheIndex *idx, const char *fname, FILE *file); + +/* copied from url.c */ +static method_t +cacheIndexParseMethod(const char *s) +{ + if (strcasecmp(s, "GET") == 0) { + return METHOD_GET; + } else if (strcasecmp(s, "POST") == 0) { + return METHOD_POST; + } else if (strcasecmp(s, "PUT") == 0) { + return METHOD_PUT; + } else if (strcasecmp(s, "HEAD") == 0) { + return METHOD_HEAD; + } else if (strcasecmp(s, "CONNECT") == 0) { + return METHOD_CONNECT; + } else if (strcasecmp(s, "TRACE") == 0) { + return METHOD_TRACE; + } else if (strcasecmp(s, "PURGE") == 0) { + return METHOD_PURGE; + } + return METHOD_NONE; +} static CacheEntry * @@ -127,7 +152,7 @@ cacheIndexInitDigest(CacheIndex *idx) struct timeval t_start, t_end; assert(idx && !idx->digest); fprintf(stderr, "%s: init-ing digest with %d entries\n", idx->name, idx->count); - idx->digest = cacheDigestCreate(idx->count); /* no "fat" */ + idx->digest = cacheDigestCreate(2*idx->count); /* 50% utilization */ gettimeofday(&t_start, NULL); for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) { cacheDigestAdd(idx->digest, hashr->key); @@ -284,6 +309,140 @@ cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file) return count; } +/* Us */ + +static int we_icp_query_count = 0; +static int we_true_hit_count = 0; +static int we_true_miss_count = 0; +static int we_false_hit_count = 0; +static int we_false_miss_count = 0; + +static void +cacheIndexQueryPeer(CacheIndex *idx, const cache_key *key) +{ + const int peer_has_it = hash_lookup(Peer->hash, key) != NULL; + const int we_think_we_have_it = cacheDigestTest(Peer->digest, key); + + we_icp_query_count++; + if (peer_has_it) + if (we_think_we_have_it) + we_true_hit_count++; + else + we_false_miss_count++; + else + if (we_think_we_have_it) + we_false_hit_count++; + else + we_true_miss_count++; +} + +static void +cacheIndexIcpReport(CacheIndex *idx) +{ + fprintf(stdout, "we: icp: %d\n", we_icp_query_count); + fprintf(stdout, "we: t-hit: %d (%d%%) t-miss: %d (%d%%) t-*: %d (%d%%)\n", + we_true_hit_count, xpercentInt(we_true_hit_count, we_icp_query_count), + we_true_miss_count, xpercentInt(we_true_miss_count, we_icp_query_count), + we_true_hit_count+we_true_miss_count, + xpercentInt(we_true_hit_count+we_true_miss_count, we_icp_query_count) + ); + fprintf(stdout, "we: f-hit: %d (%d%%) f-miss: %d (%d%%) f-*: %d (%d%%)\n", + we_false_hit_count, xpercentInt(we_false_hit_count, we_icp_query_count), + we_false_miss_count, xpercentInt(we_false_miss_count, we_icp_query_count), + we_false_hit_count+we_false_miss_count, + xpercentInt(we_false_hit_count+we_false_miss_count, we_icp_query_count) + ); +} + +static int +cacheIndexAddAccessLog(CacheIndex *idx, const char *fname) +{ + FILE *file; + int scanned_count = 0; + assert(!idx); + assert(fname && strlen(fname)); + + file = fopen(fname, "r"); + if (!file) { + fprintf(stderr, "cannot open %s: %s\n", fname, strerror(errno)); + return 0; + } + scanned_count = cacheIndexScanAccessLog(idx, fname, file); + fclose(file); + return scanned_count; +} + +static int +cacheIndexScanAccessLog(CacheIndex *idx, const char *fname, FILE *file) +{ + static char buf[4096]; + int count = 0; + int scanned_count = 0; + int icp_count = 0; + assert(!idx); + fprintf(stderr, "%s scanning\n", fname); + while (fgets(buf, sizeof(buf), file)) { + char *url = strstr(buf, "://"); + char *method; + int method_id = METHOD_NONE; + char *hier = url ? strstr(url, " - ") : NULL; + const cache_key *key = NULL; + + scanned_count++; + if (!(scanned_count % 50000)) + fprintf(stderr, "%s scanned %d K entries (%d bad)\n", + fname, scanned_count/1000, scanned_count-count-1); + if (!url || !hier) { + fprintf(stderr, "%s:%d: strange access log entry '%s'\n", + fname, scanned_count, buf); + continue; + } + method = url; + while (!isdigit(*method)) { + if (*method == ' ') + *method = '\0'; + --method; + } + method += 2; + method_id = cacheIndexParseMethod(method); + if (method_id == METHOD_NONE) { + fprintf(stderr, "%s:%d: invalid method %s in '%s'\n", + fname, scanned_count, method, buf); + continue; + } + while (*url) url--; + url++; + *hier = '\0'; + hier += 3; + *strchr(hier, '/') = '\0'; + /*fprintf(stdout, "%s:%d: %s %s %s\n", + fname, count, method, url, hier);*/ + count++; + /* no ICP lookup for these status codes */ + if (!strcmp(hier, "NONE") || + !strcmp(hier, "DIRECT") || + !strcmp(hier, "FIREWALL_IP_DIRECT") || + !strcmp(hier, "LOCAL_IP_DIRECT") || + !strcmp(hier, "NO_DIRECT_FAIL") || + !strcmp(hier, "NO_PARENT_DIRECT") || + !strcmp(hier, "SINGLE_PARENT") || + !strcmp(hier, "PASSTHROUGH_PARENT") || + !strcmp(hier, "SSL_PARENT_MISS") || + !strcmp(hier, "DEFAULT_PARENT")) + continue; + key = storeKeyPublic(url, method_id); + /*fprintf(stdout, "%s:%d: %s %s %s %s\n", + fname, count, method, storeKeyText(key), url, hier);*/ + cacheIndexQueryPeer(idx, key); + icp_count++; + } + fprintf(stderr, "%s: scanned %d access log entries; bad: %d\n", + fname, scanned_count, scanned_count-count); + fprintf(stderr, "%s: icp: %d (%d%%)\n", + fname, icp_count, xpercentInt(icp_count, count)); + return count; +} + static int usage(const char *prg_name) { @@ -301,14 +460,18 @@ main(int argc, char *argv[]) if (argc < 3) return usage(argv[0]); - they = cacheIndexCreate("they"); + they = Peer = cacheIndexCreate("they"); for (i = 2; i < argc; ++i) { cacheIndexAddLog(they, argv[i]); } cacheIndexInitDigest(they); cacheIndexInitReport(they); + if (!cacheIndexAddAccessLog(NULL, argv[1])) + return 1; + cacheIndexIcpReport(NULL); + cacheIndexDestroy(they); - return 1; + return 0; }