[thirdparty/squid.git] / src / test_cache_digest.cc

/*
 * $Id: test_cache_digest.cc,v 1.1 1998/03/30 20:42:42 rousskov Exp $
 *
 * AUTHOR: Alex Rousskov
 *
 * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
 * --------------------------------------------------------
 *
 *  Squid is the result of efforts by numerous individuals from the
 *  Internet community.  Development is led by Duane Wessels of the
 *  National Laboratory for Applied Network Research and funded by
 *  the National Science Foundation.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *  
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *  
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *  
 */

/*
 * Test-suite for playing with cache digests
 */

#include "squid.h"

typedef struct {
    const char *name;
    hash_table *hash;
    CacheDigest *digest;
    int count;          /* #currently cached entries */
    int scanned_count;  /* #scanned entries */
    int bad_add_count;  /* #duplicate adds */
    int bad_del_count;  /* #dels with no prior add */
} CacheIndex;


typedef struct _CacheEntry {
    const cache_key *key;
    struct _CacheEntry *next;
    /* storeSwapLogData s; */
    unsigned char key_arr[MD5_DIGEST_CHARS];
} CacheEntry;


/* copied from url.c */
const char *RequestMethodStr[] =
{
    "NONE",
    "GET",
    "POST",
    "PUT",
    "HEAD",
    "CONNECT",
    "TRACE",
    "PURGE"
};


static int cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file);


static CacheEntry *
cacheEntryCreate(const storeSwapLogData *s)
{
    CacheEntry *e = xcalloc(1, sizeof(CacheEntry));
    assert(s);
    /* e->s = *s; */
    xmemcpy(e->key_arr, s->key, MD5_DIGEST_CHARS);
    e->key = &e->key_arr[0];
    return e;
}

static void
cacheEntryDestroy(CacheEntry *e)
{
    assert(e);
    xfree(e);
}

static CacheIndex *
cacheIndexCreate(const char *name)
{
    CacheIndex *idx;
    if (!name || !strlen(name))
	return NULL;

    idx = xcalloc(1, sizeof(CacheIndex));
    idx->name = name;
    idx->hash = hash_create(storeKeyHashCmp, 2e6, storeKeyHashHash);

    return idx;
}

static void
cacheIndexDestroy(CacheIndex *idx)
{
    hash_link *hashr = NULL;
    if (idx) {
	/* destroy hash list contents */
	for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) {
	    hash_remove_link(idx->hash, hashr);
	    cacheEntryDestroy((CacheEntry*)hashr);
	}
	/* destroy the hash table itself */
	hashFreeMemory(idx->hash);
	if (idx->digest)
	    cacheDigestDestroy(idx->digest);
	xfree(idx);
    }
}

/* makes digest based on currently hashed entries */
static void
cacheIndexInitDigest(CacheIndex *idx)
{
    hash_link *hashr = NULL;
    struct timeval t_start, t_end;
    assert(idx && !idx->digest);
    fprintf(stderr, "%s: init-ing digest with %d entries\n", idx->name, idx->count);
    idx->digest = cacheDigestCreate(idx->count); /* no "fat" */
    gettimeofday(&t_start, NULL);
    for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) {
	cacheDigestAdd(idx->digest, hashr->key);
    }
    gettimeofday(&t_end, NULL);
    assert(idx->digest->count == idx->count);
    fprintf(stderr, "%s: init-ed  digest with %d entries\n", 
	idx->name, idx->digest->count);
    fprintf(stderr, "%s: init took: %f sec, %f sec/M\n",
	idx->name,
	tvSubDsec(t_start, t_end),
	(double)1e6*tvSubDsec(t_start, t_end)/idx->count);
}

static int
cacheIndexAddLog(CacheIndex *idx, const char *fname)
{
    FILE *file;
    int scanned_count = 0;
    assert(idx);
    assert(fname && strlen(fname));

    file = fopen(fname, "r");
    if (!file) {
	fprintf(stderr, "cannot open %s: %s\n", fname, strerror(errno));
	return 0;
    }
    scanned_count = cacheIndexScanCleanPrefix(idx, fname, file);
    fclose(file);
    return scanned_count;
}

static void
cacheIndexInitReport(CacheIndex *idx)
{
    assert(idx);
    fprintf(stderr, "%s: bad swap_add:  %d\n",
	idx->name, idx->bad_add_count);
    fprintf(stderr, "%s: bad swap_del:  %d\n", 
	idx->name, idx->bad_del_count);
    fprintf(stderr, "%s: scanned lines: %d\n", 
	idx->name, idx->scanned_count);
}

#if 0
static int
cacheIndexGetLogEntry(CacheIndex *idx, storeSwapLogData *s)
{
    if (!idx->has_log_entry)
	cacheIndexStepLogEntry();
    if (idx->has_log_entry) {
	*s = idx->log_entry_buf;
	return 1;
    }
    return 0;
}

static int
cacheIndexStepLogEntry(CacheIndex *idx)
{
    if (fread(&idx->log_entry_buf, sizeof(idx->log_entry_buf), 1, idx->log) == 1) {
	int op = (int) idx->log_entry_buf.op;
	idx->scanned_count++;
	idx->has_log_entry = 1;
	if (op != SWAP_LOG_ADD && op != SWAP_LOG_DEL) {
	    fprintf(stderr, "%s:%d: unknown swap log action %d\n", idx->log_fname, idx->scanned_count, op);
	    exit(-3);
	}
    } else
	idx->has_log_entry = 0;
}

static int
cacheIndexScan(CacheIndex *idx, const char *fname, FILE *file)
{
    int count = 0;
    int del_count = 0;
    storeSwapLogData s;
    fprintf(stderr, "%s scanning\n", fname);
    while (fread(&s, sizeof(s), 1, file) == 1) {
	count++;
	idx->scanned_count++;
	if (s.op == SWAP_LOG_ADD) {
	    CacheEntry *olde = (CacheEntry *) hash_lookup(idx->hash, s.key);
	    if (olde) {
		idx->bad_add_count++;
	    } else {
		CacheEntry *e = cacheEntryCreate(&s);
		hash_join(idx->hash, (hash_link*) e);
		idx->count++;
	    }
	} else
	if (s.op == SWAP_LOG_DEL) {
	    CacheEntry *olde = (CacheEntry *) hash_lookup(idx->hash, s.key);
	    if (!olde)
		idx->bad_del_count++;
	    else {
		assert(idx->count);
		hash_remove_link(idx->hash, (hash_link*) olde);
		cacheEntryDestroy(olde);
		idx->count--;
	    }
	    del_count++;
	} else {
	    fprintf(stderr, "%s:%d: unknown swap log action\n", fname, count);
	    exit(-3);
	}
    }
    fprintf(stderr, "%s scanned %d entries, alloc: %d bytes\n",
	fname, count, 
	(int)(count*sizeof(CacheEntry)));
    return count;
}
#endif

static int
cacheIndexScanCleanPrefix(CacheIndex *idx, const char *fname, FILE *file)
{
    int count = 0;
    storeSwapLogData s;
    fprintf(stderr, "%s scanning\n", fname);
    while (fread(&s, sizeof(s), 1, file) == 1) {
	count++;
	idx->scanned_count++;
	if (s.op == SWAP_LOG_ADD) {
	    CacheEntry *olde = (CacheEntry *) hash_lookup(idx->hash, s.key);
	    if (olde) {
		idx->bad_add_count++;
	    } else {
		CacheEntry *e = cacheEntryCreate(&s);
		hash_join(idx->hash, (hash_link*) e);
		idx->count++;
	    }
	} else
	if (s.op == SWAP_LOG_DEL) {
	    break;
	} else {
	    fprintf(stderr, "%s:%d: unknown swap log action\n", fname, count);
	    exit(-3);
	}
    }
    fprintf(stderr, "%s scanned %d entries, alloc: %d bytes\n",
	fname, count, 
	(int)(count*sizeof(CacheEntry)));
    return count;
}

static int
usage(const char *prg_name)
{
    fprintf(stderr, "usage: %s <access_log> <swap_state> ...\n",
	prg_name);
    return -1;
}

int
main(int argc, char *argv[])
{
    CacheIndex *they = NULL;
    int i;

    if (argc < 3)
	return usage(argv[0]);

    they = cacheIndexCreate("they");
    for (i = 2; i < argc; ++i) {
	cacheIndexAddLog(they, argv[i]);
    }
    cacheIndexInitDigest(they);
    cacheIndexInitReport(they);

    cacheIndexDestroy(they);

    return 1;
}
Commit	Line	Data
c411be12	1	/*
	2	* $Id: test_cache_digest.cc,v 1.1 1998/03/30 20:42:42 rousskov Exp $
	3	*
	4	* AUTHOR: Alex Rousskov
	5	*
	6	* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
	7	* --------------------------------------------------------
	8	*
	9	* Squid is the result of efforts by numerous individuals from the
	10	* Internet community. Development is led by Duane Wessels of the
	11	* National Laboratory for Applied Network Research and funded by
	12	* the National Science Foundation.
	13	*
	14	* This program is free software; you can redistribute it and/or modify
	15	* it under the terms of the GNU General Public License as published by
	16	* the Free Software Foundation; either version 2 of the License, or
	17	* (at your option) any later version.
	18	*
	19	* This program is distributed in the hope that it will be useful,
	20	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	21	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	22	* GNU General Public License for more details.
	23	*
	24	* You should have received a copy of the GNU General Public License
	25	* along with this program; if not, write to the Free Software
	26	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	27	*
	28	*/
	29
	30	/*
	31	* Test-suite for playing with cache digests
	32	*/
	33
	34	#include "squid.h"
	35
	36	typedef struct {
	37	const char *name;
	38	hash_table *hash;
	39	CacheDigest *digest;
	40	int count; /* #currently cached entries */
	41	int scanned_count; /* #scanned entries */
	42	int bad_add_count; /* #duplicate adds */
	43	int bad_del_count; /* #dels with no prior add */
	44	} CacheIndex;
	45
	46
	47	typedef struct _CacheEntry {
	48	const cache_key *key;
	49	struct _CacheEntry *next;
	50	/* storeSwapLogData s; */
	51	unsigned char key_arr[MD5_DIGEST_CHARS];
	52	} CacheEntry;
	53
	54
	55	/* copied from url.c */
	56	const char *RequestMethodStr[] =
	57	{
	58	"NONE",
	59	"GET",
	60	"POST",
	61	"PUT",
	62	"HEAD",
	63	"CONNECT",
	64	"TRACE",
65	"PURGE"
66	};
67
68
69	static int cacheIndexScanCleanPrefix(CacheIndex idx, const char fname, FILE *file);
70
71
72	static CacheEntry *
73	cacheEntryCreate(const storeSwapLogData *s)
74	{
75	CacheEntry *e = xcalloc(1, sizeof(CacheEntry));
76	assert(s);
77	/* e->s = s; /
78	xmemcpy(e->key_arr, s->key, MD5_DIGEST_CHARS);
79	e->key = &e->key_arr[0];
80	return e;
81	}
82
83	static void
84	cacheEntryDestroy(CacheEntry *e)
85	{
86	assert(e);
87	xfree(e);
88	}
89
90	static CacheIndex *
91	cacheIndexCreate(const char *name)
92	{
93	CacheIndex *idx;
94	if (!name \|\| !strlen(name))
95	return NULL;
96
97	idx = xcalloc(1, sizeof(CacheIndex));
98	idx->name = name;
99	idx->hash = hash_create(storeKeyHashCmp, 2e6, storeKeyHashHash);
100
101	return idx;
102	}
103
104	static void
105	cacheIndexDestroy(CacheIndex *idx)
106	{
107	hash_link *hashr = NULL;
108	if (idx) {
109	/* destroy hash list contents */
110	for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) {
111	hash_remove_link(idx->hash, hashr);
112	cacheEntryDestroy((CacheEntry*)hashr);
113	}
114	/* destroy the hash table itself */
115	hashFreeMemory(idx->hash);
116	if (idx->digest)
117	cacheDigestDestroy(idx->digest);
118	xfree(idx);
119	}
120	}
121
122	/* makes digest based on currently hashed entries */
123	static void
124	cacheIndexInitDigest(CacheIndex *idx)
125	{
126	hash_link *hashr = NULL;
127	struct timeval t_start, t_end;
128	assert(idx && !idx->digest);
129	fprintf(stderr, "%s: init-ing digest with %d entries\n", idx->name, idx->count);
130	idx->digest = cacheDigestCreate(idx->count); /* no "fat" */
131	gettimeofday(&t_start, NULL);
132	for (hashr = hash_first(idx->hash); hashr; hashr = hash_next(idx->hash)) {
133	cacheDigestAdd(idx->digest, hashr->key);
134	}
135	gettimeofday(&t_end, NULL);
136	assert(idx->digest->count == idx->count);
137	fprintf(stderr, "%s: init-ed digest with %d entries\n",
138	idx->name, idx->digest->count);
139	fprintf(stderr, "%s: init took: %f sec, %f sec/M\n",
140	idx->name,
141	tvSubDsec(t_start, t_end),
142	(double)1e6*tvSubDsec(t_start, t_end)/idx->count);
143	}
144
145	static int
146	cacheIndexAddLog(CacheIndex idx, const char fname)
147	{
148	FILE *file;
149	int scanned_count = 0;
150	assert(idx);
151	assert(fname && strlen(fname));
152
153	file = fopen(fname, "r");
154	if (!file) {
155	fprintf(stderr, "cannot open %s: %s\n", fname, strerror(errno));
156	return 0;
157	}
158	scanned_count = cacheIndexScanCleanPrefix(idx, fname, file);
159	fclose(file);
160	return scanned_count;
161	}
162
163	static void
164	cacheIndexInitReport(CacheIndex *idx)
165	{
166	assert(idx);
167	fprintf(stderr, "%s: bad swap_add: %d\n",
168	idx->name, idx->bad_add_count);
169	fprintf(stderr, "%s: bad swap_del: %d\n",
170	idx->name, idx->bad_del_count);
171	fprintf(stderr, "%s: scanned lines: %d\n",
172	idx->name, idx->scanned_count);
173	}
174
175	#if 0
176	static int
177	cacheIndexGetLogEntry(CacheIndex idx, storeSwapLogData s)
178	{
179	if (!idx->has_log_entry)
180	cacheIndexStepLogEntry();
181	if (idx->has_log_entry) {
182	*s = idx->log_entry_buf;
183	return 1;
184	}
185	return 0;
186	}
187
188	static int
189	cacheIndexStepLogEntry(CacheIndex *idx)
190	{
191	if (fread(&idx->log_entry_buf, sizeof(idx->log_entry_buf), 1, idx->log) == 1) {
192	int op = (int) idx->log_entry_buf.op;
193	idx->scanned_count++;
194	idx->has_log_entry = 1;
195	if (op != SWAP_LOG_ADD && op != SWAP_LOG_DEL) {
196	fprintf(stderr, "%s:%d: unknown swap log action %d\n", idx->log_fname, idx->scanned_count, op);
197	exit(-3);
198	}
199	} else
200	idx->has_log_entry = 0;
201	}
202
203	static int
204	cacheIndexScan(CacheIndex idx, const char fname, FILE *file)
205	{
206	int count = 0;
207	int del_count = 0;
208	storeSwapLogData s;
209	fprintf(stderr, "%s scanning\n", fname);
210	while (fread(&s, sizeof(s), 1, file) == 1) {
211	count++;
212	idx->scanned_count++;
213	if (s.op == SWAP_LOG_ADD) {
214	CacheEntry olde = (CacheEntry ) hash_lookup(idx->hash, s.key);
215	if (olde) {
216	idx->bad_add_count++;
217	} else {
218	CacheEntry *e = cacheEntryCreate(&s);
219	hash_join(idx->hash, (hash_link*) e);
220	idx->count++;
221	}
222	} else
223	if (s.op == SWAP_LOG_DEL) {
224	CacheEntry olde = (CacheEntry ) hash_lookup(idx->hash, s.key);
225	if (!olde)
226	idx->bad_del_count++;
227	else {
228	assert(idx->count);
229	hash_remove_link(idx->hash, (hash_link*) olde);
230	cacheEntryDestroy(olde);
231	idx->count--;
232	}
233	del_count++;
234	} else {
235	fprintf(stderr, "%s:%d: unknown swap log action\n", fname, count);
236	exit(-3);
237	}
238	}
239	fprintf(stderr, "%s scanned %d entries, alloc: %d bytes\n",
240	fname, count,
241	(int)(count*sizeof(CacheEntry)));
242	return count;
243	}
244	#endif
245
246	static int
247	cacheIndexScanCleanPrefix(CacheIndex idx, const char fname, FILE *file)
248	{
249	int count = 0;
250	storeSwapLogData s;
251	fprintf(stderr, "%s scanning\n", fname);
252	while (fread(&s, sizeof(s), 1, file) == 1) {
253	count++;
254	idx->scanned_count++;
255	if (s.op == SWAP_LOG_ADD) {
256	CacheEntry olde = (CacheEntry ) hash_lookup(idx->hash, s.key);
257	if (olde) {
258	idx->bad_add_count++;
259	} else {
260	CacheEntry *e = cacheEntryCreate(&s);
261	hash_join(idx->hash, (hash_link*) e);
262	idx->count++;
263	}
264	} else
265	if (s.op == SWAP_LOG_DEL) {
266	break;
267	} else {
268	fprintf(stderr, "%s:%d: unknown swap log action\n", fname, count);
269	exit(-3);
270	}
271	}
272	fprintf(stderr, "%s scanned %d entries, alloc: %d bytes\n",
273	fname, count,
274	(int)(count*sizeof(CacheEntry)));
275	return count;
276	}
277
278	static int
279	usage(const char *prg_name)
280	{
281	fprintf(stderr, "usage: %s <access_log> <swap_state> ...\n",
282	prg_name);
283	return -1;
284	}
285
286	int
287	main(int argc, char *argv[])
288	{
289	CacheIndex *they = NULL;
290	int i;
291
292	if (argc < 3)
293	return usage(argv[0]);
294
295	they = cacheIndexCreate("they");
296	for (i = 2; i < argc; ++i) {
297	cacheIndexAddLog(they, argv[i]);
298	}
299	cacheIndexInitDigest(they);
300	cacheIndexInitReport(they);
301
302	cacheIndexDestroy(they);
303
304	return 1;
305	}