/*
- * $Id: store.cc,v 1.375 1998/02/04 00:18:04 wessels Exp $
+ * $Id: store.cc,v 1.376 1998/02/04 23:41:00 wessels Exp $
*
* DEBUG: section 20 Storeage Manager
* AUTHOR: Harvest Derived
return;
}
#if USE_ASYNC_IO
- aioCancel(-1, e); /* Make sure all forgotten async ops are cancelled */
+ /*
+ * Make sure all forgotten async ops are cancelled
+ */
+ aioCancel(-1, e);
#else
if (store_rebuilding) {
debug(20, 2) ("storeRelease: Delaying release until store is rebuilt: '%s'\n",
#endif
storeLog(STORE_LOG_RELEASE, e);
if (e->swap_file_number > -1) {
-#if MONOTONIC_STORE
-#if USE_ASYNC_IO
- safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
-#else
- unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL));
-#endif
-#else
- storePutUnusedFileno(e);
-#endif
+ storeUnlinkFileno(e->swap_file_number);
if (e->swap_status == SWAPOUT_DONE)
storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1);
storeDirSwapLog(e, SWAP_LOG_DEL);
void
storeInit(void)
{
+ storeKeyInit();
storeInitHashValues();
store_table = hash_create(storeKeyHashCmp,
store_hash_buckets, storeKeyHashHash);
entry->timestamp = served_date;
}
-#define FILENO_STACK_SIZE 128
-static int fileno_stack[FILENO_STACK_SIZE];
-
-#if !MONOTONIC_STORE
-int
-storeGetUnusedFileno(void)
-{
- int fn;
- if (fileno_stack_count < 1)
- return -1;
- fn = fileno_stack[--fileno_stack_count];
- assert(!storeDirMapBitTest(fn));
- storeDirMapBitSet(fn);
- return fn;
-}
-
-void
-storePutUnusedFileno(StoreEntry * e)
-{
- assert(storeDirMapBitTest(e->swap_file_number));
- storeDirMapBitReset(e->swap_file_number);
- /* If we're still rebuilding the swap state, then we need to avoid the */
- /* race condition where a new object gets pulled in, it expires, gets */
- /* its swapfileno added to the stack, and then that swapfileno gets */
- /* claimed by the rebuild. Must still remove the file though in any */
- /* event to avoid serving up the wrong data. This will leave us with */
- /* a URL pointing to no file at all, but that's okay since it'll fail */
- /* and get removed later anyway. */
- if (store_rebuilding) {
- if (EBIT_TEST(e->flag, ENTRY_VALIDATED))
- safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
- return;
- }
- if (fileno_stack_count < FILENO_STACK_SIZE)
- fileno_stack[fileno_stack_count++] = e->swap_file_number;
- else
-#if USE_ASYNC_IO
- safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
-#else
- unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL));
-#endif
-}
-#endif
-
void
storeRegisterAbort(StoreEntry * e, STABH * cb, void *data)
{
EBIT_CLR(e->flag, DELAY_SENDING);
InvokeHandlers(e);
}
+
+void
+storeUnlinkFileno(int fileno)
+{
+#if USE_ASYNC_IO
+ safeunlink(storeSwapFullPath(fileno, NULL), 1);
+#else
+ unlinkdUnlink(storeSwapFullPath(fileno, NULL));
+#endif
+}
storeClientReadBody(int fd, const char *buf, int len, int flagnotused, void *data)
{
store_client *sc = data;
+ MemObject *mem = sc->mem;
STCB *callback = sc->callback;
assert(sc->disk_op_in_progress != 0);
sc->disk_op_in_progress = 0;
assert(sc->callback != NULL);
debug(20, 3) ("storeClientReadBody: FD %d, len %d\n", fd, len);
+ if (sc->copy_offset == 0 && len > 0 && mem->reply->code == 0)
+ httpParseReplyHeaders(sc->copy_buf, mem->reply);
sc->callback = NULL;
callback(sc->callback_data, sc->copy_buf, len);
}
assert(sc->callback != NULL);
debug(20, 3) ("storeClientReadHeader: FD %d, len %d\n", fd, len);
if (len < 0) {
- debug(20, 3) ("storeClientReadHeader: FD %d: %s\n", fd, xstrerror());
+ debug(20, 3) ("storeClientReadHeader: FD %d: %s\n", fd, xstrerror());
memFree(MEM_DISK_BUF, (void *) buf);
- sc->callback = NULL;
- callback(sc->callback_data, sc->copy_buf, len);
+ sc->callback = NULL;
+ callback(sc->callback_data, sc->copy_buf, len);
return;
}
tlv_list = storeSwapMetaUnpack(buf, &swap_hdr_sz);
if (tlv_list == NULL) {
- debug(20, 1) ("storeClientReadHeader: failed to unpack meta data\n");
+ debug(20, 1) ("storeClientReadHeader: failed to unpack meta data\n");
memFree(MEM_DISK_BUF, (void *) buf);
- sc->callback = NULL;
- callback(sc->callback_data, sc->copy_buf, -1);
+ sc->callback = NULL;
+ callback(sc->callback_data, sc->copy_buf, -1);
return;
}
/*
body_sz = len - swap_hdr_sz;
if (sc->copy_offset < body_sz) {
/*
- * we have (part of )what they want
+ * we have (part of) what they want
*/
copy_sz = XMIN(sc->copy_size, body_sz);
- debug(20,3)("storeClientReadHeader: copying %d bytes of body\n",
- copy_sz);
- xmemcpy(sc->copy_buf, buf+swap_hdr_sz, copy_sz);
+ debug(20, 3) ("storeClientReadHeader: copying %d bytes of body\n",
+ copy_sz);
+ xmemcpy(sc->copy_buf, buf + swap_hdr_sz, copy_sz);
memFree(MEM_DISK_BUF, (void *) buf);
+ if (sc->copy_offset == 0 && len > 0 && mem->reply->code == 0)
+ httpParseReplyHeaders(sc->copy_buf, mem->reply);
sc->callback = NULL;
- callback(sc->callback_data, sc->copy_buf, copy_sz);
+ callback(sc->callback_data, sc->copy_buf, copy_sz);
return;
}
/*
/*
- * $Id: store_dir.cc,v 1.45 1998/02/04 07:01:21 wessels Exp $
+ * $Id: store_dir.cc,v 1.46 1998/02/04 23:41:02 wessels Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
if (!fullpath)
fullpath = fullfilename;
fullpath[0] = '\0';
-#if MONOTONIC_STORE
snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
Config.cacheSwap.swapDirs[dirn].path,
((filn / Config.cacheSwap.swapDirs[dirn].l2) / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l1,
(filn / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l2,
filn);
-#else
- snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X",
- Config.cacheSwap.swapDirs[dirn].path,
- filn % Config.cacheSwap.swapDirs[dirn].l1,
- filn / Config.cacheSwap.swapDirs[dirn].l1 % Config.cacheSwap.swapDirs[dirn].l2,
- filn);
-#endif
return fullpath;
}
if (!fullpath)
fullpath = fullfilename;
fullpath[0] = '\0';
-#if MONOTONIC_STORE
snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
Config.cacheSwap.swapDirs[dirn].path,
((filn / Config.cacheSwap.swapDirs[dirn].l2) / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l1,
(filn / Config.cacheSwap.swapDirs[dirn].l2) % Config.cacheSwap.swapDirs[dirn].l2);
-#else
- snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- Config.cacheSwap.swapDirs[dirn].path,
- filn % Config.cacheSwap.swapDirs[dirn].l1,
- filn / Config.cacheSwap.swapDirs[dirn].l1 % Config.cacheSwap.swapDirs[dirn].l2);
-#endif
return fullpath;
}
int dirn = fn >> SWAP_DIR_SHIFT;
int filn = fn & SWAP_FILE_MASK;
file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn);
-#if !MONOTONIC_STORE
- Config.cacheSwap.swapDirs[dirn].suggest++;
-#endif
}
void
int dirn = fn >> SWAP_DIR_SHIFT;
int filn = fn & SWAP_FILE_MASK;
file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn);
-#if !MONOTONIC_STORE
- if (fn < Config.cacheSwap.swapDirs[dirn].suggest)
- Config.cacheSwap.swapDirs[dirn].suggest = fn;
-#endif
}
int
int dirn = storeMostFreeSwapDir();
SwapDir *SD = &Config.cacheSwap.swapDirs[dirn];
int filn = file_map_allocate(SD->map, SD->suggest);
-#if MONOTONIC_STORE
SD->suggest = filn + 1;
-#endif
return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK);
}
}
FILE *
-storeDirOpenTmpSwapLog(int dirn, int *clean_flag)
+storeDirOpenTmpSwapLog(int dirn, int *clean_flag, int *zero_flag)
{
char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL));
char *clean_path = xstrdup(storeDirSwapLogFile(dirn, ".last-clean"));
safe_free(new_path);
return NULL;
}
+ *zero_flag = log_sb.st_size == 0 ? 1 : 0;
/* close the existing write-only FD */
if (SD->swaplog_fd >= 0)
file_close(SD->swaplog_fd);
assert(dirn < Config.cacheSwap.n_configured);
if (fd[dirn] < 0)
continue;
- s = (void *) outbuf[dirn] + outbufoffset[dirn];
+ s = (void *) outbuf[dirn] + outbufoffset[dirn];
outbufoffset[dirn] += sizeof(storeSwapData);
- memset(s, '\0', sizeof(storeSwapData));
- s->op = (char) SWAP_LOG_ADD;
- s->swap_file_number = e->swap_file_number;
- s->timestamp = e->timestamp;
- s->lastref = e->lastref;
- s->expires = e->expires;
- s->lastmod = e->lastmod;
- s->object_len = e->object_len;
- s->refcount = e->refcount;
- s->flags = e->flag;
- xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
+ memset(s, '\0', sizeof(storeSwapData));
+ s->op = (char) SWAP_LOG_ADD;
+ s->swap_file_number = e->swap_file_number;
+ s->timestamp = e->timestamp;
+ s->lastref = e->lastref;
+ s->expires = e->expires;
+ s->lastmod = e->lastmod;
+ s->object_len = e->object_len;
+ s->refcount = e->refcount;
+ s->flags = e->flag;
+ xmemcpy(s->key, e->key, MD5_DIGEST_CHARS);
/* buffered write */
if (outbufoffset[dirn] + sizeof(storeSwapData) > CLEAN_BUF_SZ) {
if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) {
debug(50, 0) ("storeDirWriteCleanLogs: %s: %s\n",
- new[dirn], xstrerror());
+ new[dirn], xstrerror());
debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n");
file_close(fd[dirn]);
fd[dirn] = -1;
#include "squid.h"
+static cache_key null_key[MD5_DIGEST_CHARS];
+
const char *
storeKeyText(const unsigned char *key)
{
return 0x8000;
return 0x10000;
}
+
+int
+storeKeyNull(const cache_key * key)
+{
+ if (memcmp(key, null_key, MD5_DIGEST_CHARS) == 0)
+ return 1;
+ else
+ return 0;
+}
+
+void
+storeKeyInit(void)
+{
+ memset(null_key, '\0', MD5_DIGEST_CHARS);
+}
+
#include "squid.h"
static char *storeLogTags[] =
storelog_fd = -1;
else
storelog_fd = file_open(Config.Log.store,
- O_WRONLY | O_CREAT,
- NULL,
- NULL,
- NULL);
+ O_WRONLY | O_CREAT,
+ NULL,
+ NULL,
+ NULL);
if (storelog_fd < 0)
- debug(20, 1) ("Store logging disabled\n");
+ debug(20, 1) ("Store logging disabled\n");
}
int objcount; /* # objects successfully reloaded */
int expcount; /* # objects expired */
int linecount; /* # lines parsed from cache logfile */
+ int statcount; /* # entries from directory walking */
int clashcount; /* # swapfile clashes avoided */
- int cancelcount; /* # objects cancelled */
int dupcount; /* # duplicates purged */
int invalid; /* # bad lines */
int badflags; /* # bad e->flags */
int need_to_validate;
+ int zero_object_len;
time_t start;
time_t stop;
} RebuildState;
LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE);
StoreEntry *e = NULL;
StoreEntry tmpe;
+ cache_key key[MD5_DIGEST_CHARS];
int sfileno = 0;
int count;
int size;
struct stat fst;
- static int filecount;
int hdr_len;
int fd = -1;
tlv *tlv_list;
tlv *t;
+ double x;
assert(d != NULL);
debug(20, 3) ("storeRebuildFromDirectory: DIR #%s\n", d->dirn);
for (count = 0; count < d->speed; count++) {
fd = -1;
continue;
}
- if ((++filecount & 0x3FFF) == 0)
- debug(20, 1) (" %7d files processed so far.\n", RebuildState.linecount);
+ if ((++RebuildState.statcount & 0x3FFF) == 0)
+ debug(20, 1) (" %7d files opened so far.\n",
+ RebuildState.statcount);
debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno);
if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) {
debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n",
tlv_list = storeSwapMetaUnpack(hdr_buf, &hdr_len);
if (tlv_list == NULL) {
debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n");
- safeunlink(storeSwapFullPath(sfileno, NULL), 1);
+ storeUnlinkFileno(sfileno);
continue;
}
- debug(20, 1) ("storeRebuildFromDirectory: successful swap meta unpacking\n");
- storeKeyFree(tmpe.key);
+ debug(20, 3) ("storeRebuildFromDirectory: successful swap meta unpacking\n");
+ memset(key, '\0', MD5_DIGEST_CHARS);
memset(&tmpe, '\0', sizeof(StoreEntry));
for (t = tlv_list; t; t = t->next) {
switch (t->type) {
case STORE_META_KEY:
- tmpe.key = storeKeyDup(t->value);
+ assert(t->length == MD5_DIGEST_CHARS);
+ xmemcpy(key, t->value, MD5_DIGEST_CHARS);
break;
case STORE_META_STD:
+ assert(t->length == STORE_HDR_METASIZE);
xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
break;
default:
}
storeSwapTLVFree(tlv_list);
tlv_list = NULL;
- if (tmpe.key == NULL) {
+ if (storeKeyNull(key)) {
debug(20, 1) ("storeRebuildFromDirectory: NULL key\n");
- safeunlink(storeSwapFullPath(sfileno, NULL), 1);
+ storeUnlinkFileno(sfileno);
continue;
}
+ tmpe.key = key;
if (tmpe.object_len == 0) {
- debug(20, 1) ("storeRebuildFromDirectory: ZERO object length\n");
- safeunlink(storeSwapFullPath(sfileno, NULL), 1);
+ RebuildState.invalid++;
+ x = log(++RebuildState.zero_object_len) / log(10.0);
+ if (0.0 == x - (double) (int) x)
+ debug(20, 1) ("WARNING: %d swapfiles found with ZERO length\n",
+ RebuildState.zero_object_len);
+ storeUnlinkFileno(sfileno);
continue;
}
/* check sizes */
if (hdr_len + tmpe.object_len != fst.st_size) {
debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d+%d!=%d\n",
hdr_len, tmpe.object_len, fst.st_size);
- safeunlink(storeSwapFullPath(sfileno, NULL), 1);
+ storeUnlinkFileno(sfileno);
continue;
}
if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) {
- safeunlink(storeSwapFullPath(sfileno, NULL), 1);
+ storeUnlinkFileno(sfileno);
RebuildState.badflags++;
continue;
}
- if ((e = storeGet(tmpe.key)) != NULL) {
+ if ((e = storeGet(key)) != NULL) {
/* URL already exists, this swapfile not being used */
/* junk old, load new */
storeRelease(e); /* release old entry */
}
RebuildState.objcount++;
storeEntryDump(&tmpe, 5);
- e = storeAddDiskRestore(tmpe.key,
+ e = storeAddDiskRestore(key,
sfileno,
(int) tmpe.object_len,
tmpe.expires,
continue;
}
if ((++RebuildState.linecount & 0x3FFF) == 0)
- debug(20, 1) (" %7d Entries read so far.\n", RebuildState.linecount);
+ debug(20, 1) (" %7d Entries read so far.\n",
+ RebuildState.linecount);
if (!storeDirValidFileno(s.swap_file_number)) {
RebuildState.invalid++;
continue;
debug(20, 1) ("Finished rebuilding storage disk.\n");
debug(20, 1) (" %7d Entries read from previous logfile.\n",
RebuildState.linecount);
+ debug(20, 1) (" %7d Entries scanned from swap files.\n",
+ RebuildState.statcount);
debug(20, 1) (" %7d Invalid entries.\n", RebuildState.invalid);
debug(20, 1) (" %7d With invalid flags.\n", RebuildState.badflags);
debug(20, 1) (" %7d Objects loaded.\n", RebuildState.objcount);
debug(20, 1) (" %7d Objects expired.\n", RebuildState.expcount);
- debug(20, 1) (" %7d Objects cancelled.\n", RebuildState.cancelcount);
debug(20, 1) (" %7d Duplicate URLs purged.\n", RebuildState.dupcount);
debug(20, 1) (" %7d Swapfile clashes avoided.\n", RebuildState.clashcount);
debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n",
{
rebuild_dir *d;
int clean = 0;
+ int zero = 0;
FILE *fp;
int i;
memset(&RebuildState, '\0', sizeof(RebuildState));
* use storeRebuildFromDirectory() to open up each file
* and suck in the meta data.
*/
- fp = storeDirOpenTmpSwapLog(i, &clean);
- if (fp == NULL) {
+ fp = storeDirOpenTmpSwapLog(i, &clean, &zero);
+ if (fp == NULL || zero) {
d->rebuild_func = storeRebuildFromDirectory;
} else {
d->rebuild_func = storeRebuildFromSwapLog;
xfree(ctrlp->path);
xfree(ctrlp);
}
-
+
#include "squid.h"
static tlv **
swapout_ctrl_t *ctrlp;
LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN);
storeLockObject(e);
-#if !MONOTONIC_STORE
- if ((e->swap_file_number = storeGetUnusedFileno()) < 0)
-#endif
- e->swap_file_number = storeDirMapAllocate();
+ e->swap_file_number = storeDirMapAllocate();
storeSwapFullPath(e->swap_file_number, swapfilename);
ctrlp = xmalloc(sizeof(swapout_ctrl_t));
ctrlp->swapfilename = xstrdup(swapfilename);
flag);
e->swap_status = SWAPOUT_NONE;
if (e->swap_file_number > -1) {
-#if MONOTONIC_STORE
-#if USE_ASYNC_IO
- safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
-#else
- unlinkdUnlink(storeSwapFullPath(e->swap_file_number, NULL));
-#endif
-#else
- storePutUnusedFileno(e);
-#endif
+ storeUnlinkFileno(e->swap_file_number);
e->swap_file_number = -1;
}
if (flag == DISK_NO_SPACE_LEFT) {