From: wessels <> Date: Wed, 4 Feb 1998 07:18:00 +0000 (+0000) Subject: - Changed swap.state to be binary X-Git-Tag: SQUID_3_0_PRE1~4191 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=5830cdb3384654baf21ef474b9d04545d6a468b7;p=thirdparty%2Fsquid.git - Changed swap.state to be binary - Added Stew's transaction logging to storeDirSwapLog(). - Munged store_rebuild.c to support rebuilding one cache_dir from swap.state, and another from walking the directory. --- diff --git a/src/disk.cc b/src/disk.cc index 804f3fadb5..e610add521 100644 --- a/src/disk.cc +++ b/src/disk.cc @@ -1,6 +1,6 @@ /* - * $Id: disk.cc,v 1.99 1998/02/03 22:08:11 wessels Exp $ + * $Id: disk.cc,v 1.100 1998/02/04 00:18:00 wessels Exp $ * * DEBUG: section 6 Disk I/O Routines * AUTHOR: Harvest Derived @@ -358,7 +358,7 @@ diskHandleWriteComplete(void *data, int len, int errcode) void file_write(int fd, off_t file_offset, - char *ptr_to_buf, + void *ptr_to_buf, int len, DWCB handle, void *handle_data, diff --git a/src/enums.h b/src/enums.h index 92c49f587c..a337c8f813 100644 --- a/src/enums.h +++ b/src/enums.h @@ -465,8 +465,14 @@ enum { }; enum { - STORE_LOG_CREATE, - STORE_LOG_SWAPIN, - STORE_LOG_SWAPOUT, - STORE_LOG_RELEASE + STORE_LOG_CREATE, + STORE_LOG_SWAPIN, + STORE_LOG_SWAPOUT, + STORE_LOG_RELEASE +}; + +enum { + SWAP_LOG_NOP, + SWAP_LOG_ADD, + SWAP_LOG_DEL }; diff --git a/src/globals.h b/src/globals.h index fa818cac07..fbdfa6584f 100644 --- a/src/globals.h +++ b/src/globals.h @@ -1,6 +1,6 @@ /* - * $Id: globals.h,v 1.31 1998/02/03 22:08:12 wessels Exp $ + * $Id: globals.h,v 1.32 1998/02/04 00:18:01 wessels Exp $ */ extern FILE *debug_log; /* NULL */ @@ -97,9 +97,8 @@ extern char *err_type_str[]; extern char *icp_opcode_str[]; extern struct radix_node_head *AS_tree_head; extern double request_failure_ratio; /* 0.0 */ -extern int store_hash_buckets; /* 0 */ -extern hash_table *store_table; /* NULL */ -extern size_t cacheKeySize; +extern int store_hash_buckets; /* 0 */ +extern hash_table *store_table; /* NULL */ extern dlink_list store_list; #ifdef HAVE_SYSLOG diff --git a/src/protos.h b/src/protos.h index 2a0473ee39..63ca70ca9a 100644 --- a/src/protos.h +++ b/src/protos.h @@ -122,7 +122,7 @@ extern void _db_print(); extern int file_open(const char *path, int mode, FOCB *, void *callback_data, void *tag); extern void file_close(int fd); -extern void file_write(int, off_t, char *, int len, DWCB *, void *, FREE *); +extern void file_write(int, off_t, void *, int len, DWCB *, void *, FREE *); extern int file_read(int, char *, int, off_t, DRCB *, void *); extern int disk_init(void); extern int diskWriteIsComplete(int); @@ -493,7 +493,7 @@ extern void storeDirCloseTmpSwapLog(int dirn); extern void storeDirOpenSwapLogs(void); extern void storeDirCloseSwapLogs(void); extern char *storeDirSwapLogFile(int, const char *); -extern void storeDirSwapLog(const StoreEntry *); +extern void storeDirSwapLog(const StoreEntry *, int op); extern int storeDirNumber(int fileno); extern void storeDirUpdateSwapSize(int fn, size_t size, int sign); extern int storeDirProperFileno(int dirn, int fn); @@ -506,16 +506,15 @@ extern int storeDirValidFileno(int fn); /* * store_swapmeta.c */ -char *storeSwapMetaPack(tlv *tlv_list, int *length); +char *storeSwapMetaPack(tlv * tlv_list, int *length); tlv *storeSwapMetaBuild(StoreEntry * e); tlv *storeSwapMetaUnpack(const char *buf, int *hdrlen); -void storeSwapTLVFree(tlv *n); +void storeSwapTLVFree(tlv * n); /* * store_rebuild.c */ extern void storeDoRebuildFromSwapFiles(void *data); -extern void storeConvert(void); extern void storeConvertFile(const cache_key * key, int file_number, int size, @@ -526,8 +525,7 @@ extern void storeConvertFile(const cache_key * key, u_short refcount, u_short flags, int clean); -extern int storeGetNextFile(int *sfileno, int *size); -extern StoreEntry * storeAddDiskRestore(const cache_key * key, +extern StoreEntry *storeAddDiskRestore(const cache_key * key, int file_number, int size, time_t expires, @@ -537,11 +535,10 @@ extern StoreEntry * storeAddDiskRestore(const cache_key * key, u_num32 refcount, u_num32 flags, int clean); -extern void storeDoConvertFromLog(void *data); extern void storeCleanup(void *datanotused); extern void storeValidate(StoreEntry *, STVLDCB, void *callback_data, void *tag); extern void storeValidateComplete(void *data, int retcode, int errcode); -extern void storeStartRebuildFromDisk(void); +extern void storeRebuildStart(void); /* @@ -563,9 +560,9 @@ extern void storeSwapOutFileClose(StoreEntry * e); /* * store_client.c */ -extern store_client * storeClientListSearch(const MemObject * mem, void *data); +extern store_client *storeClientListSearch(const MemObject * mem, void *data); extern void storeClientListAdd(StoreEntry * e, void *data); -extern void storeClientCopy(StoreEntry *, off_t , off_t , size_t , char *, STCB *, void *); +extern void storeClientCopy(StoreEntry *, off_t, off_t, size_t, char *, STCB *, void *); extern int storeClientCopyPending(StoreEntry * e, void *data); extern int storeUnregister(StoreEntry * e, void *data); extern off_t storeLowestMemReaderOffset(const StoreEntry * entry); diff --git a/src/store.cc b/src/store.cc index 1718461f64..2130899469 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.374 1998/02/03 22:08:14 wessels Exp $ + * $Id: store.cc,v 1.375 1998/02/04 00:18:04 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -716,9 +716,7 @@ storeRelease(StoreEntry * e) storeExpireNow(e); storeSetPrivateKey(e); EBIT_SET(e->flag, RELEASE_REQUEST); - e->object_len = -(e->object_len); - storeDirSwapLog(e); - e->object_len = -(e->object_len); + storeDirSwapLog(e, SWAP_LOG_DEL); return; } #endif @@ -735,9 +733,7 @@ storeRelease(StoreEntry * e) #endif if (e->swap_status == SWAPOUT_DONE) storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); - e->object_len = -(e->object_len); - storeDirSwapLog(e); - e->object_len = -(e->object_len); + storeDirSwapLog(e, SWAP_LOG_DEL); } storeSetMemStatus(e, NOT_IN_MEMORY); destroy_StoreEntry(e); @@ -845,9 +841,9 @@ storeInit(void) fatal(tmp_error_buf); } storeDirOpenSwapLogs(); - storeStartRebuildFromDisk(); store_list.head = store_list.tail = NULL; inmem_list.head = inmem_list.tail = NULL; + storeRebuildStart(); } void diff --git a/src/store_dir.cc b/src/store_dir.cc index dc3d221c96..bbea1bb2a2 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir.cc,v 1.43 1998/02/03 22:08:16 wessels Exp $ + * $Id: store_dir.cc,v 1.44 1998/02/04 00:18:05 wessels Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -221,7 +221,7 @@ storeDirValidFileno(int fn) SwapDir *SD; if (dirn > Config.cacheSwap.n_configured) return 0; - SD = &Config.cacheSwap.swapDirs[dirn]; + SD = &Config.cacheSwap.swapDirs[dirn]; if (filn > SD->map->max_n_files) return 0; return 1; @@ -290,32 +290,34 @@ storeDirProperFileno(int dirn, int fn) } void -storeDirSwapLog(const StoreEntry * e) +storeDirSwapLog(const StoreEntry * e, int op) { - LOCAL_ARRAY(char, logmsg, MAX_URL << 1); + storeSwapData *s = xcalloc(1, sizeof(storeSwapData)); int dirn; - - assert(e->swap_file_number >= 0); dirn = e->swap_file_number >> SWAP_DIR_SHIFT; assert(dirn < Config.cacheSwap.n_configured); -#if !USE_ASYNC_IO - assert(!EBIT_TEST(e->flag, KEY_PRIVATE)); -#endif - /* Note this printf format appears in storeWriteCleanLog() too */ - snprintf(logmsg, MAX_URL << 1, "%08x %08x %08x %08x %08x %d %6d %08x %s\n", - (int) e->swap_file_number, - (int) e->timestamp, - (int) e->lastref, - (int) e->expires, - (int) e->lastmod, - e->object_len, - e->refcount, - e->flag, - storeKeyText(e->key)); + if (op == SWAP_LOG_ADD) { + assert(!EBIT_TEST(e->flag, KEY_PRIVATE)); + assert(e->swap_file_number >= 0); + } + if (op == SWAP_LOG_DEL) { + if (e->swap_file_number < 0) /* was never swapped out */ + return; + } + s->op = (char) op; + s->swap_file_number = e->swap_file_number; + s->timestamp = e->timestamp; + s->lastref = e->lastref; + s->expires = e->expires; + s->lastmod = e->lastmod; + s->object_len = e->object_len; + s->refcount = e->refcount; + s->flags = e->flag; + xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd, -1, - xstrdup(logmsg), - strlen(logmsg), + s, + sizeof(storeSwapData), NULL, NULL, xfree); diff --git a/src/store_key_md5.cc b/src/store_key_md5.cc index aae4c2dee6..28a7739dde 100644 --- a/src/store_key_md5.cc +++ b/src/store_key_md5.cc @@ -1,10 +1,5 @@ #include "squid.h" -/* - * Size of a cache_key in bytes - */ -size_t cacheKeySize = MD5_DIGEST_CHARS; - const char * storeKeyText(const unsigned char *key) { diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc index 7256daba91..0e1df4640d 100644 --- a/src/store_rebuild.cc +++ b/src/store_rebuild.cc @@ -2,16 +2,29 @@ #define STORE_META_BUFSZ 4096 + +typedef struct _rebuild_dir rebuild_dir; +typedef RBHD(rebuild_dir * d); + struct _rebuild_dir { int dirn; FILE *log; int speed; int clean; - struct _rebuild_dir *next; + int curlvl1; + int curlvl2; + int flag; + int done; + int in_dir; + int fn; + struct dirent *entry; + DIR *td; + RBHD *rebuild_func; + rebuild_dir *next; }; struct storeRebuildState { - struct _rebuild_dir *rebuild_dir; + rebuild_dir *rebuild_dir; int objcount; /* # objects successfully reloaded */ int expcount; /* # objects expired */ int linecount; /* # lines parsed from cache logfile */ @@ -23,9 +36,7 @@ struct storeRebuildState { int need_to_validate; time_t start; time_t stop; - char *line_in; - size_t line_in_sz; -}; +} RebuildState; typedef struct valid_ctrl_t { struct stat *sb; @@ -34,50 +45,53 @@ typedef struct valid_ctrl_t { void *callback_data; } valid_ctrl_t; -static void storeRebuiltFromDisk(struct storeRebuildState *data); +static RBHD storeRebuildFromDirectory; +static RBHD storeRebuildFromSwapLog; +static void storeRebuildComplete(void); +static EVH storeRebuildADirectory; +static int storeGetNextFile(rebuild_dir *, int *sfileno, int *size); -void -storeRebuildRawFile(void *data) +static int +storeRebuildFromDirectory(rebuild_dir * d) { - struct storeRebuildState *RB = data; LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE); StoreEntry *e = NULL; StoreEntry tmpe; int sfileno = 0; int count; int size; - struct _rebuild_dir *d = RB->rebuild_dir; struct stat fst; static int filecount; int hdr_len; int fd = -1; tlv *tlv_list; tlv *t; - debug(20, 3) (" Starting storeRebuildRawFile at speed %d\n", d->speed); + assert(d != NULL); + debug(20, 3) ("storeRebuildFromDirectory: DIR #%s\n", d->dirn); for (count = 0; count < d->speed; count++) { assert(fd == -1); - fd = storeGetNextFile(&sfileno, &size); + fd = storeGetNextFile(d, &sfileno, &size); if (fd == -2) { - debug(20, 1) ("storeRebuildRawFile: done!\n"); + debug(20, 1) ("storeRebuildFromDirectory: done!\n"); store_rebuilding = 0; - return; + return -1; } else if (fd < 0) { continue; } assert(fd > -1); /* lets get file stats here */ if (fstat(fd, &fst) < 0) { - debug(20, 1) ("storeRebuildRawFile: fstat(FD %d): %s\n", + debug(20, 1) ("storeRebuildFromDirectory: fstat(FD %d): %s\n", fd, xstrerror()); file_close(fd); fd = -1; continue; } if ((++filecount & 0x3FFF) == 0) - debug(20, 1) (" %7d files processed so far.\n", RB->linecount); + debug(20, 1) (" %7d files processed so far.\n", RebuildState.linecount); debug(20, 9) ("file_in: fd=%d %08x\n", fd, sfileno); if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) { - debug(20, 1) ("storeRebuildRawFile: read(FD %d): %s\n", + debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n", fd, xstrerror()); file_close(fd); fd = -1; @@ -88,11 +102,11 @@ storeRebuildRawFile(void *data) hdr_len = 0; tlv_list = storeSwapMetaUnpack(hdr_buf, &hdr_len); if (tlv_list == NULL) { - debug(20, 1) ("storeRebuildRawFile: failed to get meta data\n"); + debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } - debug(20,1)("storeRebuildRawFile: successful swap meta unpacking\n"); + debug(20, 1) ("storeRebuildFromDirectory: successful swap meta unpacking\n"); storeKeyFree(tmpe.key); memset(&tmpe, '\0', sizeof(StoreEntry)); for (t = tlv_list; t; t = t->next) { @@ -110,34 +124,34 @@ storeRebuildRawFile(void *data) storeSwapTLVFree(tlv_list); tlv_list = NULL; if (tmpe.key == NULL) { - debug(20, 1) ("storeRebuildRawFile: NULL key\n"); + debug(20, 1) ("storeRebuildFromDirectory: NULL key\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } if (tmpe.object_len == 0) { - debug(20, 1) ("storeRebuildRawFile: ZERO object length\n"); + debug(20, 1) ("storeRebuildFromDirectory: ZERO object length\n"); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } /* check sizes */ if (hdr_len + tmpe.object_len != fst.st_size) { - debug(20, 1) ("storeRebuildRawFile: SIZE MISMATCH %d+%d!=%d\n", + debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d+%d!=%d\n", hdr_len, tmpe.object_len, fst.st_size); safeunlink(storeSwapFullPath(sfileno, NULL), 1); continue; } if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) { safeunlink(storeSwapFullPath(sfileno, NULL), 1); - RB->badflags++; + RebuildState.badflags++; continue; } if ((e = storeGet(tmpe.key)) != NULL) { /* URL already exists, this swapfile not being used */ /* junk old, load new */ storeRelease(e); /* release old entry */ - RB->dupcount++; + RebuildState.dupcount++; } - RB->objcount++; + RebuildState.objcount++; storeEntryDump(&tmpe, 5); e = storeAddDiskRestore(tmpe.key, sfileno, @@ -150,39 +164,137 @@ storeRebuildRawFile(void *data) tmpe.flag, /* flags */ d->clean); } - eventAdd("storeRebuild", storeRebuildRawFile, RB, 0); + return count; } - -void -storeConvert(void) +static int +storeRebuildFromSwapLog(rebuild_dir * d) { - int i; - struct storeRebuildState *RB; - struct _rebuild_dir *d; - FILE *fp; - int clean; - RB = xcalloc(1, sizeof(struct storeRebuildState)); - RB->start = squid_curtime; - for (i = 0; i < Config.cacheSwap.n_configured; i++) { - fp = storeDirOpenTmpSwapLog(i, &clean); - if (fp == NULL) + StoreEntry *e = NULL; + storeSwapData s; + size_t ss = sizeof(storeSwapData); + int count; + int used; /* is swapfile already in use? */ + int newer; /* is the log entry newer than current entry? */ + assert(d != NULL); + /* load a number of objects per invocation */ + for (count = 0; count < d->speed; count++) { + if (fread(&s, ss, 1, d->log) != ss) { + debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); + fclose(d->log); + d->log = NULL; + storeDirCloseTmpSwapLog(d->dirn); + return -1; + } + if (s.op == SWAP_LOG_ADD) { + (void) 0; + } else if (s.op == SWAP_LOG_DEL) { + /* XXX we should just storeRelease()? */ continue; - d = xcalloc(1, sizeof(struct _rebuild_dir)); - d->dirn = i; - d->log = fp; - d->clean = clean; - d->speed = 1 << 30; - d->next = RB->rebuild_dir; - RB->rebuild_dir = d; - if (!clean) - RB->need_to_validate = 1; - debug(20, 1) ("Converting storage in Cache Dir #%d (%s)\n", - i, clean ? "CLEAN" : "DIRTY"); + } else { + debug(20, 1) ("storeRebuildFromSwapLog: Invalid swap entry\n"); + RebuildState.invalid++; + continue; + } + if ((++RebuildState.linecount & 0x3FFF) == 0) + debug(20, 1) (" %7d Entries read so far.\n", RebuildState.linecount); + if (!storeDirValidFileno(s.swap_file_number)) { + RebuildState.invalid++; + continue; + } + if (EBIT_TEST(s.flags, KEY_PRIVATE)) { + RebuildState.badflags++; + continue; + } + e = storeGet(s.key); + used = storeDirMapBitTest(s.swap_file_number); + /* If this URL already exists in the cache, does the swap log + * appear to have a newer entry? Compare 'lastref' from the + * swap log to e->lastref. */ + newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0; + if (used && !newer) { + /* log entry is old, ignore it */ + RebuildState.clashcount++; + continue; + } else if (used && e && e->swap_file_number == s.swap_file_number) { + /* swapfile taken, same URL, newer, update meta */ + e->lastref = s.timestamp; + e->timestamp = s.timestamp; + e->expires = s.expires; + e->lastmod = s.lastmod; + e->flag |= s.flags; + e->refcount += s.refcount; + continue; + } else if (used) { + /* swapfile in use, not by this URL, log entry is newer */ + /* This is sorta bad: the log entry should NOT be newer at this + * point. If the log is dirty, the filesize check should have + * caught this. If the log is clean, there should never be a + * newer entry. */ + debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", + s.swap_file_number); + /* I'm tempted to remove the swapfile here just to be safe, + * but there is a bad race condition in the NOVM version if + * the swapfile has recently been opened for writing, but + * not yet opened for reading. Because we can't map + * swapfiles back to StoreEntrys, we don't know the state + * of the entry using that file. */ + /* We'll assume the existing entry is valid, probably because + * were in a slow rebuild and the the swap file number got taken + * and the validation procedure hasn't run. */ + assert(RebuildState.need_to_validate); + RebuildState.clashcount++; + continue; + } else if (e) { + /* URL already exists, this swapfile not being used */ + /* junk old, load new */ + storeRelease(e); /* release old entry */ + RebuildState.dupcount++; + } else { + /* URL doesnt exist, swapfile not in use */ + /* load new */ + (void) 0; + } + /* update store_swap_size */ + RebuildState.objcount++; + e = storeAddDiskRestore(s.key, + s.swap_file_number, + s.object_len, + s.expires, + s.timestamp, + s.lastref, + s.lastmod, + s.refcount, + s.flags, + d->clean); + storeDirSwapLog(e, SWAP_LOG_ADD); + } + return count; +} + +static void +storeRebuildADirectory(void *unused) +{ + int count; + rebuild_dir *d; + rebuild_dir **D; + if ((d = RebuildState.rebuild_dir) == NULL) { + storeRebuildComplete(); + return; + } + count = d->rebuild_func(d); + RebuildState.rebuild_dir = d->next; + if (count < 0) { + xfree(d); + } else { + for (D = &RebuildState.rebuild_dir; *D; D = &(*D)->next); + *D = d; + d->next = NULL; } - RB->line_in_sz = 4096; - RB->line_in = xcalloc(1, RB->line_in_sz); - storeDoConvertFromLog(RB); + if (opt_foreground_rebuild) + storeRebuildADirectory(NULL); + else + eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0); } void @@ -229,77 +341,76 @@ storeConvertFile(const cache_key * key, storeSwapTLVFree(tlv_list); } -int -storeGetNextFile(int *sfileno, int *size) +static int +storeGetNextFile(rebuild_dir * d, int *sfileno, int *size) { - static int dirn, curlvl1, curlvl2, flag, done, in_dir, fn; - static struct dirent *entry; - static DIR *td; int fd = -1; int used = 0; LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); LOCAL_ARRAY(char, fullpath, SQUID_MAXPATHLEN); - debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", flag, - dirn, curlvl1, curlvl2); - if (done) + debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n", + d->flag, + d->dirn, + d->curlvl1, + d->curlvl2); + if (d->done) return -2; - while (fd < 0 && done == 0) { + while (fd < 0 && d->done == 0) { fd = -1; - if (!flag) { /* initialize, open first file */ - done = dirn = curlvl1 = curlvl2 = in_dir = 0; - flag = 1; + if (!d->flag) { /* initialize, open first file */ + d->done = d->dirn = d->curlvl1 = d->curlvl2 = d->in_dir = 0; + d->flag = 1; assert(Config.cacheSwap.n_configured > 0); } - if (!in_dir) { /* we need to read in a new directory */ + if (!d->in_dir) { /* we need to read in a new directory */ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", - Config.cacheSwap.swapDirs[dirn].path, - curlvl1, curlvl2); - if (flag && td) - closedir(td); - td = opendir(fullpath); - entry = readdir(td); /* skip . and .. */ - entry = readdir(td); + Config.cacheSwap.swapDirs[d->dirn].path, + d->curlvl1, d->curlvl2); + if (d->flag && d->td) + closedir(d->td); + d->td = opendir(fullpath); + d->entry = readdir(d->td); /* skip . and .. */ + d->entry = readdir(d->td); if (errno == ENOENT) { debug(20, 3) ("storeGetNextFile: directory does not exist!.\n"); } debug(20, 3) ("storeGetNextFile: Directory %s/%02X/%02X\n", - Config.cacheSwap.swapDirs[dirn].path, - curlvl1, curlvl2); + Config.cacheSwap.swapDirs[d->dirn].path, + d->curlvl1, d->curlvl2); } - if ((entry = readdir(td))) { - in_dir++; - if (sscanf(entry->d_name, "%x", sfileno) != 1) { + if ((d->entry = readdir(d->td))) { + d->in_dir++; + if (sscanf(d->entry->d_name, "%x", sfileno) != 1) { debug(20, 3) ("storeGetNextFile: invalid %s\n", - entry->d_name); + d->entry->d_name); continue; } - fn = *sfileno; - fn = storeDirProperFileno(dirn, fn); - *sfileno = fn; - used = storeDirMapBitTest(fn); + d->fn = *sfileno; + d->fn = storeDirProperFileno(d->dirn, d->fn); + *sfileno = d->fn; + used = storeDirMapBitTest(d->fn); if (used) { debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n"); continue; } snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", - fullpath, entry->d_name); + fullpath, d->entry->d_name); debug(20, 3) ("storeGetNextFile: Opening %s\n", fullfilename); fd = file_open(fullfilename, O_RDONLY, NULL, NULL, NULL); continue; } -#if 0 - else if (!in_dir) - debug(20, 3) ("storeGetNextFile: empty dir.\n"); -#endif - in_dir = 0; - if ((curlvl2 = (curlvl2 + 1) % Config.cacheSwap.swapDirs[dirn].l2) != 0) + d->in_dir = 0; + d->curlvl2 = (d->curlvl2 + 1) % Config.cacheSwap.swapDirs[d->dirn].l2; + if (d->curlvl2 != 0) continue; - if ((curlvl1 = (curlvl1 + 1) % Config.cacheSwap.swapDirs[dirn].l1) != 0) + d->curlvl1 = (d->curlvl1 + 1) % Config.cacheSwap.swapDirs[d->dirn].l1; + if (d->curlvl1 != 0) continue; - if ((dirn = (dirn + 1) % Config.cacheSwap.n_configured) != 0) + d->dirn = (d->dirn + 1) % Config.cacheSwap.n_configured; + if (d->dirn != 0) continue; else - done = 1; + d->done = 1; } return fd; } @@ -346,187 +457,6 @@ storeAddDiskRestore(const cache_key * key, return e; } -/* convert storage .. this is the old storeDoRebuildFromDisk() */ - -void -storeDoConvertFromLog(void *data) -{ - struct storeRebuildState *RB = data; - LOCAL_ARRAY(char, swapfile, MAXPATHLEN); - LOCAL_ARRAY(char, keytext, MAX_URL); - StoreEntry *e = NULL; - time_t expires; - time_t timestamp; - time_t lastref; - time_t lastmod; - int scan1; - int scan2; - int scan3; - int scan4; - int scan5; - int scan6; - int scan7; - off_t size; - int sfileno = 0; - int count; - int x; - struct _rebuild_dir *d; - struct _rebuild_dir **D; - int used; /* is swapfile already in use? */ - int newer; /* is the log entry newer than current entry? */ - const cache_key *key; - /* load a number of objects per invocation */ - if ((d = RB->rebuild_dir) == NULL) { - debug(20, 3) ("Done Converting, here are the stats.\n"); - storeRebuiltFromDisk(RB); - return; - } - for (count = 0; count < d->speed; count++) { - if (fgets(RB->line_in, RB->line_in_sz, d->log) == NULL) { - debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); - fclose(d->log); - d->log = NULL; - storeDirCloseTmpSwapLog(d->dirn); - RB->rebuild_dir = d->next; - safe_free(d); - eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); - return; - } - if ((++RB->linecount & 0x3FFF) == 0) - debug(20, 1) (" %7d Lines read so far.\n", RB->linecount); - debug(20, 9) ("line_in: %s", RB->line_in); - if (RB->line_in[0] == '\0') - continue; - if (RB->line_in[0] == '\n') - continue; - if (RB->line_in[0] == '#') - continue; - keytext[0] = '\0'; - sfileno = 0; - scan1 = 0; - scan2 = 0; - scan3 = 0; - scan4 = 0; - scan5 = 0; - scan6 = 0; - scan7 = 0; - x = sscanf(RB->line_in, "%x %x %x %x %x %d %d %x %s", - &sfileno, /* swap_file_number */ - &scan1, /* timestamp */ - &scan2, /* lastref */ - &scan3, /* expires */ - &scan4, /* last modified */ - &scan5, /* size */ - &scan6, /* refcount */ - &scan7, /* flags */ - keytext); /* key */ - if (x < 1) { - RB->invalid++; - continue; - } - if (x != 9) { - RB->invalid++; - continue; - } - timestamp = (time_t) scan1; - lastref = (time_t) scan2; - expires = (time_t) scan3; - lastmod = (time_t) scan4; - size = (off_t) scan5; - if (size < 0) { - if ((key = storeKeyScan(keytext)) == NULL) - continue; - if ((e = storeGet(key)) == NULL) - continue; - if (e->lastref > lastref) - continue; - debug(20, 3) ("storeRebuildFromDisk: Cancelling: '%s'\n", keytext); - storeRelease(e); - RB->objcount--; - RB->cancelcount++; - continue; - } - storeSwapFullPath(sfileno, swapfile); - if (EBIT_TEST(scan7, KEY_PRIVATE)) { - RB->badflags++; - continue; - } - sfileno = storeDirProperFileno(d->dirn, sfileno); - key = storeKeyScan(keytext); - if (key == NULL) { - debug(20, 1) ("storeDoConvertFromLog: bad key: '%s'\n", keytext); - continue; - } - e = storeGet(key); - used = storeDirMapBitTest(sfileno); - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - newer = e ? (lastref > e->lastref ? 1 : 0) : 0; - if (used && !newer) { - /* log entry is old, ignore it */ - RB->clashcount++; - continue; - } else if (used && e && e->swap_file_number == sfileno) { - /* swapfile taken, same URL, newer, update meta */ - e->lastref = timestamp; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->flag |= (u_num32) scan6; - e->refcount += (u_num32) scan7; - continue; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", - sfileno); - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - /* assert(RB->need_to_validate); */ - RB->clashcount++; - continue; - } else if (e) { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - storeRelease(e); /* release old entry */ - RB->dupcount++; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - /* update store_swap_size */ - RB->objcount++; - storeConvertFile(key, - sfileno, - (int) size, - expires, - timestamp, - lastref, - lastmod, - (u_short) scan6, /* refcount */ - (u_short) scan7, /* flags */ - d->clean); - storeDirSwapLog(e); - } - RB->rebuild_dir = d->next; - for (D = &RB->rebuild_dir; *D; D = &(*D)->next); - *D = d; - d->next = NULL; - eventAdd("storeRebuild", storeDoConvertFromLog, RB, 0); -} - void storeCleanup(void *datanotused) { @@ -639,48 +569,61 @@ storeValidateComplete(void *data, int retcode, int errcode) /* meta data recreated from disk image in swap directory */ static void -storeRebuiltFromDisk(struct storeRebuildState *data) +storeRebuildComplete(void) { time_t r; time_t stop; stop = squid_curtime; - r = stop - data->start; - debug(20, 1) ("Finished rebuilding storage from disk image.\n"); - debug(20, 1) (" %7d Lines read from previous logfile.\n", data->linecount); - debug(20, 1) (" %7d Invalid lines.\n", data->invalid); - debug(20, 1) (" %7d With invalid flags.\n", data->badflags); - debug(20, 1) (" %7d Objects loaded.\n", data->objcount); - debug(20, 1) (" %7d Objects expired.\n", data->expcount); - debug(20, 1) (" %7d Objects cancelled.\n", data->cancelcount); - debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); - debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); + r = stop - RebuildState.start; + debug(20, 1) ("Finished rebuilding storage disk.\n"); + debug(20, 1) (" %7d Entries read from previous logfile.\n", + RebuildState.linecount); + debug(20, 1) (" %7d Invalid entries.\n", RebuildState.invalid); + debug(20, 1) (" %7d With invalid flags.\n", RebuildState.badflags); + debug(20, 1) (" %7d Objects loaded.\n", RebuildState.objcount); + debug(20, 1) (" %7d Objects expired.\n", RebuildState.expcount); + debug(20, 1) (" %7d Objects cancelled.\n", RebuildState.cancelcount); + debug(20, 1) (" %7d Duplicate URLs purged.\n", RebuildState.dupcount); + debug(20, 1) (" %7d Swapfile clashes avoided.\n", RebuildState.clashcount); debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", - r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); + r > 0 ? r : 0, (double) RebuildState.objcount / (r > 0 ? r : 1)); debug(20, 1) ("Beginning Validation Procedure\n"); eventAdd("storeCleanup", storeCleanup, NULL, 0); - memFree(MEM_4K_BUF, data->line_in); - safe_free(data); } void -storeStartRebuildFromDisk(void) +storeRebuildStart(void) { - struct storeRebuildState *RB; - struct _rebuild_dir *d; - int clean = 1; - RB = xcalloc(1, sizeof(struct storeRebuildState)); - RB->start = squid_curtime; - d = xcalloc(1, sizeof(struct _rebuild_dir)); - d->clean = clean; - d->speed = opt_foreground_rebuild ? 1 << 30 : 50; - RB->rebuild_dir = d; - if (!clean) - RB->need_to_validate = 1; - debug(20, 1) ("Rebuilding storage (%s)\n", - clean ? "CLEAN" : "DIRTY"); - if (opt_foreground_rebuild) { - storeRebuildRawFile(RB); - } else { - eventAdd("storeRebuild", storeRebuildRawFile, RB, 0); + rebuild_dir *d; + int clean = 0; + FILE *fp; + int i; + memset(&RebuildState, '\0', sizeof(RebuildState)); + RebuildState.start = squid_curtime; + for (i = 0; i < Config.cacheSwap.n_configured; i++) { + d = xcalloc(1, sizeof(rebuild_dir)); + d->dirn = i; + d->speed = opt_foreground_rebuild ? 1 << 30 : 50; + /* + * If the swap.state file exists in the cache_dir, then + * we'll use storeRebuildFromSwapLog(), otherwise we'll + * use storeRebuildFromDirectory() to open up each file + * and suck in the meta data. + */ + fp = storeDirOpenTmpSwapLog(i, &clean); + if (fp == NULL) { + d->rebuild_func = storeRebuildFromDirectory; + } else { + d->rebuild_func = storeRebuildFromSwapLog; + d->log = fp; + d->clean = clean; + d->next = RebuildState.rebuild_dir; + } + RebuildState.rebuild_dir = d; + if (!clean) + RebuildState.need_to_validate = 1; + debug(20, 1) ("Rebuilding storage (%s)\n", + clean ? "CLEAN" : "DIRTY"); } + eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0); } diff --git a/src/store_swapmeta.cc b/src/store_swapmeta.cc index b9ae995256..39fe0fb3cf 100644 --- a/src/store_swapmeta.cc +++ b/src/store_swapmeta.cc @@ -37,9 +37,9 @@ storeSwapMetaBuild(StoreEntry * e) assert(e->swap_status == SWAPOUT_WRITING); url = storeUrl(e); debug(20, 3) ("storeSwapMetaBuild: %s\n", url); - T = storeSwapTLVAdd(STORE_META_KEY, e->key, cacheKeySize, T); + T = storeSwapTLVAdd(STORE_META_KEY, e->key, MD5_DIGEST_CHARS, T); T = storeSwapTLVAdd(STORE_META_STD, &e->timestamp, STORE_HDR_METASIZE, T); - T = storeSwapTLVAdd(STORE_META_URL, url, strlen(url)+1, T); + T = storeSwapTLVAdd(STORE_META_URL, url, strlen(url) + 1, T); return TLV; } diff --git a/src/store_swapout.cc b/src/store_swapout.cc index a34419c619..ea04cdf340 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -86,7 +86,7 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); - storeDirSwapLog(e); + storeDirSwapLog(e, SWAP_LOG_ADD); } /* Note, we don't otherwise call storeReleaseRequest() here because * storeCheckCachable() does it for is if necessary */ @@ -173,9 +173,9 @@ storeCheckSwapOut(StoreEntry * e) swapout_size = STORE_SWAP_BUF; swap_buf = memAllocate(MEM_DISK_BUF, 1); swap_buf_len = stmemCopy(mem->data, - mem->swapout.queue_offset, - swap_buf, - swapout_size); + mem->swapout.queue_offset, + swap_buf, + swapout_size); if (swap_buf_len < 0) { debug(20, 1) ("stmemCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); /* XXX This is probably wrong--we should storeRelease()? */ diff --git a/src/structs.h b/src/structs.h index 28452cba0d..403a3ff19f 100644 --- a/src/structs.h +++ b/src/structs.h @@ -951,8 +951,21 @@ struct _StatCounters { }; struct _tlv { - char type; - int length; - void *value; - struct _tlv *next; + char type; + int length; + void *value; + struct _tlv *next; +}; + +struct _storeSwapData { + char op; + int swap_file_number; + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + int object_len; + u_short refcount; + u_short flags; + char key[MD5_DIGEST_CHARS]; }; diff --git a/src/typedefs.h b/src/typedefs.h index 48662c0bc3..5c154ce5a6 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -77,6 +77,7 @@ typedef struct _dlink_node dlink_node; typedef struct _dlink_list dlink_list; typedef struct _StatCounters StatCounters; typedef struct _tlv tlv; +typedef struct _storeSwapData storeSwapData; /* define AIOCB even without USE_ASYNC_IO */ typedef void AIOCB(void *, int aio_return, int aio_errno);