From: wessels <> Date: Thu, 12 Feb 1998 14:03:04 +0000 (+0000) Subject: - Changed StoreEntry->object_len to ->swap_file_sz so we X-Git-Tag: SQUID_3_0_PRE1~4110 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=07304bf9487f9f0f58d89d1769048a3ee4b4c66a;p=thirdparty%2Fsquid.git - Changed StoreEntry->object_len to ->swap_file_sz so we can verify the disk file size at restart. Moved object_len to MemObject->object_sz. Note object_sz is initialized to -1. If object_sz < 0, then we need to open the swap file and read the swap metadata. - Changed store_client->mem to ->entry because we need e->swap_file_sz to set mem->object_sz at swapin. - Removed some superfluous checks for e->key != NULL in storeSet*Key(). - Renamed storeSwapData structure to storeSwapLogData. - Fixed storeGetNextFile to not increment d->dirn. Added check for opendir() failure. - Fixed storeRebuildStart to properly link the directory list for storeRebuildfromDirectory mode. --- diff --git a/src/client_side.cc b/src/client_side.cc index caa628a42a..44e8668993 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -1,6 +1,6 @@ /* - * $Id: client_side.cc,v 1.206 1998/02/10 21:44:30 wessels Exp $ + * $Id: client_side.cc,v 1.207 1998/02/12 07:03:04 wessels Exp $ * * DEBUG: section 33 Client-side Routines * AUTHOR: Duane Wessels @@ -420,7 +420,7 @@ modifiedSince(StoreEntry * entry, request_t * request) if (mem->reply->content_length >= 0) object_length = mem->reply->content_length; else - object_length = entry->object_len - mem->reply->hdr_sz; + object_length = contentLen(entry); if (entry->lastmod > request->ims) { debug(33, 3) ("--> YES: entry newer than client\n"); return 1; @@ -985,7 +985,7 @@ clientSendMoreData(void *data, char *buf, ssize_t size) if (entry->store_status == STORE_PENDING) http->out.offset = entry->mem_obj->inmem_hi; else - http->out.offset = entry->object_len; + http->out.offset = objectLen(entry); } } comm_write(fd, buf, writelen, clientWriteComplete, http, freefunc); @@ -1000,7 +1000,7 @@ clientWriteComplete(int fd, char *bufnotused, size_t size, int errflag, void *da int done; http->out.size += size; debug(33, 5) ("clientWriteComplete: FD %d, sz %d, err %d, off %d, len %d\n", - fd, size, errflag, http->out.offset, entry->object_len); + fd, size, errflag, http->out.offset, objectLen(entry)); if (errflag) { CheckQuickAbort(http); comm_close(fd); @@ -1915,10 +1915,10 @@ clientCheckTransferDone(clientHttpRequest * http) return 0; /* * Handle STORE_OK and STORE_ABORTED objects. - * entry->object_len will be set proprely. + * objectLen(entry) will be set proprely. */ if (entry->store_status != STORE_PENDING) { - if (http->out.offset >= entry->object_len) + if (http->out.offset >= objectLen(entry)) return 1; else return 0; diff --git a/src/protos.h b/src/protos.h index e1124d2ca0..e17963261c 100644 --- a/src/protos.h +++ b/src/protos.h @@ -448,6 +448,8 @@ extern void storeAppendPrintf(); extern int storeCheckCachable(StoreEntry * e); extern void storeUnlinkFileno(int fileno); extern void storeSetPrivateKey(StoreEntry *); +extern int objectLen(const StoreEntry * e); +extern int contentLen(const StoreEntry * e); /* * store_log.c @@ -518,28 +520,8 @@ void storeSwapTLVFree(tlv * n); * store_rebuild.c */ extern void storeDoRebuildFromSwapFiles(void *data); -extern void storeConvertFile(const cache_key * key, - int file_number, - int size, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_short refcount, - u_short flags, - int clean); -extern StoreEntry *storeAddDiskRestore(const cache_key * key, - int file_number, - int size, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - u_num32 refcount, - u_num32 flags, - int clean); extern void storeCleanup(void *datanotused); -extern void storeValidate(StoreEntry *, STVLDCB, void *callback_data, void *tag); +extern void storeValidate(StoreEntry *, STVLDCB *, void *, void *); extern void storeValidateComplete(void *data, int retcode, int errcode); extern void storeRebuildStart(void); diff --git a/src/store.cc b/src/store.cc index 2fa65114cb..d0b8d4d969 100644 --- a/src/store.cc +++ b/src/store.cc @@ -1,6 +1,6 @@ /* - * $Id: store.cc,v 1.378 1998/02/06 18:54:09 wessels Exp $ + * $Id: store.cc,v 1.379 1998/02/12 07:03:06 wessels Exp $ * * DEBUG: section 20 Storeage Manager * AUTHOR: Harvest Derived @@ -184,6 +184,7 @@ new_MemObject(const char *url, const char *log_url) mem->url = xstrdup(url); mem->log_url = xstrdup(log_url); mem->swapout.fd = -1; + mem->object_sz = -1; meta_data.misc += strlen(log_url); debug(20, 3) ("new_MemObject: returning %p\n", mem); return mem; @@ -357,10 +358,9 @@ storeSetPrivateKey(StoreEntry * e) { const cache_key *newkey; MemObject *mem = e->mem_obj; - if (e->key && EBIT_TEST(e->flag, KEY_PRIVATE)) + if (EBIT_TEST(e->flag, KEY_PRIVATE)) return; /* is already private */ - if (e->key) - storeHashDelete(e); + storeHashDelete(e); if (mem != NULL) { mem->reqnum = getKeyCounter(); newkey = storeKeyPrivate(mem->url, mem->method, mem->reqnum); @@ -378,7 +378,7 @@ storeSetPublicKey(StoreEntry * e) StoreEntry *e2 = NULL; const cache_key *newkey; MemObject *mem = e->mem_obj; - if (e->key && !EBIT_TEST(e->flag, KEY_PRIVATE)) + if (!EBIT_TEST(e->flag, KEY_PRIVATE)) return; /* is already public */ assert(mem); newkey = storeKeyPublic(mem->url, mem->method); @@ -388,8 +388,7 @@ storeSetPublicKey(StoreEntry * e) storeRelease(e2); newkey = storeKeyPublic(mem->url, mem->method); } - if (e->key) - storeHashDelete(e); + storeHashDelete(e); storeHashInsert(e, newkey); EBIT_CLR(e->flag, KEY_PRIVATE); } @@ -522,7 +521,7 @@ void storeComplete(StoreEntry * e) { debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key)); - e->object_len = e->mem_obj->inmem_hi; + e->mem_obj->object_sz = e->mem_obj->inmem_hi; e->store_status = STORE_OK; assert(e->mem_status == NOT_IN_MEMORY); if (!storeEntryValidLength(e)) @@ -551,7 +550,7 @@ storeAbort(StoreEntry * e, int cbflag) e->swap_status = SWAPOUT_NONE; /* We assign an object length here--The only other place we assign the * object length is in storeComplete() */ - e->object_len = mem->inmem_hi; + mem->object_sz = mem->inmem_hi; /* Notify the server side */ if (cbflag && mem->abort.callback) { mem->abort.callback(mem->abort.data); @@ -718,7 +717,6 @@ storeRelease(StoreEntry * e) storeExpireNow(e); storeSetPrivateKey(e); EBIT_SET(e->flag, RELEASE_REQUEST); - storeDirSwapLog(e, SWAP_LOG_DEL); return; } #endif @@ -726,7 +724,7 @@ storeRelease(StoreEntry * e) if (e->swap_file_number > -1) { storeUnlinkFileno(e->swap_file_number); if (e->swap_status == SWAPOUT_DONE) - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); + storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1); storeDirSwapLog(e, SWAP_LOG_DEL); } storeSetMemStatus(e, NOT_IN_MEMORY); @@ -754,23 +752,22 @@ static int storeEntryValidLength(const StoreEntry * e) { int diff; - int hdr_sz; - int content_length; + http_reply *reply; assert(e->mem_obj != NULL); - hdr_sz = e->mem_obj->reply->hdr_sz; - content_length = e->mem_obj->reply->content_length; - + reply = e->mem_obj->reply; debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key)); - debug(20, 5) ("storeEntryValidLength: object_len = %d\n", e->object_len); - debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", hdr_sz); - debug(20, 5) ("storeEntryValidLength: content_length = %d\n", content_length); - - if (content_length < 0) { + debug(20, 5) ("storeEntryValidLength: object_len = %d\n", + objectLen(e)); + debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", + reply->hdr_sz); + debug(20, 5) ("storeEntryValidLength: content_length = %d\n", + reply->content_length); + if (reply->content_length < 0) { debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", storeKeyText(e->key)); return 1; } - if (hdr_sz == 0) { + if (reply->hdr_sz == 0) { debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", storeKeyText(e->key)); return 1; @@ -780,11 +777,11 @@ storeEntryValidLength(const StoreEntry * e) storeKeyText(e->key)); return 1; } - if (e->mem_obj->reply->code == HTTP_NOT_MODIFIED) + if (reply->code == HTTP_NOT_MODIFIED) return 1; - if (e->mem_obj->reply->code == HTTP_NO_CONTENT) + if (reply->code == HTTP_NO_CONTENT) return 1; - diff = hdr_sz + content_length - e->object_len; + diff = reply->hdr_sz + reply->content_length - objectLen(e); if (diff == 0) return 1; debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", @@ -1038,7 +1035,7 @@ storeEntryDump(StoreEntry * e, int l) debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires); debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod); - debug(20, l) ("StoreEntry->object_len: %d\n", e->object_len); + debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz); debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount); debug(20, l) ("StoreEntry->flag: %X\n", e->flag); debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number); @@ -1120,9 +1117,25 @@ storeBufferFlush(StoreEntry * e) void storeUnlinkFileno(int fileno) { + debug(20, 5) ("storeUnlinkFileno: %08X\n", fileno); #if USE_ASYNC_IO safeunlink(storeSwapFullPath(fileno, NULL), 1); #else unlinkdUnlink(storeSwapFullPath(fileno, NULL)); #endif } + +int +objectLen(const StoreEntry * e) +{ + assert(e->mem_obj != NULL); + return e->mem_obj->object_sz; +} + +int +contentLen(const StoreEntry * e) +{ + assert(e->mem_obj != NULL); + assert(e->mem_obj->reply != NULL); + return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz; +} diff --git a/src/store_client.cc b/src/store_client.cc index 2295c8d972..b0842c60d4 100644 --- a/src/store_client.cc +++ b/src/store_client.cc @@ -54,7 +54,7 @@ storeClientListAdd(StoreEntry * e, void *data) sc->copy_offset = 0; sc->swapin_fd = -1; sc->disk_op_in_progress = 0; - sc->mem = mem; + sc->entry = e; if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) sc->type = STORE_MEM_CLIENT; else @@ -97,6 +97,29 @@ storeClientCopy(StoreEntry * e, recurse_detect--; } +/* + * This function is used below to decide if we have any more data to + * send to the client. If the store_status is STORE_ABORTED, that case + * should be handled before this function gets called. If the + * store_status is STORE_PENDING, then we do have more data to send. + * If its STORE_OK, then we continue checking. If the object length is + * negative, then we don't know the real length and must open the swap + * file to find out. If the lenght is >= 0, then we compare it to the + * requested copy offset. + */ +static int +storeClientNoMoreToSend(StoreEntry * e, store_client * sc) +{ + ssize_t len; + if (e->store_status != STORE_OK) + return 0; + if ((len = objectLen(e)) < 0) + return 0; + if (sc->copy_offset < len) + return 0; + return 1; +} + static void storeClientCopy2(StoreEntry * e, store_client * sc) { @@ -119,7 +142,7 @@ storeClientCopy2(StoreEntry * e, store_client * sc) sc->disk_op_in_progress = 0; sc->callback = NULL; callback(sc->callback_data, sc->copy_buf, 0); - } else if (e->store_status == STORE_OK && sc->copy_offset == e->object_len) { + } else if (storeClientNoMoreToSend(e, sc)) { /* There is no more to send! */ #if USE_ASYNC_IO if (sc->disk_op_in_progress == 1) { @@ -194,7 +217,7 @@ storeClientFileOpened(int fd, void *data) static void storeClientFileRead(store_client * sc) { - MemObject *mem = sc->mem; + MemObject *mem = sc->entry->mem_obj; assert(sc->callback != NULL); if (mem->swap_hdr_sz == 0) file_read(sc->swapin_fd, @@ -216,7 +239,7 @@ static void storeClientReadBody(int fd, const char *buf, int len, int flagnotused, void *data) { store_client *sc = data; - MemObject *mem = sc->mem; + MemObject *mem = sc->entry->mem_obj; STCB *callback = sc->callback; assert(sc->disk_op_in_progress != 0); sc->disk_op_in_progress = 0; @@ -235,7 +258,8 @@ storeClientReadHeader(int fd, const char *buf, int len, int flagnotused, void *d * 'buf' should have been allocated by memAllocate(MEM_DISK_BUF) */ store_client *sc = data; - MemObject *mem = sc->mem; + StoreEntry *e = sc->entry; + MemObject *mem = e->mem_obj; STCB *callback = sc->callback; int swap_hdr_sz = 0; size_t body_sz; @@ -264,7 +288,9 @@ storeClientReadHeader(int fd, const char *buf, int len, int flagnotused, void *d * XXX Here we should check the meta data and make sure we got * the right object. */ + storeSwapTLVFree(tlv_list); mem->swap_hdr_sz = swap_hdr_sz; + mem->object_sz = e->swap_file_sz - swap_hdr_sz; /* * If our last read got some data the client wants, then give * it to them, otherwise schedule another read. diff --git a/src/store_dir.cc b/src/store_dir.cc index 492f94dcb5..b8d9e8d3ae 100644 --- a/src/store_dir.cc +++ b/src/store_dir.cc @@ -1,6 +1,6 @@ /* - * $Id: store_dir.cc,v 1.53 1998/02/11 18:48:52 wessels Exp $ + * $Id: store_dir.cc,v 1.54 1998/02/12 07:03:07 wessels Exp $ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Duane Wessels @@ -301,7 +301,7 @@ storeDirProperFileno(int dirn, int fn) void storeDirSwapLog(const StoreEntry * e, int op) { - storeSwapData *s = xcalloc(1, sizeof(storeSwapData)); + storeSwapLogData *s = xcalloc(1, sizeof(storeSwapLogData)); int dirn; dirn = e->swap_file_number >> SWAP_DIR_SHIFT; assert(dirn < Config.cacheSwap.n_configured); @@ -318,20 +318,24 @@ storeDirSwapLog(const StoreEntry * e, int op) */ if (EBIT_TEST(e->flag, ENTRY_SPECIAL)) return; + debug(20, 3) ("storeDirSwapLog: %s %s %08X\n", + op == SWAP_LOG_DEL ? "SWAP_LOG_DEL" : "SWAP_LOG_ADD", + storeKeyText(e->key), + e->swap_file_number); s->op = (char) op; s->swap_file_number = e->swap_file_number; s->timestamp = e->timestamp; s->lastref = e->lastref; s->expires = e->expires; s->lastmod = e->lastmod; - s->object_len = e->object_len; + s->swap_file_sz = e->swap_file_sz; s->refcount = e->refcount; s->flags = e->flag; xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd, -1, s, - sizeof(storeSwapData), + sizeof(storeSwapLogData), NULL, NULL, xfree); @@ -521,7 +525,7 @@ storeDirWriteCleanLogs(int reopen) dlink_node *m; char **outbuf; off_t *outbufoffset; - storeSwapData *s; + storeSwapLogData *s; if (store_rebuilding) { debug(20, 1) ("Not currently OK to rewrite swap log.\n"); debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n"); @@ -569,7 +573,7 @@ storeDirWriteCleanLogs(int reopen) continue; if (e->swap_status != SWAPOUT_DONE) continue; - if (e->object_len <= 0) + if (e->swap_file_sz <= 0) continue; if (EBIT_TEST(e->flag, RELEASE_REQUEST)) continue; @@ -581,21 +585,21 @@ storeDirWriteCleanLogs(int reopen) assert(dirn < Config.cacheSwap.n_configured); if (fd[dirn] < 0) continue; - s = (storeSwapData *) (outbuf[dirn] + outbufoffset[dirn]); - outbufoffset[dirn] += sizeof(storeSwapData); - memset(s, '\0', sizeof(storeSwapData)); + s = (storeSwapLogData *) (outbuf[dirn] + outbufoffset[dirn]); + outbufoffset[dirn] += sizeof(storeSwapLogData); + memset(s, '\0', sizeof(storeSwapLogData)); s->op = (char) SWAP_LOG_ADD; s->swap_file_number = e->swap_file_number; s->timestamp = e->timestamp; s->lastref = e->lastref; s->expires = e->expires; s->lastmod = e->lastmod; - s->object_len = e->object_len; + s->swap_file_sz = e->swap_file_sz; s->refcount = e->refcount; s->flags = e->flag; xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); /* buffered write */ - if (outbufoffset[dirn] + sizeof(storeSwapData) > CLEAN_BUF_SZ) { + if (outbufoffset[dirn] + sizeof(storeSwapLogData) > CLEAN_BUF_SZ) { if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) { debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", new[dirn], xstrerror()); diff --git a/src/store_rebuild.cc b/src/store_rebuild.cc index 5a8781a342..d092e102a3 100644 --- a/src/store_rebuild.cc +++ b/src/store_rebuild.cc @@ -37,7 +37,7 @@ struct storeRebuildState { int badflags; /* # bad e->flags */ int need_to_validate; int bad_log_op; - int zero_object_len; + int zero_object_sz; time_t start; time_t stop; } RebuildState; @@ -54,6 +54,16 @@ static RBHD storeRebuildFromSwapLog; static void storeRebuildComplete(void); static EVH storeRebuildADirectory; static int storeGetNextFile(rebuild_dir *, int *sfileno, int *size); +static StoreEntry *storeAddDiskRestore(const cache_key * key, + int file_number, + size_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + u_num32 refcount, + u_num32 flags, + int clean); static int storeRebuildFromDirectory(rebuild_dir * d) @@ -66,11 +76,10 @@ storeRebuildFromDirectory(rebuild_dir * d) int count; int size; struct stat fst; - int hdr_len; + int swap_hdr_len; int fd = -1; tlv *tlv_list; tlv *t; - double x; assert(d != NULL); debug(20, 3) ("storeRebuildFromDirectory: DIR #%d\n", d->dirn); for (count = 0; count < d->speed; count++) { @@ -106,8 +115,8 @@ storeRebuildFromDirectory(rebuild_dir * d) } file_close(fd); fd = -1; - hdr_len = 0; - tlv_list = storeSwapMetaUnpack(hdr_buf, &hdr_len); + swap_hdr_len = 0; + tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len); if (tlv_list == NULL) { debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n"); storeUnlinkFileno(sfileno); @@ -138,19 +147,23 @@ storeRebuildFromDirectory(rebuild_dir * d) continue; } tmpe.key = key; - if (tmpe.object_len == 0) { +#if OLD_CODE + if (tmpe.swap_file_sz == 0) { RebuildState.invalid++; - x = log(++RebuildState.zero_object_len) / log(10.0); + x = log(++RebuildState.zero_object_sz) / log(10.0); if (0.0 == x - (double) (int) x) - debug(20, 1) ("WARNING: %d swapfiles found with ZERO length\n", - RebuildState.zero_object_len); + debug(20, 1) ("WARNING: %d swapfiles found with ZERO size\n", + RebuildState.zero_object_sz); storeUnlinkFileno(sfileno); continue; } +#endif /* check sizes */ - if (hdr_len + tmpe.object_len != fst.st_size) { - debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d+%d!=%d\n", - hdr_len, tmpe.object_len, fst.st_size); + if (tmpe.swap_file_sz == 0) { + tmpe.swap_file_sz = fst.st_size; + } else if (tmpe.swap_file_sz != fst.st_size) { + debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d!=%d\n", + tmpe.swap_file_sz, fst.st_size); storeUnlinkFileno(sfileno); continue; } @@ -169,7 +182,7 @@ storeRebuildFromDirectory(rebuild_dir * d) storeEntryDump(&tmpe, 5); e = storeAddDiskRestore(key, sfileno, - (int) tmpe.object_len, + tmpe.swap_file_sz, tmpe.expires, tmpe.timestamp, tmpe.lastref, @@ -185,8 +198,8 @@ static int storeRebuildFromSwapLog(rebuild_dir * d) { StoreEntry *e = NULL; - storeSwapData s; - size_t ss = sizeof(storeSwapData); + storeSwapLogData s; + size_t ss = sizeof(storeSwapLogData); int count; int used; /* is swapfile already in use? */ int newer; /* is the log entry newer than current entry? */ @@ -214,6 +227,7 @@ storeRebuildFromSwapLog(rebuild_dir * d) * because adding to store_swap_size happens in * the cleanup procedure. */ + storeExpireNow(e); storeSetPrivateKey(e); EBIT_SET(e->flag, RELEASE_REQUEST); @@ -297,7 +311,7 @@ storeRebuildFromSwapLog(rebuild_dir * d) RebuildState.objcount++; e = storeAddDiskRestore(s.key, s.swap_file_number, - s.object_len, + s.swap_file_sz, s.expires, s.timestamp, s.lastref, @@ -335,10 +349,10 @@ storeRebuildADirectory(void *unused) eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0); } -void +static void storeConvertFile(const cache_key * key, int file_number, - int size, + size_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -355,7 +369,7 @@ storeConvertFile(const cache_key * key, tlv *tlv_list; StoreEntry e; e.key = key; - e.object_len = size; + e.swap_file_sz = swap_file_sz; e.expires = expires; e.lastref = lastref; e.refcount = refcount; @@ -395,28 +409,33 @@ storeGetNextFile(rebuild_dir * d, int *sfileno, int *size) return -2; while (fd < 0 && d->done == 0) { fd = -1; - if (!d->flag) { /* initialize, open first file */ - d->done = d->dirn = d->curlvl1 = d->curlvl2 = d->in_dir = 0; + if (0 == d->flag) { /* initialize, open first file */ + d->done = 0; + d->curlvl1 = 0; + d->curlvl2 = 0; + d->in_dir = 0; d->flag = 1; assert(Config.cacheSwap.n_configured > 0); } - if (!d->in_dir) { /* we need to read in a new directory */ + if (0 == d->in_dir) { /* we need to read in a new directory */ snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", Config.cacheSwap.swapDirs[d->dirn].path, d->curlvl1, d->curlvl2); - if (d->flag && d->td) + if (d->flag && d->td != NULL) closedir(d->td); d->td = opendir(fullpath); + if (d->td == NULL) { + debug(50, 1) ("storeGetNextFile: opendir: %s: %s\n", + fullpath, xstrerror()); + break; + } d->entry = readdir(d->td); /* skip . and .. */ d->entry = readdir(d->td); - if (errno == ENOENT) { - debug(20, 3) ("storeGetNextFile: directory does not exist!.\n"); - } - debug(20, 3) ("storeGetNextFile: Directory %s/%02X/%02X\n", - Config.cacheSwap.swapDirs[d->dirn].path, - d->curlvl1, d->curlvl2); + if (errno == ENOENT) + debug(20, 1) ("storeGetNextFile: directory does not exist!.\n"); + debug(20, 3) ("storeGetNextFile: Directory %s\n", fullpath); } - if ((d->entry = readdir(d->td))) { + if (d->td != NULL && (d->entry = readdir(d->td)) != NULL) { d->in_dir++; if (sscanf(d->entry->d_name, "%x", sfileno) != 1) { debug(20, 3) ("storeGetNextFile: invalid %s\n", @@ -424,6 +443,8 @@ storeGetNextFile(rebuild_dir * d, int *sfileno, int *size) continue; } d->fn = *sfileno; + if (!storeFilenoBelongsHere(d->fn, d->dirn, d->curlvl1, d->curlvl2)) + continue; d->fn = storeDirProperFileno(d->dirn, d->fn); *sfileno = d->fn; used = storeDirMapBitTest(d->fn); @@ -438,27 +459,23 @@ storeGetNextFile(rebuild_dir * d, int *sfileno, int *size) continue; } d->in_dir = 0; - d->curlvl2 = (d->curlvl2 + 1) % Config.cacheSwap.swapDirs[d->dirn].l2; - if (d->curlvl2 != 0) - continue; - d->curlvl1 = (d->curlvl1 + 1) % Config.cacheSwap.swapDirs[d->dirn].l1; - if (d->curlvl1 != 0) + if (++d->curlvl2 < Config.cacheSwap.swapDirs[d->dirn].l2) continue; - d->dirn = (d->dirn + 1) % Config.cacheSwap.n_configured; - if (d->dirn != 0) + d->curlvl2 = 0; + if (++d->curlvl1 < Config.cacheSwap.swapDirs[d->dirn].l1) continue; - else - d->done = 1; + d->curlvl1 = 0; + d->done = 1; } return fd; } /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. */ -StoreEntry * +static StoreEntry * storeAddDiskRestore(const cache_key * key, int file_number, - int size, + size_t swap_file_sz, time_t expires, time_t timestamp, time_t lastref, @@ -477,7 +494,7 @@ storeAddDiskRestore(const cache_key * key, storeSetMemStatus(e, NOT_IN_MEMORY); e->swap_status = SWAPOUT_DONE; e->swap_file_number = file_number; - e->object_len = size; + e->swap_file_sz = swap_file_sz; e->lock_count = 0; e->refcount = 0; e->lastref = lastref; @@ -516,10 +533,23 @@ storeCleanup(void *datanotused) continue; if (e->swap_file_number < 0) continue; +#if STORE_DOUBLECHECK + { + struct stat sb; + if (stat(storeSwapFullPath(e->swap_file_number, NULL), &sb) < 0) { + debug(0, 0) ("storeCleanup: MISSING SWAP FILE\n"); + debug(0, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number); + debug(0, 0) ("storeCleanup: PATH %s\n", + storeSwapFullPath(e->swap_file_number, NULL)); + storeEntryDump(e, 0); + assert(0); + } + } +#endif EBIT_SET(e->flag, ENTRY_VALIDATED); /* Only set the file bit if we know its a valid entry */ /* otherwise, set it in the validation procedure */ - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1); if ((++validnum & 0xFFFF) == 0) debug(20, 1) (" %7d Entries Validated so far.\n", validnum); assert(validnum <= memInUse(MEM_STOREENTRY)); @@ -542,7 +572,7 @@ storeCleanupComplete(void *data, int retcode, int errcode) #endif void -storeValidate(StoreEntry * e, STVLDCB callback, void *callback_data, void *tag) +storeValidate(StoreEntry * e, STVLDCB * callback, void *callback_data, void *tag) { valid_ctrl_t *ctrlp; char *path; @@ -595,11 +625,11 @@ storeValidateComplete(void *data, int retcode, int errcode) path = storeSwapFullPath(e->swap_file_number, NULL); retcode = stat(path, sb); } - if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { + if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->swap_file_sz) { EBIT_CLR(e->flag, ENTRY_VALIDATED); } else { EBIT_SET(e->flag, ENTRY_VALIDATED); - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1); } errno = errcode; ctrlp->callback(ctrlp->callback_data, retcode, errcode); @@ -660,8 +690,8 @@ storeRebuildStart(void) d->rebuild_func = storeRebuildFromSwapLog; d->log = fp; d->clean = clean; - d->next = RebuildState.rebuild_dir; } + d->next = RebuildState.rebuild_dir; RebuildState.rebuild_dir = d; if (!clean) RebuildState.need_to_validate = 1; diff --git a/src/store_swapin.cc b/src/store_swapin.cc index d73ce723d2..783ac7b7b9 100644 --- a/src/store_swapin.cc +++ b/src/store_swapin.cc @@ -71,6 +71,7 @@ storeSwapInFileOpened(void *data, int fd, int errcode) swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; StoreEntry *e = ctrlp->e; MemObject *mem = e->mem_obj; + struct stat sb; if (fd == -2 && errcode == -2) { xfree(ctrlp->path); xfree(ctrlp); @@ -88,20 +89,13 @@ storeSwapInFileOpened(void *data, int fd, int errcode) xfree(ctrlp); return; } - /* - * We can't use fstat() to check file size here because of the - * metadata header. We have to parse the header first and find - * the header size. - */ -#if OLD_CODE if (e->swap_status == SWAPOUT_DONE && fstat(fd, &sb) == 0) { - if (sb.st_size == 0 || sb.st_size != e->object_len) { - debug(20, 0) ("storeSwapInFileOpened: %s: Size mismatch: %d(fstat) != %d(object)\n", ctrlp->path, sb.st_size, e->object_len); + if (sb.st_size == 0 || sb.st_size != e->swap_file_sz) { + debug(20, 0) ("storeSwapInFileOpened: %s: Size mismatch: %d(fstat) != %d(object)\n", ctrlp->path, sb.st_size, e->swap_file_sz); file_close(fd); fd = -1; } } -#endif debug(20, 5) ("storeSwapInFileOpened: initialized '%s' for '%s'\n", ctrlp->path, storeUrl(e)); (ctrlp->callback) (fd, ctrlp->callback_data); diff --git a/src/store_swapout.cc b/src/store_swapout.cc index 9122140c00..d7ff632dae 100644 --- a/src/store_swapout.cc +++ b/src/store_swapout.cc @@ -67,15 +67,16 @@ storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) if (e->store_status == STORE_PENDING) { storeCheckSwapOut(e); return; - } else if (mem->swapout.done_offset < e->object_len + mem->swap_hdr_sz) { + } else if (mem->swapout.done_offset < objectLen(e) + mem->swap_hdr_sz) { storeCheckSwapOut(e); return; } /* swapping complete */ debug(20, 5) ("storeSwapOutHandle: SwapOut complete: '%s' to %s.\n", storeUrl(e), storeSwapFullPath(e->swap_file_number, NULL)); + e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz; e->swap_status = SWAPOUT_DONE; - storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); + storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1); if (storeCheckCachable(e)) { storeLog(STORE_LOG_SWAPOUT, e); storeDirSwapLog(e, SWAP_LOG_ADD); diff --git a/src/structs.h b/src/structs.h index f453d235a9..c47ceb5561 100644 --- a/src/structs.h +++ b/src/structs.h @@ -435,7 +435,7 @@ struct _http_reply { double version; int code; int content_length; - int hdr_sz; + size_t hdr_sz; int cache_control; int misc_headers; time_t date; @@ -785,7 +785,7 @@ struct _store_client { char *copy_buf; STCB *callback; void *callback_data; - MemObject *mem; /* ptr to the parent structure, argh! */ + StoreEntry *entry; /* ptr to the parent StoreEntry, argh! */ int swapin_fd; int disk_op_in_progress; struct _store_client *next; @@ -819,6 +819,7 @@ struct _MemObject { char *log_url; dlink_node lru; u_num32 reqnum; + ssize_t object_sz; size_t swap_hdr_sz; }; @@ -831,7 +832,7 @@ struct _StoreEntry { time_t lastref; time_t expires; time_t lastmod; - int object_len; + size_t swap_file_sz; u_short refcount; u_short flag; @@ -958,14 +959,14 @@ struct _tlv { struct _tlv *next; }; -struct _storeSwapData { +struct _storeSwapLogData { char op; int swap_file_number; time_t timestamp; time_t lastref; time_t expires; time_t lastmod; - int object_len; + size_t swap_file_sz; u_short refcount; u_short flags; unsigned char key[MD5_DIGEST_CHARS]; diff --git a/src/typedefs.h b/src/typedefs.h index 50e9f09d0a..dc5e04672a 100644 --- a/src/typedefs.h +++ b/src/typedefs.h @@ -77,7 +77,7 @@ typedef struct _dlink_node dlink_node; typedef struct _dlink_list dlink_list; typedef struct _StatCounters StatCounters; typedef struct _tlv tlv; -typedef struct _storeSwapData storeSwapData; +typedef struct _storeSwapLogData storeSwapLogData; typedef struct _cacheSwap cacheSwap; /* define AIOCB even without USE_ASYNC_IO */