/*
- * $Id: store_rebuild.cc,v 1.52 1998/10/09 17:46:37 wessels Exp $
+ * $Id$
*
* DEBUG: section 20 Store Rebuild Routines
* AUTHOR: Duane Wessels
*
- * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
+ * SQUID Web Proxy Cache http://www.squid-cache.org/
* ----------------------------------------------------------
*
- * Squid is the result of efforts by numerous individuals from the
- * Internet community. Development is led by Duane Wessels of the
- * National Laboratory for Applied Network Research and funded by the
- * National Science Foundation. Squid is Copyrighted (C) 1998 by
- * Duane Wessels and the University of California San Diego. Please
- * see the COPYRIGHT file for full details. Squid incorporates
- * software developed and/or copyrighted by other sources. Please see
- * the CREDITS file for full details.
+ * Squid is the result of efforts by numerous individuals from
+ * the Internet community; see the CONTRIBUTORS file for full
+ * details. Many organizations have provided support for Squid's
+ * development; see the SPONSORS file for full details. Squid is
+ * Copyrighted (C) 2001 by the Regents of the University of
+ * California; see the COPYRIGHT file for full details. Squid
+ * incorporates software developed and/or copyrighted by other
+ * sources; see the CREDITS file for full details.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
- *
+ *
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
- *
+ *
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
*/
#include "squid.h"
+#include "event.h"
+#include "globals.h"
+#include "md5.h"
+#include "protos.h"
+#include "StatCounters.h"
+#include "Store.h"
+#include "SwapDir.h"
+#include "StoreSearch.h"
+#include "SquidTime.h"
-#define STORE_META_BUFSZ 4096
-
-
-typedef struct _rebuild_dir rebuild_dir;
-typedef int RBHD(rebuild_dir * d);
-
-struct _rebuild_dir {
- int dirn;
- int n_read;
- FILE *log;
- int speed;
- int clean;
- int curlvl1;
- int curlvl2;
- int flag;
- int done;
- int in_dir;
- int fn;
- struct dirent *entry;
- DIR *td;
- RBHD *rebuild_func;
- rebuild_dir *next;
- char fullpath[SQUID_MAXPATHLEN];
- char fullfilename[SQUID_MAXPATHLEN];
-};
+static struct _store_rebuild_data counts;
+
+static struct timeval rebuild_start;
+static void storeCleanup(void *);
-struct storeRebuildState {
- rebuild_dir *rebuild_dir;
- int objcount; /* # objects successfully reloaded */
- int expcount; /* # objects expired */
- int linecount; /* # lines parsed from cache logfile */
- int statcount; /* # entries from directory walking */
- int clashcount; /* # swapfile clashes avoided */
- int dupcount; /* # duplicates purged */
- int cancelcount; /* # SWAP_LOG_DEL objects purged */
- int invalid; /* # bad lines */
- int badflags; /* # bad e->flags */
- int need_to_validate;
- int bad_log_op;
- int zero_object_sz;
- time_t start;
- time_t stop;
-} RebuildState;
-
-typedef struct valid_ctrl_t {
- struct stat *sb;
- StoreEntry *e;
- STVLDCB *callback;
- void *callback_data;
-} valid_ctrl_t;
-
-static RBHD storeRebuildFromDirectory;
-static RBHD storeRebuildFromSwapLog;
-static void storeRebuildComplete(void);
-static EVH storeRebuildADirectory;
-static int storeGetNextFile(rebuild_dir *, int *sfileno, int *size);
-static StoreEntry *storeAddDiskRestore(const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_num32 refcount,
- u_short flags,
- int clean);
-static AIOCB storeValidateComplete;
+typedef struct {
+ /* total number of "swap.state" entries that will be read */
+ int total;
+ /* number of entries read so far */
+ int scanned;
+} store_rebuild_progress;
+
+static store_rebuild_progress *RebuildProgress = NULL;
static int
-storeRebuildFromDirectory(rebuild_dir * d)
+storeCleanupDoubleCheck(StoreEntry * e)
{
- LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE);
- StoreEntry *e = NULL;
- StoreEntry tmpe;
- cache_key key[MD5_DIGEST_CHARS];
- int sfileno = 0;
- int count;
- int size;
- struct stat sb;
- int swap_hdr_len;
- int fd = -1;
- tlv *tlv_list;
- tlv *t;
- assert(d != NULL);
- debug(20, 3) ("storeRebuildFromDirectory: DIR #%d\n", d->dirn);
- for (count = 0; count < d->speed; count++) {
- assert(fd == -1);
- fd = storeGetNextFile(d, &sfileno, &size);
- if (fd == -2) {
- debug(20, 1) ("storeRebuildFromDirectory: DIR #%d done!\n", d->dirn);
- storeDirCloseTmpSwapLog(d->dirn);
- store_rebuilding = 0;
- return -1;
- } else if (fd < 0) {
- continue;
- }
- assert(fd > -1);
- /* lets get file stats here */
- if (fstat(fd, &sb) < 0) {
- debug(20, 1) ("storeRebuildFromDirectory: fstat(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- fd = -1;
- continue;
- }
- if ((++RebuildState.statcount & 0xFFFF) == 0)
- debug(20, 1) (" %7d files opened so far.\n",
- RebuildState.statcount);
- debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
- Counter.syscalls.disk.reads++;
- if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) {
- debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n",
- fd, xstrerror());
- file_close(fd);
- fd = -1;
- continue;
- }
- file_close(fd);
- fd = -1;
- swap_hdr_len = 0;
- tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
- if (tlv_list == NULL) {
- debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n");
- storeUnlinkFileno(sfileno);
- continue;
- }
- debug(20, 3) ("storeRebuildFromDirectory: successful swap meta unpacking\n");
- memset(key, '\0', MD5_DIGEST_CHARS);
- memset(&tmpe, '\0', sizeof(StoreEntry));
- for (t = tlv_list; t; t = t->next) {
- switch (t->type) {
- case STORE_META_KEY:
- assert(t->length == MD5_DIGEST_CHARS);
- xmemcpy(key, t->value, MD5_DIGEST_CHARS);
- break;
- case STORE_META_STD:
- assert(t->length == STORE_HDR_METASIZE);
- xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
- break;
- default:
- break;
- }
- }
- storeSwapTLVFree(tlv_list);
- tlv_list = NULL;
- if (storeKeyNull(key)) {
- debug(20, 1) ("storeRebuildFromDirectory: NULL key\n");
- storeUnlinkFileno(sfileno);
- continue;
- }
- tmpe.key = key;
- /* check sizes */
- if (tmpe.swap_file_sz == 0) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
- tmpe.swap_file_sz = sb.st_size;
- } else if (tmpe.swap_file_sz != sb.st_size) {
- debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
- tmpe.swap_file_sz, (int) sb.st_size);
- storeUnlinkFileno(sfileno);
- continue;
- }
- if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- storeUnlinkFileno(sfileno);
- RebuildState.badflags++;
- continue;
- }
- e = storeGet(key);
- if (e && e->lastref >= tmpe.lastref) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- RebuildState.dupcount++;
- continue;
- } else if (NULL != e) {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- storeRelease(e); /* release old entry */
- RebuildState.dupcount++;
- }
- RebuildState.objcount++;
- storeEntryDump(&tmpe, 5);
- e = storeAddDiskRestore(key,
- sfileno,
- tmpe.swap_file_sz,
- tmpe.expires,
- tmpe.timestamp,
- tmpe.lastref,
- tmpe.lastmod,
- tmpe.refcount, /* refcount */
- tmpe.flags, /* flags */
- d->clean);
- }
- return count;
+ SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
+ return (SD->doubleCheck(*e));
}
-static int
-storeRebuildFromSwapLog(rebuild_dir * d)
+static void
+storeCleanup(void *datanotused)
{
- StoreEntry *e = NULL;
- storeSwapLogData s;
- size_t ss = sizeof(storeSwapLogData);
- int count;
- int used; /* is swapfile already in use? */
- int disk_entry_newer; /* is the log entry newer than current entry? */
- double x;
- assert(d != NULL);
- /* load a number of objects per invocation */
- for (count = 0; count < d->speed; count++) {
- if (fread(&s, ss, 1, d->log) != 1) {
- debug(20, 1) ("Done reading Cache Dir #%d swaplog (%d entries)\n",
- d->dirn, d->n_read);
- fclose(d->log);
- d->log = NULL;
- storeDirCloseTmpSwapLog(d->dirn);
- return -1;
- }
- d->n_read++;
- if (s.op <= SWAP_LOG_NOP)
- continue;
- if (s.op >= SWAP_LOG_MAX)
- continue;
- s.swap_file_number = storeDirProperFileno(d->dirn, s.swap_file_number);
- debug(20, 3) ("storeRebuildFromSwapLog: %s %s %08X\n",
- swap_log_op_str[(int) s.op],
- storeKeyText(s.key),
- s.swap_file_number);
- if (s.op == SWAP_LOG_ADD) {
- (void) 0;
- } else if (s.op == SWAP_LOG_DEL) {
- if ((e = storeGet(s.key)) != NULL) {
- /*
- * Make sure we don't unlink the file, it might be
- * in use by a subsequent entry. Also note that
- * we don't have to subtract from store_swap_size
- * because adding to store_swap_size happens in
- * the cleanup procedure.
- */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_file_number > -1) {
- storeDirMapBitReset(e->swap_file_number);
- e->swap_file_number = -1;
- }
- RebuildState.objcount--;
- RebuildState.cancelcount++;
- }
- continue;
- } else {
- x = log(++RebuildState.bad_log_op) / log(10.0);
- if (0.0 == x - (double) (int) x)
- debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
- RebuildState.bad_log_op);
- RebuildState.invalid++;
- continue;
- }
- if ((++RebuildState.linecount & 0xFFFF) == 0)
- debug(20, 1) (" %7d Entries read so far.\n",
- RebuildState.linecount);
- if (!storeDirValidFileno(s.swap_file_number)) {
- RebuildState.invalid++;
- continue;
- }
- if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
- RebuildState.badflags++;
- continue;
- }
- e = storeGet(s.key);
- used = storeDirMapBitTest(s.swap_file_number);
- /* If this URL already exists in the cache, does the swap log
- * appear to have a newer entry? Compare 'lastref' from the
- * swap log to e->lastref. */
- disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
- /* log entry is old, ignore it */
- RebuildState.clashcount++;
- continue;
- } else if (used && e && e->swap_file_number == s.swap_file_number) {
- /* swapfile taken, same URL, newer, update meta */
- if (e->store_status == STORE_OK) {
- e->lastref = s.timestamp;
- e->timestamp = s.timestamp;
- e->expires = s.expires;
- e->lastmod = s.lastmod;
- e->flags = s.flags;
- e->refcount += s.refcount;
- } else {
- debug_trap("storeRebuildFromSwapLog: bad condition");
- debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
- }
- continue;
- } else if (used) {
- /* swapfile in use, not by this URL, log entry is newer */
- /* This is sorta bad: the log entry should NOT be newer at this
- * point. If the log is dirty, the filesize check should have
- * caught this. If the log is clean, there should never be a
- * newer entry. */
- debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n",
- s.swap_file_number);
- /* I'm tempted to remove the swapfile here just to be safe,
- * but there is a bad race condition in the NOVM version if
- * the swapfile has recently been opened for writing, but
- * not yet opened for reading. Because we can't map
- * swapfiles back to StoreEntrys, we don't know the state
- * of the entry using that file. */
- /* We'll assume the existing entry is valid, probably because
- * were in a slow rebuild and the the swap file number got taken
- * and the validation procedure hasn't run. */
- assert(RebuildState.need_to_validate);
- RebuildState.clashcount++;
- continue;
- } else if (e && !disk_entry_newer) {
- /* key already exists, current entry is newer */
- /* keep old, ignore new */
- RebuildState.dupcount++;
- continue;
- } else if (e) {
- /* key already exists, this swapfile not being used */
- /* junk old, load new */
- storeExpireNow(e);
- storeReleaseRequest(e);
- if (e->swap_file_number > -1) {
- storeDirMapBitReset(e->swap_file_number);
- e->swap_file_number = -1;
- }
- RebuildState.dupcount++;
- } else {
- /* URL doesnt exist, swapfile not in use */
- /* load new */
- (void) 0;
- }
- /* update store_swap_size */
- RebuildState.objcount++;
- e = storeAddDiskRestore(s.key,
- s.swap_file_number,
- s.swap_file_sz,
- s.expires,
- s.timestamp,
- s.lastref,
- s.lastmod,
- s.refcount,
- s.flags,
- d->clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
+ static int store_errors = 0;
+ static StoreSearchPointer currentSearch;
+ static int validated = 0;
+
+ if (currentSearch == NULL || currentSearch->isDone())
+ currentSearch = Store::Root().search(NULL, NULL);
+
+ size_t statCount = 500;
+
+ // TODO: Avoid the loop (and ENTRY_VALIDATED) unless opt_store_doublecheck.
+ while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
+ StoreEntry *e;
+
+ e = currentSearch->currentItem();
+
+ if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
+ continue;
+
+ /*
+ * Calling StoreEntry->release() has no effect because we're
+ * still in 'store_rebuilding' state
+ */
+ if (e->swap_filen < 0)
+ continue;
+
+ if (opt_store_doublecheck)
+ if (storeCleanupDoubleCheck(e))
+ ++store_errors;
+
+ EBIT_SET(e->flags, ENTRY_VALIDATED);
+
+ /*
+ * Only set the file bit if we know its a valid entry
+ * otherwise, set it in the validation procedure
+ */
+
+ if ((++validated & 0x3FFFF) == 0)
+ /* TODO format the int with with a stream operator */
+ debugs(20, DBG_IMPORTANT, " " << validated << " Entries Validated so far.");
}
- return count;
+
+ if (currentSearch->isDone()) {
+ debugs(20, DBG_IMPORTANT, " Completed Validation Procedure");
+ debugs(20, DBG_IMPORTANT, " Validated " << validated << " Entries");
+ debugs(20, DBG_IMPORTANT, " store_swap_size = " << Store::Root().currentSize() / 1024.0 << " KB");
+ --StoreController::store_dirs_rebuilding;
+ assert(0 == StoreController::store_dirs_rebuilding);
+
+ if (opt_store_doublecheck && store_errors) {
+ fatalf("Quitting after finding %d cache index inconsistencies. " \
+ "Removing cache index will force its slow rebuild. " \
+ "Removing -S will let Squid start with an inconsistent " \
+ "cache index (at your own risk).\n", store_errors);
+ }
+
+ if (store_digest)
+ storeDigestNoteStoreReady();
+
+ currentSearch = NULL;
+ } else
+ eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
}
-static void
-storeRebuildADirectory(void *unused)
+/* meta data recreated from disk image in swap directory */
+void
+
+storeRebuildComplete(struct _store_rebuild_data *dc)
{
- int count;
- rebuild_dir *d;
- rebuild_dir **D;
- if ((d = RebuildState.rebuild_dir) == NULL) {
- storeRebuildComplete();
- return;
- }
- count = d->rebuild_func(d);
- RebuildState.rebuild_dir = d->next;
- if (count < 0) {
- xfree(d);
- } else {
- for (D = &RebuildState.rebuild_dir; *D; D = &(*D)->next);
- *D = d;
- d->next = NULL;
- }
- if (opt_foreground_rebuild)
- storeRebuildADirectory(NULL);
- else
- eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
+ double dt;
+ counts.objcount += dc->objcount;
+ counts.expcount += dc->expcount;
+ counts.scancount += dc->scancount;
+ counts.clashcount += dc->clashcount;
+ counts.dupcount += dc->dupcount;
+ counts.cancelcount += dc->cancelcount;
+ counts.invalid += dc->invalid;
+ counts.badflags += dc->badflags;
+ counts.bad_log_op += dc->bad_log_op;
+ counts.zero_object_sz += dc->zero_object_sz;
+ /*
+ * When store_dirs_rebuilding == 1, it means we are done reading
+ * or scanning all cache_dirs. Now report the stats and start
+ * the validation (storeCleanup()) thread.
+ */
+
+ if (StoreController::store_dirs_rebuilding > 1)
+ return;
+
+ dt = tvSubDsec(rebuild_start, current_time);
+
+ debugs(20, DBG_IMPORTANT, "Finished rebuilding storage from disk.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.scancount << " Entries scanned");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.invalid << " Invalid entries.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.badflags << " With invalid flags.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.objcount << " Objects loaded.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.expcount << " Objects expired.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.cancelcount << " Objects cancelled.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.dupcount << " Duplicate URLs purged.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.clashcount << " Swapfile clashes avoided.");
+ debugs(20, DBG_IMPORTANT, " Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
+ ((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
+ debugs(20, DBG_IMPORTANT, "Beginning Validation Procedure");
+
+ eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
+
+ xfree(RebuildProgress);
+
+ RebuildProgress = NULL;
}
-#if TEMP_UNUSED_CODE
-static void
-storeConvertFile(const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_short refcount,
- u_short flags,
- int clean)
+/*
+ * this is ugly. We don't actually start any rebuild threads here,
+ * but only initialize counters, etc. The rebuild threads are
+ * actually started by the filesystem "fooDirInit" function.
+ */
+void
+storeRebuildStart(void)
{
- int fd_r, fd_w;
- int hdr_len, x, y;
- LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN);
- LOCAL_ARRAY(char, copybuf, DISK_PAGE_SIZE);
- char *buf;
- tlv *tlv_list;
- StoreEntry e;
- e.key = key;
- e.swap_file_sz = swap_file_sz;
- e.expires = expires;
- e.lastref = lastref;
- e.refcount = refcount;
- e.flag = flags;
- storeSwapFullPath(file_number, swapfilename);
- fd_r = file_open(swapfilename, O_RDONLY, NULL, NULL, NULL);
- if (fd_r < 0)
- return;
- safeunlink(swapfilename, 1);
- fd_w = file_open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC, NULL, NULL, NULL);
- tlv_list = storeSwapMetaBuild(&e);
- buf = storeSwapMetaPack(tlv_list, &hdr_len);
- x = write(fd_w, buf, hdr_len);
- while (x > 0) {
- y = read(fd_r, copybuf, DISK_PAGE_SIZE);
- x = write(fd_w, copybuf, y);
- }
- file_close(fd_r);
- file_close(fd_w);
- xfree(buf);
- storeSwapTLVFree(tlv_list);
+ memset(&counts, '\0', sizeof(counts));
+ rebuild_start = current_time;
+ /*
+ * Note: store_dirs_rebuilding is initialized to 1.
+ *
+ * When we parse the configuration and construct each swap dir,
+ * the construction of that raises the rebuild count.
+ *
+ * This prevents us from trying to write clean logs until we
+ * finished rebuilding - including after a reconfiguration that opens an
+ * existing swapdir. The corresponding decrement * occurs in
+ * storeCleanup(), when it is finished.
+ */
+ RebuildProgress = (store_rebuild_progress *)xcalloc(Config.cacheSwap.n_configured,
+ sizeof(store_rebuild_progress));
}
-#endif
-static int
-storeGetNextFile(rebuild_dir * d, int *sfileno, int *size)
+/*
+ * A fs-specific rebuild procedure periodically reports its
+ * progress.
+ */
+void
+storeRebuildProgress(int sd_index, int total, int sofar)
{
- int fd = -1;
- int used = 0;
- debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n",
- d->flag,
- d->dirn,
- d->curlvl1,
- d->curlvl2);
- if (d->done)
- return -2;
- while (fd < 0 && d->done == 0) {
- fd = -1;
- if (0 == d->flag) { /* initialize, open first file */
- d->done = 0;
- d->curlvl1 = 0;
- d->curlvl2 = 0;
- d->in_dir = 0;
- d->flag = 1;
- assert(Config.cacheSwap.n_configured > 0);
- }
- if (0 == d->in_dir) { /* we need to read in a new directory */
- snprintf(d->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
- Config.cacheSwap.swapDirs[d->dirn].path,
- d->curlvl1, d->curlvl2);
- if (d->flag && d->td != NULL)
- closedir(d->td);
- d->td = opendir(d->fullpath);
- if (d->td == NULL) {
- debug(50, 1) ("storeGetNextFile: opendir: %s: %s\n",
- d->fullpath, xstrerror());
- break;
- }
- d->entry = readdir(d->td); /* skip . and .. */
- d->entry = readdir(d->td);
- if (d->entry == NULL && errno == ENOENT)
- debug(20, 1) ("storeGetNextFile: directory does not exist!.\n");
- debug(20, 3) ("storeGetNextFile: Directory %s\n", d->fullpath);
- }
- if (d->td != NULL && (d->entry = readdir(d->td)) != NULL) {
- d->in_dir++;
- if (sscanf(d->entry->d_name, "%x", &d->fn) != 1) {
- debug(20, 3) ("storeGetNextFile: invalid %s\n",
- d->entry->d_name);
- continue;
- }
- if (!storeFilenoBelongsHere(d->fn, d->dirn, d->curlvl1, d->curlvl2)) {
- debug(20, 3) ("storeGetNextFile: %08X does not belong in %d/%d/%d\n",
- d->fn, d->dirn, d->curlvl1, d->curlvl2);
- continue;
- }
- d->fn = storeDirProperFileno(d->dirn, d->fn);
- used = storeDirMapBitTest(d->fn);
- if (used) {
- debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n");
- continue;
- }
- snprintf(d->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
- d->fullpath, d->entry->d_name);
- debug(20, 3) ("storeGetNextFile: Opening %s\n", d->fullfilename);
- fd = file_open(d->fullfilename, O_RDONLY, NULL, NULL, NULL);
- if (fd < 0)
- debug(50, 1) ("storeGetNextFile: %s: %s\n", d->fullfilename, xstrerror());
- continue;
- }
- d->in_dir = 0;
- if (++d->curlvl2 < Config.cacheSwap.swapDirs[d->dirn].l2)
- continue;
- d->curlvl2 = 0;
- if (++d->curlvl1 < Config.cacheSwap.swapDirs[d->dirn].l1)
- continue;
- d->curlvl1 = 0;
- d->done = 1;
+ static time_t last_report = 0;
+ double n = 0.0;
+ double d = 0.0;
+
+ if (sd_index < 0)
+ return;
+
+ if (sd_index >= Config.cacheSwap.n_configured)
+ return;
+
+ if (NULL == RebuildProgress)
+ return;
+
+ RebuildProgress[sd_index].total = total;
+
+ RebuildProgress[sd_index].scanned = sofar;
+
+ if (squid_curtime - last_report < 15)
+ return;
+
+ for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; ++sd_index) {
+ n += (double) RebuildProgress[sd_index].scanned;
+ d += (double) RebuildProgress[sd_index].total;
}
- *sfileno = d->fn;
- return fd;
+
+ debugs(20, DBG_IMPORTANT, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
+ last_report = squid_curtime;
}
-/* Add a new object to the cache with empty memory copy and pointer to disk
- * use to rebuild store from disk. */
-static StoreEntry *
-storeAddDiskRestore(const cache_key * key,
- int file_number,
- size_t swap_file_sz,
- time_t expires,
- time_t timestamp,
- time_t lastref,
- time_t lastmod,
- u_num32 refcount,
- u_short flags,
- int clean)
+#include "fde.h"
+#include "StoreMetaUnpacker.h"
+#include "StoreMeta.h"
+#include "Generic.h"
+
+struct InitStoreEntry : public unary_function<StoreMeta, void> {
+ InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
+
+ void operator()(StoreMeta const &x) {
+ switch (x.getType()) {
+
+ case STORE_META_KEY:
+ assert(x.length == SQUID_MD5_DIGEST_LENGTH);
+ memcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH);
+ break;
+
+ case STORE_META_STD:
+ struct old_metahdr {
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ size_t swap_file_sz;
+ uint16_t refcount;
+ uint16_t flags;
+ } *tmp;
+ tmp = (struct old_metahdr *)x.value;
+ assert(x.length == STORE_HDR_METASIZE_OLD);
+ what->timestamp = tmp->timestamp;
+ what->lastref = tmp->lastref;
+ what->expires = tmp->expires;
+ what->lastmod = tmp->lastmod;
+ what->swap_file_sz = tmp->swap_file_sz;
+ what->refcount = tmp->refcount;
+ what->flags = tmp->flags;
+ break;
+
+ case STORE_META_STD_LFS:
+ assert(x.length == STORE_HDR_METASIZE);
+ memcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ StoreEntry *what;
+ cache_key *index;
+};
+
+bool
+storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf,
+ struct _store_rebuild_data &counts)
{
- StoreEntry *e = NULL;
- debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
- /* if you call this you'd better be sure file_number is not
- * already in use! */
- e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
- e->store_status = STORE_OK;
- storeSetMemStatus(e, NOT_IN_MEMORY);
- e->swap_status = SWAPOUT_DONE;
- e->swap_file_number = file_number;
- e->swap_file_sz = swap_file_sz;
- e->lock_count = 0;
- e->refcount = 0;
- e->lastref = lastref;
- e->timestamp = timestamp;
- e->expires = expires;
- e->lastmod = lastmod;
- e->refcount = refcount;
- e->flags = flags;
- EBIT_SET(e->flags, ENTRY_CACHABLE);
- EBIT_CLR(e->flags, RELEASE_REQUEST);
- EBIT_CLR(e->flags, KEY_PRIVATE);
- e->ping_status = PING_NONE;
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- storeDirMapBitSet(e->swap_file_number);
- storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */
- return e;
+ if (fd < 0)
+ return false;
+
+ assert(buf.hasSpace()); // caller must allocate
+
+ const int len = FD_READ_METHOD(fd, buf.space(), buf.spaceSize());
+ ++ statCounter.syscalls.disk.reads;
+ if (len < 0) {
+ const int xerrno = errno;
+ debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << diskIndex << "]: " <<
+ "Ignoring cached entry after meta data read failure: " << xstrerr(xerrno));
+ return false;
+ }
+
+ buf.appended(len);
+ return true;
}
-static void
-storeCleanup(void *datanotused)
+bool
+storeRebuildParseEntry(MemBuf &buf, StoreEntry &tmpe, cache_key *key,
+ struct _store_rebuild_data &counts,
+ uint64_t expectedSize)
{
- static int bucketnum = -1;
- static int validnum = 0;
- static int store_errors = 0;
- StoreEntry *e;
- hash_link *link_ptr = NULL;
- hash_link *link_next = NULL;
- if (++bucketnum >= store_hash_buckets) {
- debug(20, 1) (" Completed Validation Procedure\n");
- debug(20, 1) (" Validated %d Entries\n", validnum);
- debug(20, 1) (" store_swap_size = %dk\n", store_swap_size);
- store_rebuilding = 0;
- if (opt_store_doublecheck)
- assert(store_errors == 0);
- if (store_digest)
- storeDigestNoteStoreReady();
- return;
+ int swap_hdr_len = 0;
+ StoreMetaUnpacker aBuilder(buf.content(), buf.contentSize(), &swap_hdr_len);
+ if (aBuilder.isBufferZero()) {
+ debugs(47,5, HERE << "skipping empty record.");
+ return false;
}
- link_next = hash_get_bucket(store_table, bucketnum);
- while (NULL != (link_ptr = link_next)) {
- link_next = link_ptr->next;
- e = (StoreEntry *) link_ptr;
- if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
- continue;
- /*
- * Calling storeRelease() has no effect because we're
- * still in 'store_rebuilding' state
- */
- if (e->swap_file_number < 0)
- continue;
- if (opt_store_doublecheck) {
- struct stat sb;
- if (stat(storeSwapFullPath(e->swap_file_number, NULL), &sb) < 0) {
- store_errors++;
- debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n");
- debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
- debug(20, 0) ("storeCleanup: PATH %s\n",
- storeSwapFullPath(e->swap_file_number, NULL));
- storeEntryDump(e, 0);
- continue;
- }
- if (e->swap_file_sz != sb.st_size) {
- store_errors++;
- debug(20, 0) ("storeCleanup: SIZE MISMATCH\n");
- debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
- debug(20, 0) ("storeCleanup: PATH %s\n",
- storeSwapFullPath(e->swap_file_number, NULL));
- debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n",
- e->swap_file_sz, (int) sb.st_size);
- storeEntryDump(e, 0);
- continue;
- }
- }
- EBIT_SET(e->flags, ENTRY_VALIDATED);
- /* Only set the file bit if we know its a valid entry */
- /* otherwise, set it in the validation procedure */
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
- if ((++validnum & 0xFFFF) == 0)
- debug(20, 1) (" %7d Entries Validated so far.\n", validnum);
+
+ if (!aBuilder.isBufferSane()) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring malformed cache entry.");
+ return false;
}
- eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
-}
-void
-storeValidate(StoreEntry * e, STVLDCB * callback, void *callback_data, void *tag)
-{
- valid_ctrl_t *ctrlp;
- char *path;
- struct stat *sb;
-#if !USE_ASYNC_IO
- int x;
-#endif
- assert(!EBIT_TEST(e->flags, ENTRY_VALIDATED));
- if (e->swap_file_number < 0) {
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- callback(callback_data, 0, 0);
- return;
+ StoreMeta *tlv_list = aBuilder.createStoreMeta();
+ if (!tlv_list) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with invalid " <<
+ "meta data");
+ return false;
}
- path = storeSwapFullPath(e->swap_file_number, NULL);
- sb = xmalloc(sizeof(struct stat));
- ctrlp = xmalloc(sizeof(valid_ctrl_t));
- ctrlp->sb = sb;
- ctrlp->e = e;
- ctrlp->callback = callback;
- ctrlp->callback_data = callback_data;
-#if USE_ASYNC_IO
- aioStat(path, sb, storeValidateComplete, ctrlp, tag);
-#else
- /*
- * When evaluating the actual arguments in a function call, the order
- * in which the arguments and the function expression are evaluated is
- * not specified;
- */
- x = stat(path, sb);
- storeValidateComplete(-1, ctrlp, x, errno);
-#endif
- return;
-}
-static void
-storeValidateComplete(int fd, void *data, int retcode, int errcode)
-{
- valid_ctrl_t *ctrlp = data;
- struct stat *sb = ctrlp->sb;
- StoreEntry *e = ctrlp->e;
- char *path;
-
- if (retcode == -2 && errcode == -2) {
- xfree(sb);
- xfree(ctrlp);
- ctrlp->callback(ctrlp->callback_data, retcode, errcode);
- return;
+ // TODO: consume parsed metadata?
+
+ debugs(47,7, HERE << "successful swap meta unpacking");
+ memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
+
+ InitStoreEntry visitor(&tmpe, key);
+ for_each(*tlv_list, visitor);
+ storeSwapTLVFree(tlv_list);
+ tlv_list = NULL;
+
+ if (storeKeyNull(key)) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring keyless cache entry");
+ return false;
}
- if (retcode < 0 && errcode == EWOULDBLOCK) {
- path = storeSwapFullPath(e->swap_file_number, NULL);
- retcode = stat(path, sb);
+
+ tmpe.key = key;
+ /* check sizes */
+
+ if (expectedSize > 0) {
+ if (tmpe.swap_file_sz == 0) {
+ tmpe.swap_file_sz = expectedSize;
+ } else if (tmpe.swap_file_sz == (uint64_t)(expectedSize - swap_hdr_len)) {
+ tmpe.swap_file_sz = expectedSize;
+ } else if (tmpe.swap_file_sz != expectedSize) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry due to a " <<
+ "SIZE MISMATCH " << tmpe.swap_file_sz << "!=" << expectedSize);
+ return false;
+ }
+ } else if (tmpe.swap_file_sz <= 0) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with " <<
+ "unknown size: " << tmpe);
+ return false;
}
- if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->swap_file_sz) {
- EBIT_CLR(e->flags, ENTRY_VALIDATED);
- } else {
- EBIT_SET(e->flags, ENTRY_VALIDATED);
- storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
+
+ if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
+ ++ counts.badflags;
+ return false;
}
- errno = errcode;
- ctrlp->callback(ctrlp->callback_data, retcode, errcode);
- xfree(sb);
- xfree(ctrlp);
-}
-/* meta data recreated from disk image in swap directory */
-static void
-storeRebuildComplete(void)
-{
- time_t r;
- time_t stop;
- stop = squid_curtime;
- r = stop - RebuildState.start;
- debug(20, 1) ("Finished rebuilding storage disk.\n");
- debug(20, 1) (" %7d Entries read from previous logfile.\n",
- RebuildState.linecount);
- debug(20, 1) (" %7d Entries scanned from swap files.\n",
- RebuildState.statcount);
- debug(20, 1) (" %7d Invalid entries.\n", RebuildState.invalid);
- debug(20, 1) (" %7d With invalid flags.\n", RebuildState.badflags);
- debug(20, 1) (" %7d Objects loaded.\n", RebuildState.objcount);
- debug(20, 1) (" %7d Objects expired.\n", RebuildState.expcount);
- debug(20, 1) (" %7d Objects cancelled.\n", RebuildState.cancelcount);
- debug(20, 1) (" %7d Duplicate URLs purged.\n", RebuildState.dupcount);
- debug(20, 1) (" %7d Swapfile clashes avoided.\n", RebuildState.clashcount);
- debug(20, 1) (" Took %d seconds (%6.1f objects/sec).\n",
- r > 0 ? (int) r : 0,
- (double) RebuildState.objcount / (r > 0 ? r : 1));
- debug(20, 1) ("Beginning Validation Procedure\n");
- eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
+ return true;
}
-void
-storeRebuildStart(void)
+bool
+storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key,
+ struct _store_rebuild_data &counts)
{
- rebuild_dir *d;
- int clean = 0;
- int zero = 0;
- FILE *fp;
- int i;
- memset(&RebuildState, '\0', sizeof(RebuildState));
- RebuildState.start = squid_curtime;
- for (i = 0; i < Config.cacheSwap.n_configured; i++) {
- d = xcalloc(1, sizeof(rebuild_dir));
- d->dirn = i;
- d->speed = opt_foreground_rebuild ? 1 << 30 : 50;
- /*
- * If the swap.state file exists in the cache_dir, then
- * we'll use storeRebuildFromSwapLog(), otherwise we'll
- * use storeRebuildFromDirectory() to open up each file
- * and suck in the meta data.
- */
- fp = storeDirOpenTmpSwapLog(i, &clean, &zero);
- if (fp == NULL || zero) {
- if (fp != NULL)
- fclose(fp);
- d->rebuild_func = storeRebuildFromDirectory;
- } else {
- d->rebuild_func = storeRebuildFromSwapLog;
- d->log = fp;
- d->clean = clean;
- }
- d->next = RebuildState.rebuild_dir;
- RebuildState.rebuild_dir = d;
- if (!clean)
- RebuildState.need_to_validate = 1;
- debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n",
- i, clean ? "CLEAN" : "DIRTY");
+ /* this needs to become
+ * 1) unpack url
+ * 2) make synthetic request with headers ?? or otherwise search
+ * for a matching object in the store
+ * TODO FIXME change to new async api
+ * TODO FIXME I think there is a race condition here with the
+ * async api :
+ * store A reads in object foo, searchs for it, and finds nothing.
+ * store B reads in object foo, searchs for it, finds nothing.
+ * store A gets called back with nothing, so registers the object
+ * store B gets called back with nothing, so registers the object,
+ * which will conflict when the in core index gets around to scanning
+ * store B.
+ *
+ * this suggests that rather than searching for duplicates, the
+ * index rebuild should just assume its the most recent accurate
+ * store entry and whoever indexes the stores handles duplicates.
+ */
+ if (StoreEntry *e = Store::Root().get(key)) {
+
+ if (e->lastref >= tmpe.lastref) {
+ /* key already exists, old entry is newer */
+ /* keep old, ignore new */
+ ++counts.dupcount;
+
+ // For some stores, get() creates/unpacks a store entry. Signal
+ // such stores that we will no longer use the get() result:
+ e->lock();
+ e->unlock();
+
+ return false;
+ } else {
+ /* URL already exists, this swapfile not being used */
+ /* junk old, load new */
+ e->release(); /* release old entry */
+ ++counts.dupcount;
+ }
}
- eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
+
+ return true;
}
projects / thirdparty / squid.git / blobdiff