by me.
This code implements a two-tiered doublecheck - -S reportonly which warns
for mismatched file lengths, and -S force which will unlink files which
fail a doublecheck. Actual file unlinking is also implemented now, so when
a double check is performed, it will actually clean up the cache store.
/*
- * $Id: enums.h,v 1.175 2000/11/09 03:09:47 wessels Exp $
+ * $Id: enums.h,v 1.176 2000/11/10 09:04:50 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
NETDB_EX_RTT,
NETDB_EX_HOPS
};
+
+/*
+ * These are used for the store double-check code
+ */
+typedef enum {
+ DBLCHECK_NONE,
+ DBLCHECK_REPORTONLY,
+ DBLCHECK_FORCE
+} dblcheck_t;
/*
- * $Id: store_dir_aufs.cc,v 1.14 2000/11/01 22:04:28 wessels Exp $
+ * $Id: store_dir_aufs.cc,v 1.15 2000/11/10 09:04:52 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static int storeAufsDirIs(SwapDir * sd);
static int storeAufsFilenoBelongsHere(int fn, int F0, int F1, int F2);
static int storeAufsCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static int storeAufsCheckFile(SwapDir * sd, sfileno filn, size_t swap_file_sz );
static void storeAufsDirStats(SwapDir *, StoreEntry *);
static void storeAufsDirInitBitmap(SwapDir *);
static int storeAufsDirValidFileno(SwapDir *, sfileno, int);
debug(20, 1) ("storeAufsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
tmpe.swap_file_sz, (int) sb.st_size);
storeAufsDirUnlinkFile(SD, sfileno);
+ rb->counts.filesizemismatchcount++;
continue;
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
int count;
int used; /* is swapfile already in use? */
int disk_entry_newer; /* is the log entry newer than current entry? */
+ int file_bad;
double x;
assert(rb != NULL);
/* load a number of objects per invocation */
* appear to have a newer entry? Compare 'lastref' from the
* swap log to e->lastref. */
disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
+ if (!used && (rb->flags.need_to_validate ||
+ (opt_store_doublecheck != DBLCHECK_NONE))){
+ file_bad = storeAufsCheckFile(SD, s.swap_filen, s.swap_file_sz);
+ } else
+ file_bad = 0;
+ if (file_bad){
+ if (file_bad==-2){
+ if (opt_store_doublecheck == DBLCHECK_FORCE)
+ storeAufsDirUnlinkFile(SD, s.swap_filen);
+ rb->counts.filesizemismatchcount++;
+ continue;
+ }
+ rb->counts.missingcount++;
+ continue;
+ } else if (used && !disk_entry_newer) {
/* log entry is old, ignore it */
rb->counts.clashcount++;
continue;
*/
static int
storeAufsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
+{
+ int rv= storeAufsCheckFile(sd, e->swap_filen, e->swap_file_sz);
+ if (rv) storeEntryDump(e, 0);
+ return rv;
+}
+
+/*
+ * storeAufsCheckFile
+ *
+ * This is called by storerebuildFromSwapLog if -S was given on the command
+ * line. or a Dirty state is found
+ */
+static int
+storeAufsCheckFile(SwapDir * sd, sfileno filen, size_t swap_file_sz )
{
struct stat sb;
- if (stat(storeAufsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(20, 0) ("storeAufsCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
- storeAufsDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
+ if (stat(storeAufsDirFullPath(sd, filen, NULL), &sb) < 0) {
+ debug(20, 1) ("storeAufsCheckFile: MISSING SWAP FILE\n");
+ debug(20, 1) ("storeAufsCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeAufsCheckFile: PATH %s\n", storeAufsDirFullPath(sd, filen, NULL));
+ return -1;
}
- if (e->swap_file_sz != sb.st_size) {
- debug(20, 0) ("storeAufsCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(20, 0) ("storeAufsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeAufsCleanupDoubleCheck: PATH %s\n",
- storeAufsDirFullPath(sd, e->swap_filen, NULL));
- debug(20, 0) ("storeAufsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
- e->swap_file_sz, (int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
+ if (swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeAufsCheckFile: SIZE MISMATCH\n");
+ debug(20, 1) ("storeAufsCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeAufsCheckFile: PATH %s\n", storeAufsDirFullPath(sd, filen, NULL));
+ debug(20, 1) ("storeAufsCheckFile: ENTRY SIZE: %d, FILE SIZE: %d\n", swap_file_sz, (int) sb.st_size);
+ return -2;
}
return 0;
}
/*
- * $Id: store_dir_diskd.cc,v 1.23 2000/11/01 03:35:48 wessels Exp $
+ * $Id: store_dir_diskd.cc,v 1.24 2000/11/10 09:04:52 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static int storeDiskdDirIs(SwapDir * sd);
static int storeDiskdFilenoBelongsHere(int fn, int F0, int F1, int F2);
static int storeDiskdCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static int storeDiskdCheckFile(SwapDir * sd, sfileno filn, size_t swap_file_sz);
static void storeDiskdDirStats(SwapDir *, StoreEntry *);
static void storeDiskdDirInitBitmap(SwapDir *);
static int storeDiskdDirValidFileno(SwapDir *, sfileno, int);
debug(20, 1) ("storeDiskdDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
tmpe.swap_file_sz, (int) sb.st_size);
storeDiskdDirUnlinkFile(SD, sfileno);
+ rb->counts.filesizemismatchcount++;
continue;
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
int count;
int used; /* is swapfile already in use? */
int disk_entry_newer; /* is the log entry newer than current entry? */
+ int file_bad;
double x;
assert(rb != NULL);
/* load a number of objects per invocation */
* appear to have a newer entry? Compare 'lastref' from the
* swap log to e->lastref. */
disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
+ if (!used && (rb->flags.need_to_validate ||
+ (opt_store_doublecheck != DBLCHECK_NONE))){
+ file_bad = storeDiskdCheckFile(SD, s.swap_filen, s.swap_file_sz);
+ } else
+ file_bad = 0;
+ if (file_bad){
+ if (file_bad==-2){
+ if (opt_store_doublecheck == DBLCHECK_FORCE)
+ storeDiskdDirUnlinkFile(SD, s.swap_filen);
+ rb->counts.filesizemismatchcount++;
+ continue;
+ }
+ rb->counts.missingcount++;
+ continue;
+ } else if (used && !disk_entry_newer) {
/* log entry is old, ignore it */
rb->counts.clashcount++;
continue;
static int
storeDiskdCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
{
- struct stat sb;
+ int rv= storeDiskdCheckFile(sd, e->swap_filen, e->swap_file_sz);
+ if (rv) storeEntryDump(e, 0);
+ return rv;
+}
- if (stat(storeDiskdDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
- storeDiskdDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
- }
- if (e->swap_file_sz != sb.st_size) {
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: PATH %s\n",
- storeDiskdDirFullPath(sd, e->swap_filen, NULL));
- debug(20, 0) ("storeDiskdCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
- e->swap_file_sz, (int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
+/*
+ * storeDiskdCheckFile
+ *
+ * This is called by storerebuildFromSwapLog if -S was given on the command line * or a Dirty state is found
+ */
+static int
+storeDiskdCheckFile(SwapDir * sd, sfileno filen, size_t swap_file_sz )
+{
+ struct stat sb;
+
+ if (stat(storeDiskdDirFullPath(sd, filen, NULL), &sb) < 0) {
+ debug(20, 1) ("storeDiskdCheckFile: MISSING SWAP FILE\n");
+ debug(20, 1) ("storeDiskdCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeDiskdCheckFile: PATH %s\n", storeDiskdDirFullPath(sd, filen, NULL));
+
+ return -1;
}
+ if (swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeDiskdCheckFile: SIZE MISMATCH\n");
+ debug(20, 1) ("storeDiskdCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeDiskdCheckFile: PATH %s\n", storeDiskdDirFullPath(sd, filen, NULL));
+ debug(20, 1) ("storeDiskdCheckFile: ENTRY SIZE: %d, FILE SIZE: %d\n", swap_file_sz, (int) sb.st_size);
+ return -2;
+ }
return 0;
}
/*
- * $Id: store_dir_ufs.cc,v 1.13 2000/11/01 21:48:18 wessels Exp $
+ * $Id: store_dir_ufs.cc,v 1.14 2000/11/10 09:04:53 adrian Exp $
*
* DEBUG: section 47 Store Directory Routines
* AUTHOR: Duane Wessels
static int storeUfsDirIs(SwapDir * sd);
static int storeUfsFilenoBelongsHere(int fn, int F0, int F1, int F2);
static int storeUfsCleanupDoubleCheck(SwapDir *, StoreEntry *);
+static int storeUfsCheckFile(SwapDir * sd, sfileno filn, size_t swap_file_sz );
static void storeUfsDirStats(SwapDir *, StoreEntry *);
static void storeUfsDirInitBitmap(SwapDir *);
static int storeUfsDirValidFileno(SwapDir *, sfileno, int);
debug(20, 1) ("storeUfsDirRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
tmpe.swap_file_sz, (int) sb.st_size);
storeUfsDirUnlinkFile(SD, sfileno);
+ rb->counts.filesizemismatchcount++;
continue;
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
int count;
int used; /* is swapfile already in use? */
int disk_entry_newer; /* is the log entry newer than current entry? */
+ int file_bad;
double x;
assert(rb != NULL);
/* load a number of objects per invocation */
* appear to have a newer entry? Compare 'lastref' from the
* swap log to e->lastref. */
disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
- if (used && !disk_entry_newer) {
+ if (!used && (rb->flags.need_to_validate ||
+ (opt_store_doublecheck != DBLCHECK_NONE))){
+ file_bad = storeUfsCheckFile(SD, s.swap_filen, s.swap_file_sz);
+ } else
+ file_bad = 0;
+ if (file_bad){
+ if (file_bad==-2){
+ if (opt_store_doublecheck == DBLCHECK_FORCE)
+ storeUfsDirUnlinkFile(SD, s.swap_filen);
+ rb->counts.filesizemismatchcount++;
+ continue;
+ }
+ rb->counts.missingcount++;
+ continue;
+ }else if (used && !disk_entry_newer) {
/* log entry is old, ignore it */
rb->counts.clashcount++;
continue;
/* load new */
(void) 0;
}
- /* update store_swap_size */
- rb->counts.objcount++;
- e = storeUfsDirAddDiskRestore(SD, s.key,
- s.swap_filen,
- s.swap_file_sz,
- s.expires,
- s.timestamp,
- s.lastref,
- s.lastmod,
- s.refcount,
- s.flags,
- (int) rb->flags.clean);
- storeDirSwapLog(e, SWAP_LOG_ADD);
+ /* update store_swap_size */
+ rb->counts.objcount++;
+ e = storeUfsDirAddDiskRestore(SD, s.key,
+ s.swap_filen,
+ s.swap_file_sz,
+ s.expires,
+ s.timestamp,
+ s.lastref,
+ s.lastmod,
+ s.refcount,
+ s.flags,
+ (int) rb->flags.clean);
+ storeDirSwapLog(e, SWAP_LOG_ADD);
}
eventAdd("storeRebuild", storeUfsDirRebuildFromSwapLog, rb, 0.0, 1);
}
*/
static int
storeUfsCleanupDoubleCheck(SwapDir * sd, StoreEntry * e)
+{
+ int rv= storeUfsCheckFile(sd, e->swap_filen, e->swap_file_sz);
+ if (rv) storeEntryDump(e, 0);
+ return rv;
+}
+
+/*
+ * storeUfsCheckFile
+ *
+ * This is called by storerebuildFromSwapLog if -S was given on the command line. or a Dirty state is found
+ */
+static int
+storeUfsCheckFile(SwapDir * sd, sfileno filen, size_t swap_file_sz )
{
struct stat sb;
- if (stat(storeUfsDirFullPath(sd, e->swap_filen, NULL), &sb) < 0) {
- debug(20, 0) ("storeUfsCleanupDoubleCheck: MISSING SWAP FILE\n");
- debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
- storeUfsDirFullPath(sd, e->swap_filen, NULL));
- storeEntryDump(e, 0);
- return -1;
+ if (stat(storeUfsDirFullPath(sd, filen, NULL), &sb) < 0) {
+ debug(20, 1) ("storeUfsCheckFile: MISSING SWAP FILE\n");
+ debug(20, 1) ("storeUfsCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeUfsCheckFile: PATH %s\n", storeUfsDirFullPath(sd, filen, NULL));
+
+ return -1;
}
- if (e->swap_file_sz != sb.st_size) {
- debug(20, 0) ("storeUfsCleanupDoubleCheck: SIZE MISMATCH\n");
- debug(20, 0) ("storeUfsCleanupDoubleCheck: FILENO %08X\n", e->swap_filen);
- debug(20, 0) ("storeUfsCleanupDoubleCheck: PATH %s\n",
- storeUfsDirFullPath(sd, e->swap_filen, NULL));
- debug(20, 0) ("storeUfsCleanupDoubleCheck: ENTRY SIZE: %d, FILE SIZE: %d\n",
- e->swap_file_sz, (int) sb.st_size);
- storeEntryDump(e, 0);
- return -1;
+ if (swap_file_sz != sb.st_size) {
+ debug(20, 1) ("storeUfsCheckFile: SIZE MISMATCH\n");
+ debug(20, 1) ("storeUfsCheckFile: FILENO %08X\n", filen);
+ debug(20, 1) ("storeUfsCheckFile: PATH %s\n", storeUfsDirFullPath(sd, filen, NULL));
+ debug(20, 1) ("storeUfsCheckFile: ENTRY SIZE: %d, FILE SIZE: %d\n", swap_file_sz, (int) sb.st_size);
+ return -2;
}
return 0;
}
+
+
/*
* storeUfsDirParse
*
/*
- * $Id: globals.h,v 1.97 2000/07/18 06:16:41 wessels Exp $
+ * $Id: globals.h,v 1.98 2000/11/10 09:04:51 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
extern int opt_syslog_enable; /* 0 */
extern int opt_udp_hit_obj; /* 0 */
extern int opt_create_swap_dirs; /* 0 */
-extern int opt_store_doublecheck; /* 0 */
+extern dblcheck_t opt_store_doublecheck; /* DBLCHECK_NONE */
extern int syslog_enable; /* 0 */
extern int theInIcpConnection; /* -1 */
extern int theOutIcpConnection; /* -1 */
/*
- * $Id: main.cc,v 1.321 2000/11/01 04:03:14 wessels Exp $
+ * $Id: main.cc,v 1.322 2000/11/10 09:04:51 adrian Exp $
*
* DEBUG: section 1 Startup and Main Loop
* AUTHOR: Harvest Derived
" -F Don't serve any requests until store is rebuilt.\n"
" -N No daemon mode.\n"
" -R Do not set REUSEADDR on port.\n"
- " -S Double-check swap during rebuild.\n"
+ " -S force Force double-check swap during rebuild.\n"
+ " -S reportonly Force double-check but do not attempt repair.\n"
" -V Virtual host httpd-accelerator.\n"
" -X Force full debugging.\n"
" -Y Only return UDP_HIT or UDP_MISS_NOFETCH during fast reload.\n",
extern char *optarg;
int c;
- while ((c = getopt(argc, argv, "CDFNRSVYXa:d:f:hk:m::su:vz?")) != -1) {
+ while ((c = getopt(argc, argv, "CDFNRS:VYXa:d:f:hk:m::su:vz?")) != -1) {
switch (c) {
case 'C':
opt_catch_signals = 0;
opt_reuseaddr = 0;
break;
case 'S':
- opt_store_doublecheck = 1;
+ if ((int) strlen(optarg) < 1)
+ usage();
+ else if (!strncmp(optarg, "force", strlen(optarg)))
+ opt_store_doublecheck = DBLCHECK_FORCE; /* trigger a doublecheck on startup */
+ else if (!strncmp(optarg, "reportonly", strlen(optarg)))
+ opt_store_doublecheck = DBLCHECK_REPORTONLY; /* trigger a doublecheck that doesn't repair */
+ else
+ usage();
break;
case 'V':
vhost_mode = 1;
/*
- * $Id: store_rebuild.cc,v 1.71 2000/11/01 03:35:41 wessels Exp $
+ * $Id: store_rebuild.cc,v 1.72 2000/11/10 09:04:51 adrian Exp $
*
* DEBUG: section 20 Store Rebuild Routines
* AUTHOR: Duane Wessels
debug(20, 1) (" store_swap_size = %dk\n", store_swap_size);
store_dirs_rebuilding--;
assert(0 == store_dirs_rebuilding);
- if (opt_store_doublecheck)
+ if (opt_store_doublecheck != DBLCHECK_NONE)
+ /* we want to assert here because the storeFScode should auto-clean the entries */
assert(store_errors == 0);
if (store_digest)
storeDigestNoteStoreReady();
*/
if (e->swap_filen < 0)
continue;
- if (opt_store_doublecheck)
- if (storeCleanupDoubleCheck(e))
+ if (opt_store_doublecheck != DBLCHECK_NONE)
+ if (storeCleanupDoubleCheck(e)){
+ /* this should never happen as the storeFScode should auto-clean */
store_errors++;
+ }
EBIT_SET(e->flags, ENTRY_VALIDATED);
/*
* Only set the file bit if we know its a valid entry
counts.invalid += dc->invalid;
counts.badflags += dc->badflags;
counts.bad_log_op += dc->bad_log_op;
+ counts.missingcount += dc->missingcount;
+ counts.filesizemismatchcount += dc->filesizemismatchcount;
counts.zero_object_sz += dc->zero_object_sz;
/*
* When store_dirs_rebuilding == 1, it means we are done reading
debug(20, 1) (" %7d Objects cancelled.\n", counts.cancelcount);
debug(20, 1) (" %7d Duplicate URLs purged.\n", counts.dupcount);
debug(20, 1) (" %7d Swapfile clashes avoided.\n", counts.clashcount);
+ debug(20, 1) (" %7d Missing files ignored.\n", counts.missingcount);
+ debug(20, 1) (" %7d Incorrect length swapfiles %s.\n",
+ counts.filesizemismatchcount,
+ (opt_store_doublecheck == DBLCHECK_REPORTONLY) ? "ignored" : "unlinked");
debug(20, 1) (" Took %3.1f seconds (%6.1f objects/sec).\n", dt,
(double) counts.objcount / (dt > 0.0 ? dt : 1.0));
debug(20, 1) ("Beginning Validation Procedure\n");
/*
- * $Id: structs.h,v 1.359 2000/11/09 03:09:47 wessels Exp $
+ * $Id: structs.h,v 1.360 2000/11/10 09:04:51 adrian Exp $
*
*
* SQUID Internet Object Cache http://squid.nlanr.net/Squid/
int cancelcount; /* # SWAP_LOG_DEL objects purged */
int invalid; /* # bad lines */
int badflags; /* # bad e->flags */
+ int missingcount; /* # swapfiles not on disk */
+ int filesizemismatchcount; /* # swapfiles with a wrong size */
int bad_log_op;
int zero_object_sz;
};
projects / thirdparty / squid.git / commitdiff
