-
/*
- * $Id$
- *
- * DEBUG: section 20 Store Rebuild Routines
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- * Squid is the result of efforts by numerous individuals from
- * the Internet community; see the CONTRIBUTORS file for full
- * details. Many organizations have provided support for Squid's
- * development; see the SPONSORS file for full details. Squid is
- * Copyrighted (C) 2001 by the Regents of the University of
- * California; see the COPYRIGHT file for full details. Squid
- * incorporates software developed and/or copyrighted by other
- * sources; see the CREDITS file for full details.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
*
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
*/
+/* DEBUG: section 20 Store Rebuild Routines */
+
#include "squid.h"
#include "event.h"
+#include "globals.h"
+#include "md5.h"
+#include "SquidConfig.h"
+#include "SquidTime.h"
#include "StatCounters.h"
#include "Store.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
+#include "store_digest.h"
+#include "store_key_md5.h"
+#include "store_rebuild.h"
#include "StoreSearch.h"
-#include "SquidTime.h"
+// for tvSubDsec() which should be in SquidTime.h
+#include "util.h"
+
+#include <cerrno>
-static struct _store_rebuild_data counts;
+static StoreRebuildData counts;
static struct timeval rebuild_start;
static void storeCleanup(void *);
}
static void
-storeCleanup(void *datanotused)
+storeCleanup(void *)
{
static int store_errors = 0;
static StoreSearchPointer currentSearch;
static int validated = 0;
+ static int seen = 0;
if (currentSearch == NULL || currentSearch->isDone())
- currentSearch = Store::Root().search(NULL, NULL);
+ currentSearch = Store::Root().search();
size_t statCount = 500;
+ // TODO: Avoid the loop (and ENTRY_VALIDATED) unless opt_store_doublecheck.
while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
StoreEntry *e;
e = currentSearch->currentItem();
+ ++seen;
+
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
continue;
if (opt_store_doublecheck)
if (storeCleanupDoubleCheck(e))
- store_errors++;
+ ++store_errors;
EBIT_SET(e->flags, ENTRY_VALIDATED);
if ((++validated & 0x3FFFF) == 0)
/* TODO format the int with with a stream operator */
- debugs(20, 1, " " << validated << " Entries Validated so far.");
+ debugs(20, DBG_IMPORTANT, " " << validated << " Entries Validated so far.");
}
if (currentSearch->isDone()) {
- debugs(20, 1, " Completed Validation Procedure");
- debugs(20, 1, " Validated " << validated << " Entries");
- debugs(20, 1, " store_swap_size = " << Store::Root().currentSize() / 1024.0 << " KB");
- StoreController::store_dirs_rebuilding--;
+ debugs(20, 2, "Seen: " << seen << " entries");
+ debugs(20, DBG_IMPORTANT, " Completed Validation Procedure");
+ debugs(20, DBG_IMPORTANT, " Validated " << validated << " Entries");
+ debugs(20, DBG_IMPORTANT, " store_swap_size = " << Store::Root().currentSize() / 1024.0 << " KB");
+ --StoreController::store_dirs_rebuilding;
assert(0 == StoreController::store_dirs_rebuilding);
- if (opt_store_doublecheck)
- assert(store_errors == 0);
+ if (opt_store_doublecheck && store_errors) {
+ fatalf("Quitting after finding %d cache index inconsistencies. " \
+ "Removing cache index will force its slow rebuild. " \
+ "Removing -S will let Squid start with an inconsistent " \
+ "cache index (at your own risk).\n", store_errors);
+ }
if (store_digest)
storeDigestNoteStoreReady();
/* meta data recreated from disk image in swap directory */
void
-storeRebuildComplete(struct _store_rebuild_data *dc)
+storeRebuildComplete(StoreRebuildData *dc)
{
double dt;
counts.objcount += dc->objcount;
dt = tvSubDsec(rebuild_start, current_time);
- debugs(20, 1, "Finished rebuilding storage from disk.");
- debugs(20, 1, " " << std::setw(7) << counts.scancount << " Entries scanned");
- debugs(20, 1, " " << std::setw(7) << counts.invalid << " Invalid entries.");
- debugs(20, 1, " " << std::setw(7) << counts.badflags << " With invalid flags.");
- debugs(20, 1, " " << std::setw(7) << counts.objcount << " Objects loaded.");
- debugs(20, 1, " " << std::setw(7) << counts.expcount << " Objects expired.");
- debugs(20, 1, " " << std::setw(7) << counts.cancelcount << " Objects cancelled.");
- debugs(20, 1, " " << std::setw(7) << counts.dupcount << " Duplicate URLs purged.");
- debugs(20, 1, " " << std::setw(7) << counts.clashcount << " Swapfile clashes avoided.");
- debugs(20, 1, " Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
+ debugs(20, DBG_IMPORTANT, "Finished rebuilding storage from disk.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.scancount << " Entries scanned");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.invalid << " Invalid entries.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.badflags << " With invalid flags.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.objcount << " Objects loaded.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.expcount << " Objects expired.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.cancelcount << " Objects cancelled.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.dupcount << " Duplicate URLs purged.");
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.clashcount << " Swapfile clashes avoided.");
+ debugs(20, DBG_IMPORTANT, " Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
- debugs(20, 1, "Beginning Validation Procedure");
+ debugs(20, DBG_IMPORTANT, "Beginning Validation Procedure");
eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
if (squid_curtime - last_report < 15)
return;
- for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
+ for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; ++sd_index) {
n += (double) RebuildProgress[sd_index].scanned;
d += (double) RebuildProgress[sd_index].total;
}
- debugs(20, 1, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
+ debugs(20, DBG_IMPORTANT, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
last_report = squid_curtime;
}
#include "fde.h"
-#include "StoreMetaUnpacker.h"
-#include "StoreMeta.h"
#include "Generic.h"
+#include "StoreMeta.h"
+#include "StoreMetaUnpacker.h"
struct InitStoreEntry : public unary_function<StoreMeta, void> {
InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
what->timestamp = tmp->timestamp;
what->lastref = tmp->lastref;
what->expires = tmp->expires;
- what->lastmod = tmp->lastmod;
+ what->lastModified(tmp->lastmod);
what->swap_file_sz = tmp->swap_file_sz;
what->refcount = tmp->refcount;
what->flags = tmp->flags;
};
bool
-storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf,
- struct _store_rebuild_data &counts)
+storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &)
{
if (fd < 0)
return false;
assert(buf.hasSpace()); // caller must allocate
const int len = FD_READ_METHOD(fd, buf.space(), buf.spaceSize());
- statCounter.syscalls.disk.reads++;
+ ++ statCounter.syscalls.disk.reads;
if (len < 0) {
const int xerrno = errno;
debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << diskIndex << "]: " <<
bool
storeRebuildParseEntry(MemBuf &buf, StoreEntry &tmpe, cache_key *key,
- struct _store_rebuild_data &counts,
+ StoreRebuildData &stats,
uint64_t expectedSize)
{
int swap_hdr_len = 0;
return false;
}
- if (!aBuilder.isBufferSane()) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring malformed cache entry.");
- return false;
- }
-
- StoreMeta *tlv_list = aBuilder.createStoreMeta();
- if (!tlv_list) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with invalid " <<
- "meta data");
+ StoreMeta *tlv_list = nullptr;
+ try {
+ tlv_list = aBuilder.createStoreMeta();
+ } catch (const std::exception &e) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring store entry because " << e.what());
return false;
}
+ assert(tlv_list);
// TODO: consume parsed metadata?
- debugs(47,7, HERE << "successful swap meta unpacking");
+ debugs(47,7, "successful swap meta unpacking; swap_file_sz=" << tmpe.swap_file_sz);
memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
InitStoreEntry visitor(&tmpe, key);
return false;
}
} else if (tmpe.swap_file_sz <= 0) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with " <<
- "unknown size: " << tmpe);
- return false;
+ // if caller cannot handle unknown sizes, it must check after the call.
+ debugs(47, 7, "unknown size: " << tmpe);
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- counts.badflags++;
+ ++ stats.badflags;
return false;
}
}
bool
-storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key,
- struct _store_rebuild_data &counts)
+storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key, StoreRebuildData &stats)
{
/* this needs to become
* 1) unpack url
if (e->lastref >= tmpe.lastref) {
/* key already exists, old entry is newer */
/* keep old, ignore new */
- counts.dupcount++;
+ ++stats.dupcount;
// For some stores, get() creates/unpacks a store entry. Signal
// such stores that we will no longer use the get() result:
- e->lock();
- e->unlock();
+ e->lock("storeRebuildKeepEntry");
+ e->unlock("storeRebuildKeepEntry");
return false;
} else {
/* URL already exists, this swapfile not being used */
/* junk old, load new */
- e->release(); /* release old entry */
- counts.dupcount++;
+ e->release(); /* release old entry */
+ ++stats.dupcount;
}
}
return true;
}
+
projects / thirdparty / squid.git / blobdiff