/*
- * DEBUG: section 20 Store Rebuild Routines
- * AUTHOR: Duane Wessels
- *
- * SQUID Web Proxy Cache http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- * Squid is the result of efforts by numerous individuals from
- * the Internet community; see the CONTRIBUTORS file for full
- * details. Many organizations have provided support for Squid's
- * development; see the SPONSORS file for full details. Squid is
- * Copyrighted (C) 2001 by the Regents of the University of
- * California; see the COPYRIGHT file for full details. Squid
- * incorporates software developed and/or copyrighted by other
- * sources; see the CREDITS file for full details.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
*
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
*/
+/* DEBUG: section 20 Store Rebuild Routines */
+
#include "squid.h"
#include "event.h"
#include "globals.h"
#include "md5.h"
+#include "SquidConfig.h"
+#include "SquidTime.h"
#include "StatCounters.h"
#include "Store.h"
-#include "store_key_md5.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
#include "store_digest.h"
+#include "store_key_md5.h"
#include "store_rebuild.h"
#include "StoreSearch.h"
-#include "SquidConfig.h"
-#include "SquidTime.h"
+// for tvSubDsec() which should be in SquidTime.h
+#include "util.h"
+
+#include <cerrno>
-#if HAVE_ERRNO_H
-#include <errno.h>
-#endif
static StoreRebuildData counts;
-static struct timeval rebuild_start;
static void storeCleanup(void *);
+// TODO: Either convert to Progress or replace with StoreRebuildData.
+// TODO: Handle unknown totals (UFS cache_dir that lost swap.state) correctly.
typedef struct {
/* total number of "swap.state" entries that will be read */
int total;
static store_rebuild_progress *RebuildProgress = NULL;
-static int
-storeCleanupDoubleCheck(StoreEntry * e)
+void
+StoreRebuildData::updateStartTime(const timeval &dirStartTime)
{
- SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
- return (SD->doubleCheck(*e));
+ startTime = started() ? std::min(startTime, dirStartTime) : dirStartTime;
}
static void
-storeCleanup(void *datanotused)
+storeCleanup(void *)
{
static int store_errors = 0;
static StoreSearchPointer currentSearch;
static int validated = 0;
+ static int seen = 0;
if (currentSearch == NULL || currentSearch->isDone())
- currentSearch = Store::Root().search(NULL, NULL);
+ currentSearch = Store::Root().search();
size_t statCount = 500;
e = currentSearch->currentItem();
+ ++seen;
+
if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
continue;
* Calling StoreEntry->release() has no effect because we're
* still in 'store_rebuilding' state
*/
- if (e->swap_filen < 0)
+ if (!e->hasDisk())
continue;
if (opt_store_doublecheck)
- if (storeCleanupDoubleCheck(e))
+ if (e->disk().doubleCheck(*e))
++store_errors;
EBIT_SET(e->flags, ENTRY_VALIDATED);
}
if (currentSearch->isDone()) {
+ debugs(20, 2, "Seen: " << seen << " entries");
debugs(20, DBG_IMPORTANT, " Completed Validation Procedure");
debugs(20, DBG_IMPORTANT, " Validated " << validated << " Entries");
debugs(20, DBG_IMPORTANT, " store_swap_size = " << Store::Root().currentSize() / 1024.0 << " KB");
storeRebuildComplete(StoreRebuildData *dc)
{
- double dt;
- counts.objcount += dc->objcount;
- counts.expcount += dc->expcount;
- counts.scancount += dc->scancount;
- counts.clashcount += dc->clashcount;
- counts.dupcount += dc->dupcount;
- counts.cancelcount += dc->cancelcount;
- counts.invalid += dc->invalid;
- counts.badflags += dc->badflags;
- counts.bad_log_op += dc->bad_log_op;
- counts.zero_object_sz += dc->zero_object_sz;
+ if (dc) {
+ counts.objcount += dc->objcount;
+ counts.expcount += dc->expcount;
+ counts.scancount += dc->scancount;
+ counts.clashcount += dc->clashcount;
+ counts.dupcount += dc->dupcount;
+ counts.cancelcount += dc->cancelcount;
+ counts.invalid += dc->invalid;
+ counts.badflags += dc->badflags;
+ counts.bad_log_op += dc->bad_log_op;
+ counts.zero_object_sz += dc->zero_object_sz;
+ counts.validations += dc->validations;
+ counts.updateStartTime(dc->startTime);
+ }
+ // else the caller was not responsible for indexing its cache_dir
+
+ assert(StoreController::store_dirs_rebuilding > 1);
+ --StoreController::store_dirs_rebuilding;
+
/*
* When store_dirs_rebuilding == 1, it means we are done reading
* or scanning all cache_dirs. Now report the stats and start
if (StoreController::store_dirs_rebuilding > 1)
return;
- dt = tvSubDsec(rebuild_start, current_time);
+ const auto dt = tvSubDsec(counts.startTime, current_time);
debugs(20, DBG_IMPORTANT, "Finished rebuilding storage from disk.");
debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.scancount << " Entries scanned");
void
storeRebuildStart(void)
{
- memset(&counts, '\0', sizeof(counts));
- rebuild_start = current_time;
+ counts = StoreRebuildData(); // reset counters
/*
* Note: store_dirs_rebuilding is initialized to 1.
*
storeRebuildProgress(int sd_index, int total, int sofar)
{
static time_t last_report = 0;
+ // TODO: Switch to int64_t and fix handling of unknown totals.
double n = 0.0;
double d = 0.0;
d += (double) RebuildProgress[sd_index].total;
}
- debugs(20, DBG_IMPORTANT, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
+ debugs(20, DBG_IMPORTANT, "Indexing cache entries: " << Progress(n, d));
last_report = squid_curtime;
}
+void
+Progress::print(std::ostream &os) const
+{
+ if (goal > 0) {
+ const auto savedPrecision = os.precision(2);
+ const auto percent = 100.0 * completed / goal;
+ os << percent << "% (" << completed << " out of " << goal << ")";
+ (void)os.precision(savedPrecision);
+ } else if (!completed && !goal) {
+ os << "nothing to do";
+ } else {
+ // unknown (i.e. negative) or buggy (i.e. zero when completed != 0) goal
+ os << completed;
+ }
+}
+
#include "fde.h"
-#include "StoreMetaUnpacker.h"
-#include "StoreMeta.h"
#include "Generic.h"
+#include "StoreMeta.h"
+#include "StoreMetaUnpacker.h"
struct InitStoreEntry : public unary_function<StoreMeta, void> {
InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
what->timestamp = tmp->timestamp;
what->lastref = tmp->lastref;
what->expires = tmp->expires;
- what->lastmod = tmp->lastmod;
+ what->lastModified(tmp->lastmod);
what->swap_file_sz = tmp->swap_file_sz;
what->refcount = tmp->refcount;
what->flags = tmp->flags;
};
bool
-storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf,
- StoreRebuildData &counts)
+storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &)
{
if (fd < 0)
return false;
bool
storeRebuildParseEntry(MemBuf &buf, StoreEntry &tmpe, cache_key *key,
- StoreRebuildData &counts,
+ StoreRebuildData &stats,
uint64_t expectedSize)
{
int swap_hdr_len = 0;
return false;
}
- if (!aBuilder.isBufferSane()) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring malformed cache entry.");
- return false;
- }
-
- StoreMeta *tlv_list = aBuilder.createStoreMeta();
- if (!tlv_list) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with invalid " <<
- "meta data");
+ StoreMeta *tlv_list = nullptr;
+ try {
+ tlv_list = aBuilder.createStoreMeta();
+ } catch (const std::exception &e) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring store entry because " << e.what());
return false;
}
+ assert(tlv_list);
// TODO: consume parsed metadata?
- debugs(47,7, HERE << "successful swap meta unpacking");
+ debugs(47,7, "successful swap meta unpacking; swap_file_sz=" << tmpe.swap_file_sz);
memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
InitStoreEntry visitor(&tmpe, key);
return false;
}
} else if (tmpe.swap_file_sz <= 0) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with " <<
- "unknown size: " << tmpe);
- return false;
+ // if caller cannot handle unknown sizes, it must check after the call.
+ debugs(47, 7, "unknown size: " << tmpe);
}
if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
- ++ counts.badflags;
+ ++ stats.badflags;
return false;
}
return true;
}
-bool
-storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key,
- StoreRebuildData &counts)
-{
- /* this needs to become
- * 1) unpack url
- * 2) make synthetic request with headers ?? or otherwise search
- * for a matching object in the store
- * TODO FIXME change to new async api
- * TODO FIXME I think there is a race condition here with the
- * async api :
- * store A reads in object foo, searchs for it, and finds nothing.
- * store B reads in object foo, searchs for it, finds nothing.
- * store A gets called back with nothing, so registers the object
- * store B gets called back with nothing, so registers the object,
- * which will conflict when the in core index gets around to scanning
- * store B.
- *
- * this suggests that rather than searching for duplicates, the
- * index rebuild should just assume its the most recent accurate
- * store entry and whoever indexes the stores handles duplicates.
- */
- if (StoreEntry *e = Store::Root().get(key)) {
-
- if (e->lastref >= tmpe.lastref) {
- /* key already exists, old entry is newer */
- /* keep old, ignore new */
- ++counts.dupcount;
-
- // For some stores, get() creates/unpacks a store entry. Signal
- // such stores that we will no longer use the get() result:
- e->lock();
- e->unlock();
-
- return false;
- } else {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- e->release(); /* release old entry */
- ++counts.dupcount;
- }
- }
-
- return true;
-}
projects / thirdparty / squid.git / blobdiff