/*
- * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
+ * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
*
* Squid software is distributed under GPLv2+ license and includes
* contributions from numerous individuals and organizations.
#include "SquidTime.h"
#include "StatCounters.h"
#include "Store.h"
+#include "store/Disk.h"
#include "store_digest.h"
#include "store_key_md5.h"
#include "store_rebuild.h"
#include "StoreSearch.h"
-#include "SwapDir.h"
// for tvSubDsec() which should be in SquidTime.h
#include "util.h"
static StoreRebuildData counts;
-static struct timeval rebuild_start;
static void storeCleanup(void *);
+// TODO: Either convert to Progress or replace with StoreRebuildData.
+// TODO: Handle unknown totals (UFS cache_dir that lost swap.state) correctly.
typedef struct {
/* total number of "swap.state" entries that will be read */
int total;
static store_rebuild_progress *RebuildProgress = NULL;
-static int
-storeCleanupDoubleCheck(StoreEntry * e)
+void
+StoreRebuildData::updateStartTime(const timeval &dirStartTime)
{
- SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
- return (SD->doubleCheck(*e));
+ startTime = started() ? std::min(startTime, dirStartTime) : dirStartTime;
}
static void
-storeCleanup(void *datanotused)
+storeCleanup(void *)
{
static int store_errors = 0;
static StoreSearchPointer currentSearch;
static int seen = 0;
if (currentSearch == NULL || currentSearch->isDone())
- currentSearch = Store::Root().search(NULL, NULL);
+ currentSearch = Store::Root().search();
size_t statCount = 500;
* Calling StoreEntry->release() has no effect because we're
* still in 'store_rebuilding' state
*/
- if (e->swap_filen < 0)
+ if (!e->hasDisk())
continue;
if (opt_store_doublecheck)
- if (storeCleanupDoubleCheck(e))
+ if (e->disk().doubleCheck(*e))
++store_errors;
EBIT_SET(e->flags, ENTRY_VALIDATED);
storeRebuildComplete(StoreRebuildData *dc)
{
- double dt;
- counts.objcount += dc->objcount;
- counts.expcount += dc->expcount;
- counts.scancount += dc->scancount;
- counts.clashcount += dc->clashcount;
- counts.dupcount += dc->dupcount;
- counts.cancelcount += dc->cancelcount;
- counts.invalid += dc->invalid;
- counts.badflags += dc->badflags;
- counts.bad_log_op += dc->bad_log_op;
- counts.zero_object_sz += dc->zero_object_sz;
+ if (dc) {
+ counts.objcount += dc->objcount;
+ counts.expcount += dc->expcount;
+ counts.scancount += dc->scancount;
+ counts.clashcount += dc->clashcount;
+ counts.dupcount += dc->dupcount;
+ counts.cancelcount += dc->cancelcount;
+ counts.invalid += dc->invalid;
+ counts.badflags += dc->badflags;
+ counts.bad_log_op += dc->bad_log_op;
+ counts.zero_object_sz += dc->zero_object_sz;
+ counts.validations += dc->validations;
+ counts.updateStartTime(dc->startTime);
+ }
+ // else the caller was not responsible for indexing its cache_dir
+
+ assert(StoreController::store_dirs_rebuilding > 1);
+ --StoreController::store_dirs_rebuilding;
+
/*
* When store_dirs_rebuilding == 1, it means we are done reading
* or scanning all cache_dirs. Now report the stats and start
if (StoreController::store_dirs_rebuilding > 1)
return;
- dt = tvSubDsec(rebuild_start, current_time);
+ const auto dt = tvSubDsec(counts.startTime, current_time);
debugs(20, DBG_IMPORTANT, "Finished rebuilding storage from disk.");
debugs(20, DBG_IMPORTANT, " " << std::setw(7) << counts.scancount << " Entries scanned");
void
storeRebuildStart(void)
{
- memset(&counts, '\0', sizeof(counts));
- rebuild_start = current_time;
+ counts = StoreRebuildData(); // reset counters
/*
* Note: store_dirs_rebuilding is initialized to 1.
*
storeRebuildProgress(int sd_index, int total, int sofar)
{
static time_t last_report = 0;
+ // TODO: Switch to int64_t and fix handling of unknown totals.
double n = 0.0;
double d = 0.0;
d += (double) RebuildProgress[sd_index].total;
}
- debugs(20, DBG_IMPORTANT, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
+ debugs(20, DBG_IMPORTANT, "Indexing cache entries: " << Progress(n, d));
last_report = squid_curtime;
}
+void
+Progress::print(std::ostream &os) const
+{
+ if (goal > 0) {
+ const auto savedPrecision = os.precision(2);
+ const auto percent = 100.0 * completed / goal;
+ os << percent << "% (" << completed << " out of " << goal << ")";
+ (void)os.precision(savedPrecision);
+ } else if (!completed && !goal) {
+ os << "nothing to do";
+ } else {
+ // unknown (i.e. negative) or buggy (i.e. zero when completed != 0) goal
+ os << completed;
+ }
+}
+
#include "fde.h"
#include "Generic.h"
#include "StoreMeta.h"
what->timestamp = tmp->timestamp;
what->lastref = tmp->lastref;
what->expires = tmp->expires;
- what->lastmod = tmp->lastmod;
+ what->lastModified(tmp->lastmod);
what->swap_file_sz = tmp->swap_file_sz;
what->refcount = tmp->refcount;
what->flags = tmp->flags;
return false;
}
- if (!aBuilder.isBufferSane()) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring malformed cache entry.");
- return false;
- }
-
- StoreMeta *tlv_list = aBuilder.createStoreMeta();
- if (!tlv_list) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring cache entry with invalid " <<
- "meta data");
+ StoreMeta *tlv_list = nullptr;
+ try {
+ tlv_list = aBuilder.createStoreMeta();
+ } catch (const std::exception &e) {
+ debugs(47, DBG_IMPORTANT, "WARNING: Ignoring store entry because " << e.what());
return false;
}
+ assert(tlv_list);
// TODO: consume parsed metadata?
return true;
}
-bool
-storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key, StoreRebuildData &stats)
-{
- /* this needs to become
- * 1) unpack url
- * 2) make synthetic request with headers ?? or otherwise search
- * for a matching object in the store
- * TODO FIXME change to new async api
- * TODO FIXME I think there is a race condition here with the
- * async api :
- * store A reads in object foo, searchs for it, and finds nothing.
- * store B reads in object foo, searchs for it, finds nothing.
- * store A gets called back with nothing, so registers the object
- * store B gets called back with nothing, so registers the object,
- * which will conflict when the in core index gets around to scanning
- * store B.
- *
- * this suggests that rather than searching for duplicates, the
- * index rebuild should just assume its the most recent accurate
- * store entry and whoever indexes the stores handles duplicates.
- */
- if (StoreEntry *e = Store::Root().get(key)) {
-
- if (e->lastref >= tmpe.lastref) {
- /* key already exists, old entry is newer */
- /* keep old, ignore new */
- ++stats.dupcount;
-
- // For some stores, get() creates/unpacks a store entry. Signal
- // such stores that we will no longer use the get() result:
- e->lock("storeRebuildKeepEntry");
- e->unlock("storeRebuildKeepEntry");
-
- return false;
- } else {
- /* URL already exists, this swapfile not being used */
- /* junk old, load new */
- e->release(); /* release old entry */
- ++stats.dupcount;
- }
- }
-
- return true;
-}
-
projects / thirdparty / squid.git / blobdiff