/*
- * DEBUG: section 47 Store Directory Routines
- * AUTHOR: Robert Collins
+ * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
*
- * SQUID Web Proxy Cache http://www.squid-cache.org/
- * ----------------------------------------------------------
- *
- * Squid is the result of efforts by numerous individuals from
- * the Internet community; see the CONTRIBUTORS file for full
- * details. Many organizations have provided support for Squid's
- * development; see the SPONSORS file for full details. Squid is
- * Copyrighted (C) 2001 by the Regents of the University of
- * California; see the COPYRIGHT file for full details. Squid
- * incorporates software developed and/or copyrighted by other
- * sources; see the CREDITS file for full details.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
*/
+/* DEBUG: section 47 Store Directory Routines */
+
#define CLEAN_BUF_SZ 16384
#include "squid.h"
#include "cache_cf.h"
#include "ConfigOption.h"
-#include "disk.h"
#include "DiskIO/DiskIOModule.h"
#include "DiskIO/DiskIOStrategy.h"
#include "fde.h"
#include "FileMap.h"
+#include "fs_io.h"
#include "globals.h"
#include "Parsing.h"
#include "RebuildState.h"
#include "tools.h"
#include "UFSSwapDir.h"
-#if HAVE_MATH_H
-#include <math.h>
-#endif
+#include <cerrno>
+#include <cmath>
+#include <random>
#if HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
-#if HAVE_ERRNO_H
-#include <errno.h>
-#endif
int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0;
int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = NULL;
{
public:
- UFSCleanLog(SwapDir *);
- /** Get the next entry that is a candidate for clean log writing
- */
+ UFSCleanLog(SwapDir *aSwapDir) : sd(aSwapDir) {}
+
+ /// Get the next entry that is a candidate for clean log writing
virtual const StoreEntry *nextEntry();
- /** "write" an entry to the clean log file.
- */
+
+ /// "write" an entry to the clean log file.
virtual void write(StoreEntry const &);
- char *cur;
- char *newLog;
- char *cln;
- char *outbuf;
- off_t outbuf_offset;
- int fd;
- RemovalPolicyWalker *walker;
- SwapDir *sd;
-};
-UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) :
- cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL),
- outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir)
-{}
+ SBuf cur;
+ SBuf newLog;
+ SBuf cln;
+ char *outbuf = nullptr;
+ off_t outbuf_offset = 0;
+ int fd = -1;
+ RemovalPolicyWalker *walker = nullptr;
+ SwapDir *sd = nullptr;
+};
const StoreEntry *
UFSCleanLog::nextEntry()
s.timestamp = e.timestamp;
s.lastref = e.lastref;
s.expires = e.expires;
- s.lastmod = e.lastmod;
+ s.lastmod = e.lastModified();
s.swap_file_sz = e.swap_file_sz;
s.refcount = e.refcount;
s.flags = e.flags;
if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) {
+ int xerrno = errno;
/* XXX This error handling should probably move up to the caller */
- debugs(50, DBG_CRITICAL, HERE << newLog << ": write: " << xstrerror());
- debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced.");
+ debugs(50, DBG_CRITICAL, MYNAME << newLog << ": write: " << xstrerr(xerrno));
+ debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
file_close(fd);
fd = -1;
- unlink(newLog);
+ unlink(newLog.c_str());
sd->cleanLog = NULL;
delete this;
return;
delete anObject;
}
-static QS rev_int_sort;
static int
rev_int_sort(const void *A, const void *B)
{
IO->io = anIO;
/* Change the IO Options */
- if (currentIOOptions && currentIOOptions->options.size() > 2)
- delete currentIOOptions->options.pop_back();
+ if (currentIOOptions && currentIOOptions->options.size() > 2) {
+ delete currentIOOptions->options.back();
+ currentIOOptions->options.pop_back();
+ }
/* TODO: factor out these 4 lines */
ConfigOption *ioOptions = IO->io->getOptionTree();
/* silently ignore this */
return true;
- if (!value)
+ if (!value) {
self_destruct();
+ return false;
+ }
DiskIOModule *module = DiskIOModule::Find(value);
- if (!module)
+ if (!module) {
self_destruct();
+ return false;
+ }
changeIO(module);
started_clean_event = 1;
}
- (void) storeDirGetBlkSize(path, &fs.blksize);
+ (void) fsBlockSize(path, &fs.blksize);
}
void
createSwapSubDirs();
}
-Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0)
+Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
+ SwapDir(aType),
+ IO(NULL),
+ fsdata(NULL),
+ map(new FileMap()),
+ suggest(0),
+ l1(16),
+ l2(256),
+ swaplog_fd(-1),
+ currentIOOptions(new ConfigOptionVector()),
+ ioType(xstrdup(anIOType)),
+ cur_size(0),
+ n_disk_objects(0),
+ rebuilding_(false)
{
/* modulename is only set to disk modules that are built, by configure,
* so the Find call should never return NULL here.
file_close(swaplog_fd);
swaplog_fd = -1;
}
- safe_free(ioType);
+ xfree(ioType);
delete map;
delete IO;
delete currentIOOptions;
storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
map->numFilesInMap(), map->capacity(),
Math::intPercent(map->numFilesInMap(), map->capacity()));
- x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
+ x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
if (0 == x) {
storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
void
Fs::Ufs::UFSSwapDir::maintain()
{
- /* We can't delete objects while rebuilding swap */
+ /* TODO: possible options for improvement;
+ *
+ * Note that too much aggression here is not good. It means that disk
+ * controller is getting a long queue of removals to act on, along
+ * with its regular I/O queue, and that client traffic is 'paused'
+ * and growing the network I/O queue as well while the scan happens.
+ * Possibly bad knock-on effects as Squid catches up on all that.
+ *
+ * Bug 2448 may have been a sign of what can wrong. At the least it
+ * provides a test case for aggression effects in overflow conditions.
+ *
+ * - base removal limit on space saved, instead of count ?
+ *
+ * - base removal rate on a traffic speed counter ?
+ * as the purge took up more time out of the second it would grow to
+ * a graceful full pause
+ *
+ * - pass out a value to cause another event to be scheduled immediately
+ * instead of waiting a whole second more ?
+ * knock on; schedule less if all caches are under low-water
+ *
+ * - admin configurable removal rate or count ?
+ * the current numbers are arbitrary, config helps with experimental
+ * trials and future-proofing the install base.
+ * we also have this indirectly by shifting the relative positions
+ * of low-, high- water and the total capacity limit.
+ */
- /* XXX FIXME each store should start maintaining as it comes online. */
+ // minSize() is swap_low_watermark in bytes
+ const uint64_t lowWaterSz = minSize();
- if (StoreController::store_dirs_rebuilding)
+ if (currentSize() < lowWaterSz) {
+ debugs(47, 5, "space still available in " << path);
return;
+ }
- StoreEntry *e = NULL;
-
- int removed = 0;
-
- RemovalPurgeWalker *walker;
-
- double f = (double) (currentSize() - minSize()) / (maxSize() - minSize());
+ /* We can't delete objects while rebuilding swap */
+ /* XXX each store should start maintaining as it comes online. */
+ if (StoreController::store_dirs_rebuilding) {
+ // suppress the warnings, except once each minute
+ static int64_t lastWarn = 0;
+ int warnLevel = 3;
+ if (lastWarn+60 < squid_curtime) {
+ lastWarn = squid_curtime;
+ warnLevel = DBG_IMPORTANT;
+ }
+ debugs(47, warnLevel, StoreController::store_dirs_rebuilding << " cache_dir still rebuilding. Skip GC for " << path);
+ return;
+ }
- f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
+ // maxSize() is cache_dir total size in bytes
+ const uint64_t highWaterSz = ((maxSize() * Config.Swap.highWaterMark) / 100);
+
+ // f is percentage of 'gap' filled between low- and high-water.
+ // Used to reduced purge rate when between water markers, and
+ // to multiply it more agressively the further above high-water
+ // it reaches. But in a graceful linear growth curve.
+ double f = 1.0;
+ if (highWaterSz > lowWaterSz) {
+ // might be equal. n/0 is bad.
+ f = (double) (currentSize() - lowWaterSz) / (highWaterSz - lowWaterSz);
+ }
+ // how deep to look for a single object that can be removed
int max_scan = (int) (f * 400.0 + 100.0);
- int max_remove = (int) (f * 70.0 + 10.0);
+ // try to purge only this many objects this cycle.
+ int max_remove = (int) (f * 300.0 + 20.0);
/*
* This is kinda cheap, but so we need this priority hack?
*/
+ debugs(47, 3, "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove);
- debugs(47, 3, HERE << "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove );
+ RemovalPurgeWalker *walker = repl->PurgeInit(repl, max_scan);
- walker = repl->PurgeInit(repl, max_scan);
-
- while (1) {
- if (currentSize() < minSize())
- break;
+ int removed = 0;
+ // only purge while above low-water
+ while (currentSize() >= lowWaterSz) {
+ // stop if we reached max removals for this cycle,
+ // Bug 2448 may be from this not clearing enough,
+ // but it predates the current algorithm so not sure
if (removed >= max_remove)
break;
- e = walker->Next(walker);
+ StoreEntry *e = walker->Next(walker);
+ // stop if all objects are locked / in-use,
+ // or the cache is empty
if (!e)
break; /* no more objects */
++removed;
- e->release();
+ e->release(true);
}
walker->Done(walker);
- debugs(47, (removed ? 2 : 3), HERE << path <<
+ debugs(47, (removed ? 2 : 3), path <<
" removed " << removed << "/" << max_remove << " f=" <<
std::setprecision(4) << f << " max_scan=" << max_scan);
+
+ // what if cache is still over the high watermark ?
+ // Store::Maintain() schedules another purge in 1 second.
}
void
}
bool
-Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e, bool)
+Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e)
{
debugs(47, 3, HERE << "dereferencing " << &e << " " <<
e.swap_dirn << "/" << e.swap_filen);
debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
created = 1;
} else {
- fatalf("Failed to make swap directory %s: %s",
- aPath, xstrerror());
+ int xerrno = errno;
+ fatalf("Failed to make swap directory %s: %s", aPath, xstrerr(xerrno));
}
return created;
struct stat sb;
if (::stat(aPath, &sb) < 0) {
- debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerror());
+ int xerrno = errno;
+ debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerr(xerrno));
return false;
}
}
}
-char *
+SBuf
Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
{
- LOCAL_ARRAY(char, lpath, MAXPATHLEN);
- LOCAL_ARRAY(char, pathtmp, MAXPATHLEN);
- LOCAL_ARRAY(char, digit, 32);
- char *pathtmp2;
+ SBuf lpath;
if (Config.Log.swap) {
- xstrncpy(pathtmp, path, MAXPATHLEN - 64);
- pathtmp2 = pathtmp;
+ static char pathtmp[MAXPATHLEN];
+ char *pathtmp2 = xstrncpy(pathtmp, path, MAXPATHLEN - 64);
- while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
+ // replace all '/' with '.'
+ while ((pathtmp2 = strchr(pathtmp2, '/')))
*pathtmp2 = '.';
- while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
- pathtmp[strlen(pathtmp) - 1] = '\0';
+ // remove any trailing '.' characters
+ int pos = strlen(pathtmp);
+ while (pos && pathtmp[pos-1] == '.')
+ pathtmp[--pos] = '\0';
+ // remove any prefix '.' characters
for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
- snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
-
- if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) {
- strcat(lpath, ".");
- snprintf(digit, 32, "%02d", index);
- strncat(lpath, digit, 3);
+ // replace a '%s' (if any) in the config string
+ // with the resulting pathtmp2 string
+ lpath.appendf(Config.Log.swap, pathtmp2);
+
+ // is pathtmp2 was NOT injected, append numeric file extension
+ if (lpath.cmp(Config.Log.swap) == 0) {
+ lpath.append(".", 1);
+ lpath.appendf("%02d", index);
}
} else {
- xstrncpy(lpath, path, MAXPATHLEN - 64);
- strcat(lpath, "/swap.state");
+ lpath.append(path);
+ lpath.append("/swap.state", 11);
}
- if (ext)
- strncat(lpath, ext, 16);
+ lpath.append(ext); // may be nil, that is okay.
return lpath;
}
void
Fs::Ufs::UFSSwapDir::openLog()
{
- char *logPath;
- logPath = logFile();
- swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY);
+ assert(NumberOfUFSDirs || !UFSDirToGlobalDirMapping);
+ ++NumberOfUFSDirs;
+ assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
+
+ if (rebuilding_) { // we did not close the temporary log used for rebuilding
+ assert(swaplog_fd >= 0);
+ return;
+ }
+
+ SBuf logPath(logFile());
+ swaplog_fd = file_open(logPath.c_str(), O_WRONLY | O_CREAT | O_BINARY);
if (swaplog_fd < 0) {
- debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerror());
+ int xerrno = errno;
+ debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerr(xerrno));
fatal("UFSSwapDir::openLog: Failed to open swap log.");
}
debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
-
- if (0 == NumberOfUFSDirs)
- assert(NULL == UFSDirToGlobalDirMapping);
-
- ++NumberOfUFSDirs;
-
- assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
}
void
if (swaplog_fd < 0) /* not open */
return;
+ --NumberOfUFSDirs;
+ assert(NumberOfUFSDirs >= 0);
+ if (!NumberOfUFSDirs)
+ safe_free(UFSDirToGlobalDirMapping);
+
+ if (rebuilding_) // we cannot close the temporary log used for rebuilding
+ return;
+
file_close(swaplog_fd);
debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
swaplog_fd = -1;
-
- --NumberOfUFSDirs;
-
- assert(NumberOfUFSDirs >= 0);
-
- if (0 == NumberOfUFSDirs)
- safe_free(UFSDirToGlobalDirMapping);
}
bool
time_t lastmod,
uint32_t refcount,
uint16_t newFlags,
- int clean)
+ int)
{
StoreEntry *e = NULL;
debugs(47, 5, HERE << storeKeyText(key) <<
e = new StoreEntry();
e->store_status = STORE_OK;
e->setMemStatus(NOT_IN_MEMORY);
- e->swap_status = SWAPOUT_DONE;
- e->swap_filen = file_number;
- e->swap_dirn = index;
+ e->attachToDisk(index, file_number, SWAPOUT_DONE);
e->swap_file_sz = swap_file_sz;
e->lastref = lastref;
e->timestamp = timestamp;
e->expires = expires;
- e->lastmod = lastmod;
+ e->lastModified(lastmod);
e->refcount = refcount;
e->flags = newFlags;
- EBIT_CLR(e->flags, RELEASE_REQUEST);
- EBIT_CLR(e->flags, KEY_PRIVATE);
e->ping_status = PING_NONE;
EBIT_CLR(e->flags, ENTRY_VALIDATED);
mapBitSet(e->swap_filen);
cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
++n_disk_objects;
- e->hashInsert(key); /* do it after we clear KEY_PRIVATE */
+ e->hashInsert(key);
replacementAdd (e);
return e;
}
-void
-Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e)
-{
- debugs(47, 5, HERE << *e);
- replacementRemove(e); // checks swap_dirn so do it before we invalidate it
- // Do not unlink the file as it might be used by a subsequent entry.
- mapBitReset(e->swap_filen);
- e->swap_filen = -1;
- e->swap_dirn = -1;
- cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz);
- --n_disk_objects;
-}
-
void
Fs::Ufs::UFSSwapDir::rebuild()
{
void
Fs::Ufs::UFSSwapDir::closeTmpSwapLog()
{
- char *swaplog_path = xstrdup(logFile(NULL)); // where the swaplog should be
- char *tmp_path = xstrdup(logFile(".new")); // the temporary file we have generated
- int fd;
+ assert(rebuilding_);
+ rebuilding_ = false;
+
+ SBuf swaplog_path(logFile()); // where the swaplog should be
+ SBuf tmp_path(logFile(".new"));
+
file_close(swaplog_fd);
- if (xrename(tmp_path, swaplog_path) < 0) {
- fatalf("Failed to rename log file %s to %s", tmp_path, swaplog_path);
+ if (!FileRename(tmp_path, swaplog_path)) {
+ fatalf("Failed to rename log file " SQUIDSBUFPH " to " SQUIDSBUFPH, SQUIDSBUFPRINT(tmp_path), SQUIDSBUFPRINT(swaplog_path));
}
- fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
+ int fd = file_open(swaplog_path.c_str(), O_WRONLY | O_CREAT | O_BINARY);
if (fd < 0) {
- debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror());
- fatalf("Failed to open swap log %s", swaplog_path);
+ int xerrno = errno;
+ debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerr(xerrno));
+ fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
}
- xfree(swaplog_path);
- xfree(tmp_path);
swaplog_fd = fd;
debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
}
FILE *
Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
{
- char *swaplog_path = xstrdup(logFile(NULL));
- char *clean_path = xstrdup(logFile(".last-clean"));
- char *new_path = xstrdup(logFile(".new"));
+ assert(!rebuilding_);
+
+ SBuf swaplog_path(logFile());
+ SBuf clean_path(logFile(".last-clean"));
+ SBuf new_path(logFile(".new"));
struct stat log_sb;
struct stat clean_sb;
- FILE *fp;
- int fd;
- if (::stat(swaplog_path, &log_sb) < 0) {
+ if (::stat(swaplog_path.c_str(), &log_sb) < 0) {
debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
- safe_free(swaplog_path);
- safe_free(clean_path);
- safe_free(new_path);
return NULL;
}
file_close(swaplog_fd);
/* open a write-only FD for the new log */
- fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
-
+ int fd = file_open(new_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
if (fd < 0) {
- debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerror());
- fatalf("Failed to open swap log %s", new_path);
+ int xerrno = errno;
+ debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerr(xerrno));
+ fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(new_path));
}
swaplog_fd = fd;
+ rebuilding_ = true;
{
const StoreSwapLogHeader header;
}
/* open a read-only stream of the old log */
- fp = fopen(swaplog_path, "rb");
-
- if (fp == NULL) {
- debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerror());
- fatalf("Failed to open swap log for reading %s", swaplog_path);
+ FILE *fp = fopen(swaplog_path.c_str(), "rb");
+ if (!fp) {
+ int xerrno = errno;
+ debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerr(xerrno));
+ fatalf("Failed to open swap log for reading " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
}
memset(&clean_sb, '\0', sizeof(struct stat));
- if (::stat(clean_path, &clean_sb) < 0)
+ if (::stat(clean_path.c_str(), &clean_sb) < 0)
*clean_flag = 0;
else if (clean_sb.st_mtime < log_sb.st_mtime)
*clean_flag = 0;
else
*clean_flag = 1;
- safeunlink(clean_path, 1);
-
- safe_free(swaplog_path);
-
- safe_free(clean_path);
-
- safe_free(new_path);
+ safeunlink(clean_path.c_str(), 1);
return fp;
}
#endif
cleanLog = NULL;
- state->newLog = xstrdup(logFile(".clean"));
- state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
+ state->cur = logFile();
+ state->newLog = logFile(".clean");
+ state->fd = file_open(state->newLog.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
if (state->fd < 0) {
- xfree(state->newLog);
delete state;
return -1;
}
- state->cur = xstrdup(logFile(NULL));
- state->cln = xstrdup(logFile(".last-clean"));
+ state->cln = state->cur;
+ state->cln.append(".last-clean");
state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
state->outbuf_offset = 0;
/*copy the header */
state->outbuf_offset += header.record_size;
state->walker = repl->WalkInit(repl);
- ::unlink(state->cln);
+ ::unlink(state->cln.c_str());
debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd);
#if HAVE_FCHMOD
- if (::stat(state->cur, &sb) == 0)
+ if (::stat(state->cur.c_str(), &sb) == 0)
fchmod(state->fd, sb.st_mode);
#endif
state->walker->Done(state->walker);
if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
- debugs(50, DBG_CRITICAL, HERE << state->newLog << ": write: " << xstrerror());
- debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced.");
+ int xerrno = errno;
+ debugs(50, DBG_CRITICAL, MYNAME << state->newLog << ": write: " << xstrerr(xerrno));
+ debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
file_close(state->fd);
state->fd = -1;
- ::unlink(state->newLog);
+ ::unlink(state->newLog.c_str());
}
safe_free(state->outbuf);
state->fd = -1;
#endif
- xrename(state->newLog, state->cur);
+ FileRename(state->newLog, state->cur);
+ // TODO handle rename errors
}
/* touch a timestamp file if we're not still validating */
else if (fd < 0)
(void) 0;
else
- file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
+ file_close(file_open(state->cln.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
/* close */
- safe_free(state->cur);
-
- safe_free(state->newLog);
-
- safe_free(state->cln);
-
if (state->fd >= 0)
file_close(state->fd);
cleanLog = NULL;
}
-void
-Fs::Ufs::UFSSwapDir::CleanEvent(void *unused)
+/// safely cleans a few unused files if possible
+int
+Fs::Ufs::UFSSwapDir::HandleCleanEvent()
{
static int swap_index = 0;
int i;
int j = 0;
int n = 0;
- /*
- * Assert that there are UFS cache_dirs configured, otherwise
- * we should never be called.
- */
- assert(NumberOfUFSDirs);
+
+ if (!NumberOfUFSDirs)
+ return 0; // probably in the middle of reconfiguration
if (NULL == UFSDirToGlobalDirMapping) {
SwapDir *sd;
* value. j equals the total number of UFS level 2
* swap directories
*/
- swap_index = (int) (squid_random() % j);
+ std::mt19937 mt(static_cast<uint32_t>(getCurrentTime() & 0xFFFFFFFF));
+ xuniform_int_distribution<> dist(0, j);
+ swap_index = dist(mt);
}
/* if the rebuild is finished, start cleaning directories. */
++swap_index;
}
+ return n;
+}
+
+void
+Fs::Ufs::UFSSwapDir::CleanEvent(void *)
+{
+ const int n = HandleCleanEvent();
eventAdd("storeDirClean", CleanEvent, NULL,
15.0 * exp(-0.25 * n), 1);
}
}
void
-Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e)
+Fs::Ufs::UFSSwapDir::evictCached(StoreEntry & e)
{
- debugs(79, 3, HERE << "dirno " << index << ", fileno "<<
- std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
- if (e.swap_status == SWAPOUT_DONE) {
+ debugs(79, 3, e);
+ if (e.locked()) // somebody else may still be using this file
+ return; // nothing to do: our get() always returns nil
+
+ if (!e.hasDisk())
+ return; // see evictIfFound()
+
+ // Since these fields grow only after swap out ends successfully,
+ // do not decrement them for e.swappingOut() and e.swapoutFailed().
+ if (e.swappedOut()) {
cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
--n_disk_objects;
}
replacementRemove(&e);
mapBitReset(e.swap_filen);
UFSSwapDir::unlinkFile(e.swap_filen);
+ e.detachFromDisk();
+}
+
+void
+Fs::Ufs::UFSSwapDir::evictIfFound(const cache_key *)
+{
+ // UFS disk entries always have (attached) StoreEntries so if we got here,
+ // the entry is not cached on disk and there is nothing for us to do.
}
void
void
Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e)
{
- StorePointer SD;
-
- if (e->swap_dirn < 0)
- return;
+ assert(e->hasDisk());
- SD = INDEXSD(e->swap_dirn);
+ SwapDirPointer SD = INDEXSD(e->swap_dirn);
assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
}
void
-Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e)
+Fs::Ufs::UFSSwapDir::finalizeSwapoutSuccess(const StoreEntry &e)
{
cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
++n_disk_objects;
}
-StoreSearch *
-Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *request)
+void
+Fs::Ufs::UFSSwapDir::finalizeSwapoutFailure(StoreEntry &entry)
{
- if (url.size())
- fatal ("Cannot search by url yet\n");
-
- return new Fs::Ufs::StoreSearchUFS (this);
+ debugs(47, 5, entry);
+ // rely on the expected eventual StoreEntry::release(), evictCached(), or
+ // a similar call to call unlink(), detachFromDisk(), etc. for the entry.
}
void
Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const
{
+ if (swaplog_fd < 0) {
+ debugs(36, 5, "cannot log " << e << " in the middle of reconfiguration");
+ return;
+ }
+
StoreSwapLogData *s = new StoreSwapLogData;
s->op = (char) op;
s->swap_filen = e.swap_filen;
s->timestamp = e.timestamp;
s->lastref = e.lastref;
s->expires = e.expires;
- s->lastmod = e.lastmod;
+ s->lastmod = e.lastModified();
s->swap_file_sz = e.swap_file_sz;
s->refcount = e.refcount;
s->flags = e.flags;
Fs::Ufs::UFSSwapDir::DirClean(int swap_index)
{
DIR *dir_pointer = NULL;
-
- LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
- LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
-
int files[20];
int swapfileno;
int fn; /* same as swapfileno, but with dirn bits set */
D1 = (swap_index / N0) % N1;
N2 = SD->l2;
D2 = ((swap_index / N0) / N1) % N2;
- snprintf(p1, MAXPATHLEN, "%s/%02X/%02X",
- SD->path, D1, D2);
+
+ SBuf p1;
+ p1.appendf("%s/%02X/%02X", SD->path, D1, D2);
debugs(36, 3, HERE << "Cleaning directory " << p1);
- dir_pointer = opendir(p1);
+ dir_pointer = opendir(p1.c_str());
- if (dir_pointer == NULL) {
- if (errno == ENOENT) {
- debugs(36, DBG_CRITICAL, HERE << "WARNING: Creating " << p1);
- if (mkdir(p1, 0777) == 0)
+ if (!dir_pointer) {
+ int xerrno = errno;
+ if (xerrno == ENOENT) {
+ debugs(36, DBG_CRITICAL, MYNAME << "WARNING: Creating " << p1);
+ if (mkdir(p1.c_str(), 0777) == 0)
return 0;
}
- debugs(50, DBG_CRITICAL, HERE << p1 << ": " << xstrerror());
- safeunlink(p1, 1);
+ debugs(50, DBG_CRITICAL, MYNAME << p1 << ": " << xstrerr(xerrno));
+ safeunlink(p1.c_str(), 1);
return 0;
}
for (n = 0; n < k; ++n) {
debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
- snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
- safeunlink(p2, 0);
+ SBuf p2(p1);
+ p2.appendf("/%08X", files[n]);
+ safeunlink(p2.c_str(), 0);
++statCounter.swap.files_cleaned;
}
debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1);
return k;
}
+
projects / thirdparty / squid.git / blobdiff