From: Francesco Chemolli Date: Fri, 10 Aug 2012 06:56:49 +0000 (+0200) Subject: SourceLayout: introduce Fs::Ufs namespace, split ufscommon, store_dir_ufs and store_i... X-Git-Tag: sourceformat-review-1~120 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=58373ff8b8ac94bf427cc47c65dbb9c21f560a36;p=thirdparty%2Fsquid.git SourceLayout: introduce Fs::Ufs namespace, split ufscommon, store_dir_ufs and store_io_ufs --- diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index 679873f25c..444d35c505 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -22,11 +22,18 @@ libcoss_la_SOURCES = \ libufs_la_SOURCES = \ ufs/StoreFSufs.h \ ufs/StoreFSufs.cc \ - ufs/store_dir_ufs.cc \ - ufs/store_io_ufs.cc \ - ufs/ufscommon.cci \ - ufs/ufscommon.cc \ - ufs/ufscommon.h + ufs/UFSStoreState.cc \ + ufs/UFSSwapDir.cc \ + ufs/UFSSwapDir.h \ + ufs/UFSStrategy.cc \ + ufs/UFSStrategy.h \ + ufs/UFSStoreState.h \ + ufs/StoreSearchUFS.h \ + ufs/StoreSearchUFS.cc \ + ufs/UFSSwapLogParser.h \ + ufs/UFSSwapLogParser.cc \ + ufs/RebuildState.h \ + ufs/RebuildState.cc librock_la_SOURCES = \ rock/RockDbCell.h \ diff --git a/src/fs/Module.cc b/src/fs/Module.cc index 5c900b5dbf..5175e08e6f 100644 --- a/src/fs/Module.cc +++ b/src/fs/Module.cc @@ -2,7 +2,7 @@ #include "Module.h" #if defined(HAVE_FS_UFS) || defined(HAVE_FS_AUFS) || defined(HAVE_FS_DISKD) #include "fs/ufs/StoreFSufs.h" -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" #endif #if HAVE_FS_COSS @@ -10,16 +10,16 @@ #endif #if HAVE_FS_UFS -static StoreFSufs *UfsInstance; +static Fs::Ufs::StoreFSufs *UfsInstance; #endif #if HAVE_FS_AUFS -static StoreFSufs *AufsInstance; +static Fs::Ufs::StoreFSufs *AufsInstance; #endif #if HAVE_FS_DISKD -static StoreFSufs *DiskdInstance; +static Fs::Ufs::StoreFSufs *DiskdInstance; #endif #if HAVE_FS_ROCK @@ -42,16 +42,16 @@ void Fs::Init() { #if HAVE_FS_UFS - UfsInstance = new StoreFSufs("Blocking", "ufs"); + UfsInstance = new Fs::Ufs::StoreFSufs("Blocking", "ufs"); #endif #if HAVE_FS_AUFS - AufsInstance = new StoreFSufs("DiskThreads", "aufs");; + AufsInstance = new Fs::Ufs::StoreFSufs("DiskThreads", "aufs");; #endif #if HAVE_FS_DISKD - DiskdInstance = new StoreFSufs("DiskDaemon", "diskd");; + DiskdInstance = new Fs::Ufs::StoreFSufs("DiskDaemon", "diskd");; #endif #if HAVE_FS_ROCK diff --git a/src/fs/aufs/StoreFSaufs.cc b/src/fs/aufs/StoreFSaufs.cc index 39d43f6a8f..2fe6d16e94 100644 --- a/src/fs/aufs/StoreFSaufs.cc +++ b/src/fs/aufs/StoreFSaufs.cc @@ -45,8 +45,7 @@ #include "fs/ufs/StoreFSufs.h" -/** \todo FIXME: break UFSSwapDir out so we don't need all the guff */ -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" /** \defgroup AUFS AUFS Storage Filesystem (UFS Based) @@ -54,4 +53,3 @@ */ /* Unused variable: */ -StoreFSufs *AufsInstance_foo = NULL; diff --git a/src/fs/diskd/StoreFSdiskd.cc b/src/fs/diskd/StoreFSdiskd.cc index 1c375f55da..01d20c87c8 100644 --- a/src/fs/diskd/StoreFSdiskd.cc +++ b/src/fs/diskd/StoreFSdiskd.cc @@ -42,9 +42,7 @@ #endif #include "fs/ufs/StoreFSufs.h" - -/** \todo FIXME: break UFSSwapDir out so we don;t need all the extras */ -#include "fs/ufs/ufscommon.h" +#include "fs/ufs/UFSSwapDir.h" /** \defgroup diskd diskd Storage Filesystem (UFS Based) @@ -52,4 +50,4 @@ */ /* Unused variable: */ -StoreFSufs *DiskdInstance_foo = NULL; +Fs::Ufs::StoreFSufs *DiskdInstance_foo = NULL; diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc index d624ec7b01..51114f22a1 100644 --- a/src/fs/rock/RockSwapDir.cc +++ b/src/fs/rock/RockSwapDir.cc @@ -18,6 +18,7 @@ #include "MemObject.h" #include "Parsing.h" #include "SquidMath.h" +#include #include const int64_t Rock::SwapDir::HeaderSize = 16*1024; diff --git a/src/fs/ufs/ufscommon.cc b/src/fs/ufs/RebuildState.cc similarity index 53% rename from src/fs/ufs/ufscommon.cc rename to src/fs/ufs/RebuildState.cc index c4af21183a..9219dc595b 100644 --- a/src/fs/ufs/ufscommon.cc +++ b/src/fs/ufs/RebuildState.cc @@ -1,5 +1,4 @@ /* - * $Id$ * * DEBUG: section 47 Store Directory Routines * AUTHOR: Robert Collins @@ -30,313 +29,18 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * - * Copyright (c) 2003, Robert Collins */ #include "squid.h" -#include "ufscommon.h" -#include "Store.h" -#include "fde.h" +#include "RebuildState.h" #include "SquidTime.h" -#include "StoreMeta.h" -#include "Generic.h" -#include "StoreMetaUnpacker.h" -#include "RefCount.h" #include "StoreSwapLogData.h" -#include "swap_log_op.h" +#include "UFSSwapLogParser.h" +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,RebuildState); -CBDATA_CLASS_INIT(RebuildState); - -/// Parse a swap header entry created on a system with 32-bit size_t and sfileno -/// this is typical of 32-bit systems without large file support -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_32bs:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) - /// time_t an sfileno have no variation from the v1 baseline format - struct StoreSwapLogDataOld { - char op; - sfileno swap_filen; - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - uint32_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_32bs(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); - } - /// Convert the on-disk 32-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - swapData.swap_filen = readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -#if UNUSED_CODE -/// Parse a swap header entry created on a system with 32-bit size_t, time_t and sfileno -/// this is typical of 32-bit systems without large file support and with old kernels -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_32bst:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) - /// time_t also differs - /// sfileno has no variation from the v1 baseline format - struct StoreSwapLogDataOld { - char op; - sfileno swap_filen; - int32_t timestamp; - int32_t lastref; - int32_t expires; - int32_t lastmod; - uint32_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_32bst(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld); - } - /// Convert the on-disk 32-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - swapData.swap_filen = readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -/// Parse a swap header entry created on a system with 64-bit size_t and sfileno -/// this is typical of 64-bit systems prior to this patch fixing sfileno to 32-bits -/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. -class UFSSwapLogParser_v1_64bfn:public UFSSwapLogParser -{ -public: - /// version 1 cache swap.state entry with 64-bit sfileno - struct StoreSwapLogDataOld { - char op; - int64_t swap_filen; - time_t timestamp; - time_t lastref; - time_t expires; - time_t lastmod; - uint64_t swap_file_sz; - uint16_t refcount; - uint16_t flags; - unsigned char key[SQUID_MD5_DIGEST_LENGTH]; - }; - UFSSwapLogParser_v1_64bfn(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld); - } - /// Convert the on-disk 64-bit format to our current format while reading - bool ReadRecord(StoreSwapLogData &swapData) { - UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld readData; - int bytes = sizeof(UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld); - - assert(log); - - if (fread(&readData, bytes, 1, log) != 1) { - return false; - } - swapData.op = readData.op; - if ((readData.swap_filen>>32) != 0) { - fatalf("File ID on record is greater than maximum cache file ID."); - } - swapData.swap_filen = (int32_t)readData.swap_filen; - swapData.timestamp = readData.timestamp; - swapData.lastref = readData.lastref; - swapData.expires = readData.expires; - swapData.lastmod = readData.lastmod; - swapData.swap_file_sz = readData.swap_file_sz; - swapData.refcount = readData.refcount; - swapData.flags = readData.flags; - memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); - return true; - } -}; - -class UFSSwapLogParser_v1:public UFSSwapLogParser -{ -public: - UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) { - record_size = sizeof(StoreSwapLogData); - } - bool ReadRecord(StoreSwapLogData &swapData); -}; - - -bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData) -{ - int bytes = sizeof(StoreSwapLogData); - - assert(log); - - if (fread(&swapData, bytes, 1, log) != 1) { - return false; - } - return true; -} -#endif /* UNUSED_CODE */ - -/// swap.state v2 log parser -class UFSSwapLogParser_v2: public UFSSwapLogParser -{ -public: - UFSSwapLogParser_v2(FILE *fp): UFSSwapLogParser(fp) { - record_size = sizeof(StoreSwapLogData); - } - bool ReadRecord(StoreSwapLogData &swapData) { - assert(log); - return fread(&swapData, sizeof(StoreSwapLogData), 1, log) == 1; - } -}; - - -UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) -{ - StoreSwapLogHeader header; - - assert(fp); - - if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) - return NULL; - - if (header.op != SWAP_LOG_VERSION) { - debugs(47, 1, "Old swap file detected..."); - fseek(fp, 0, SEEK_SET); - return new UFSSwapLogParser_v1_32bs(fp); // Um. 32-bits except time_t, and can't determine that. - } - - debugs(47, 2, "Swap file version: " << header.version); - - if (header.version == 1) { - if (fseek(fp, header.record_size, SEEK_SET) != 0) - return NULL; - - debugs(47, DBG_IMPORTANT, "Rejecting swap file v1 to avoid cache " << - "index corruption. Forcing a full cache index rebuild. " << - "See Squid bug #3441."); - return NULL; - -#if UNUSED_CODE - // baseline - // 32-bit sfileno - // native time_t (hopefully 64-bit) - // 64-bit file size - if (header.record_size == sizeof(StoreSwapLogData)) { - debugs(47, 1, "Version 1 of swap file with LFS support detected... "); - return new UFSSwapLogParser_v1(fp); - } - - // which means we have a 3-way grid of permutations to import (yuck!) - // 1) sfileno 32-bit / 64-bit (64-bit was broken) - // 2) time_t 32-bit / 64-bit - // 3) size_t 32-bit / 64-bit (32-bit was pre-LFS) - - // 32-bit systems... - // only LFS (size_t) differs from baseline - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (32-bit) swap file without LFS support detected... "); - return new UFSSwapLogParser_v1_32bs(fp); - } - // LFS (size_t) and timestamps (time_t) differs from baseline - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (32-bit) swap file with short timestamps and without LFS support detected... "); - return new UFSSwapLogParser_v1_32bst(fp); - } - // No downgrade for 64-bit timestamps to 32-bit. - - // 64-bit systems - // sfileno was 64-bit for a some builds - if (header.record_size == sizeof(struct UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld)) { - debugs(47, 1, "Version 1 (64-bit) swap file with broken sfileno detected... "); - return new UFSSwapLogParser_v1_64bfn(fp); - } - // NP: 64-bit system with 32-bit size_t/time_t are not handled. - - debugs(47, 1, "WARNING: The swap file has wrong format!... "); - debugs(47, 1, "NOTE: Cannot safely downgrade caches to short (32-bit) timestamps."); - return NULL; -#endif - } - - if (header.version >= 2) { - if (!header.sane()) { - debugs(47, DBG_IMPORTANT, "ERROR: Corrupted v" << header.version << - " swap file header."); - return NULL; - } - - if (fseek(fp, header.record_size, SEEK_SET) != 0) - return NULL; - - if (header.version == 2) - return new UFSSwapLogParser_v2(fp); - } - - // TODO: v3: write to disk in network-order bytes for the larger fields? - - debugs(47, DBG_IMPORTANT, "Unknown swap file version: " << header.version); - return NULL; -} - -int UFSSwapLogParser::SwapLogEntries() -{ - struct stat sb; - - if (log_entries >= 0) - return log_entries; - - if (log && record_size && 0 == fstat(fileno(log), &sb)) { - log_entries = sb.st_size/record_size; - return log_entries; - } - - return 0; -} - - - - -RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false) +Fs::Ufs::RebuildState::RebuildState(RefCount aSwapDir) : + sd (aSwapDir), LogParser(NULL), e(NULL), fromLog(true), _done (false) { /* * If the swap.state file exists in the cache_dir, then @@ -349,7 +53,7 @@ RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir),LogPa FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); if (fp && !zeroLengthLog) - LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp); + LogParser = Fs::Ufs::UFSSwapLogParser::GetUFSSwapLogParser(fp); if (LogParser == NULL ) { fromLog = false; @@ -369,7 +73,7 @@ RebuildState::RebuildState (RefCount aSwapDir) : sd (aSwapDir),LogPa (clean ? "clean log" : (LogParser ? "dirty log" : "no log")) << ")"); } -RebuildState::~RebuildState() +Fs::Ufs::RebuildState::~RebuildState() { sd->closeTmpSwapLog(); @@ -378,7 +82,7 @@ RebuildState::~RebuildState() } void -RebuildState::RebuildStep(void *data) +Fs::Ufs::RebuildState::RebuildStep(void *data) { RebuildState *rb = (RebuildState *)data; rb->rebuildStep(); @@ -394,7 +98,7 @@ RebuildState::RebuildStep(void *data) /// load entries from swap.state or files until we run out of entries or time void -RebuildState::rebuildStep() +Fs::Ufs::RebuildState::rebuildStep() { currentEntry(NULL); @@ -431,14 +135,14 @@ RebuildState::rebuildStep() /// process one cache file void -RebuildState::rebuildFromDirectory() +Fs::Ufs::RebuildState::rebuildFromDirectory() { cache_key key[SQUID_MD5_DIGEST_LENGTH]; struct stat sb; int fd = -1; assert(this != NULL); - debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index); + debugs(47, 3, HERE << "DIR #" << sd->index); assert(fd == -1); sfileno filn = 0; @@ -460,7 +164,7 @@ RebuildState::rebuildFromDirectory() ++n_read; if (fstat(fd, &sb) < 0) { - debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror()); + debugs(47, DBG_IMPORTANT, HERE << "fstat(FD " << fd << "): " << xstrerror()); file_close(fd); --store_open_disk_fd; fd = -1; @@ -498,32 +202,32 @@ RebuildState::rebuildFromDirectory() tmpe.timestamp, tmpe.lastref, tmpe.lastmod, - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ + tmpe.refcount, /* refcount */ + tmpe.flags, /* flags */ (int) flags.clean)); storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); } StoreEntry * -RebuildState::currentEntry() const +Fs::Ufs::RebuildState::currentEntry() const { return e; } void -RebuildState::currentEntry(StoreEntry *newValue) +Fs::Ufs::RebuildState::currentEntry(StoreEntry *newValue) { e = newValue; } /// process one swap log entry void -RebuildState::rebuildFromSwapLog() +Fs::Ufs::RebuildState::rebuildFromSwapLog() { StoreSwapLogData swapData; if (LogParser->ReadRecord(swapData) != 1) { - debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); + debugs(47, DBG_IMPORTANT, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); LogParser->Close(); delete LogParser; LogParser = NULL; @@ -548,8 +252,7 @@ RebuildState::rebuildFromSwapLog() */ swapData.swap_filen &= 0x00FFFFFF; - debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " << - swap_log_op_str[(int) swapData.op] << " " << + debugs(47, 3, HERE << swap_log_op_str[(int) swapData.op] << " " << storeKeyText(swapData.key) << " "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swapData.swap_filen); @@ -577,7 +280,7 @@ RebuildState::rebuildFromSwapLog() x = ::log(static_cast(++counts.bad_log_op)) / ::log(10.0); if (0.0 == x - (double) (int) x) - debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); + debugs(47, DBG_IMPORTANT, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); ++counts.invalid; @@ -604,7 +307,7 @@ RebuildState::rebuildFromSwapLog() */ currentEntry (Store::Root().get(swapData.key)); - int used; /* is swapfile already in use? */ + int used; /* is swapfile already in use? */ used = sd->mapBitTest(swapData.swap_filen); @@ -631,7 +334,7 @@ RebuildState::rebuildFromSwapLog() sd->dereference(*currentEntry()); } else { debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); - debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__); + debugs(47, DBG_IMPORTANT, HERE << "bad condition"); } return; } else if (used) { @@ -640,7 +343,7 @@ RebuildState::rebuildFromSwapLog() * point. If the log is dirty, the filesize check should have * caught this. If the log is clean, there should never be a * newer entry. */ - debugs(47, 1, "WARNING: newer swaplog entry for dirno " << + debugs(47, DBG_IMPORTANT, "WARNING: newer swaplog entry for dirno " << sd->index << ", fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swapData.swap_filen); @@ -691,7 +394,7 @@ RebuildState::rebuildFromSwapLog() /// undo the effects of adding an entry in rebuildFromSwapLog() void -RebuildState::undoAdd() +Fs::Ufs::RebuildState::undoAdd() { StoreEntry *added = currentEntry(); assert(added); @@ -711,11 +414,11 @@ RebuildState::undoAdd() } int -RebuildState::getNextFile(sfileno * filn_p, int *size) +Fs::Ufs::RebuildState::getNextFile(sfileno * filn_p, int *size) { int fd = -1; int dirs_opened = 0; - debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init << ", " << + debugs(47, 3, HERE << "flag=" << flags.init << ", " << sd->index << ": /"<< std::setfill('0') << std::hex << std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) << curlvl2); @@ -726,7 +429,7 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) while (fd < 0 && done == 0) { fd = -1; - if (0 == flags.init) { /* initialize, open first file */ + if (0 == flags.init) { /* initialize, open first file */ done = 0; curlvl1 = 0; curlvl2 = 0; @@ -735,7 +438,7 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) assert(Config.cacheSwap.n_configured > 0); } - if (0 == in_dir) { /* we need to read in a new directory */ + if (0 == in_dir) { /* we need to read in a new directory */ snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X", sd->path, curlvl1, curlvl2); @@ -748,14 +451,14 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) ++dirs_opened; if (td == NULL) { - debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror()); + debugs(47, DBG_IMPORTANT, HERE << "error in opendir (" << fullpath << "): " << xstrerror()); } else { - entry = readdir(td); /* skip . and .. */ + entry = readdir(td); /* skip . and .. */ entry = readdir(td); if (entry == NULL && errno == ENOENT) - debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!."); - debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath); + debugs(47, DBG_IMPORTANT, HERE << "WARNING: directory does not exist!"); + debugs(47, 3, HERE << "Directory " << fullpath); } } @@ -763,12 +466,12 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) ++in_dir; if (sscanf(entry->d_name, "%x", &fn) != 1) { - debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name); + debugs(47, 3, HERE << "invalid entry " << entry->d_name); continue; } if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) { - debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') << + debugs(47, 3, HERE << std::setfill('0') << std::hex << std::uppercase << std::setw(8) << fn << " does not belong in " << std::dec << sd->index << "/" << curlvl1 << "/" << curlvl2); @@ -777,17 +480,17 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) } if (sd->mapBitTest(fn)) { - debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next."); + debugs(47, 3, HERE << "Locked, continuing with next."); continue; } snprintf(fullfilename, MAXPATHLEN, "%s/%s", fullpath, entry->d_name); - debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename); + debugs(47, 3, HERE << "Opening " << fullfilename); fd = file_open(fullfilename, O_RDONLY | O_BINARY); if (fd < 0) - debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror()); + debugs(47, DBG_IMPORTANT, HERE << "error opening " << fullfilename << ": " << xstrerror()); else ++store_open_disk_fd; @@ -819,23 +522,19 @@ RebuildState::getNextFile(sfileno * filn_p, int *size) } bool -RebuildState::error() const +Fs::Ufs::RebuildState::error() const { return false; } bool -RebuildState::isDone() const +Fs::Ufs::RebuildState::isDone() const { return _done; } StoreEntry * -RebuildState::currentItem() +Fs::Ufs::RebuildState::currentItem() { return currentEntry(); } - -#if !_USE_INLINE_ -#include "ufscommon.cci" -#endif diff --git a/src/fs/ufs/RebuildState.h b/src/fs/ufs/RebuildState.h new file mode 100644 index 0000000000..aea10bab61 --- /dev/null +++ b/src/fs/ufs/RebuildState.h @@ -0,0 +1,102 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_REBUILDSTATE_H +#define SQUID_FS_UFS_REBUILDSTATE_H + +#include "RefCount.h" +#include "UFSSwapDir.h" +#include "structs.h" +#include "UFSSwapLogParser.h" + +class StoreEntry; + +namespace Fs +{ +namespace Ufs +{ + +/// \ingroup UFS +class RebuildState : public RefCountable +{ +public: + static EVH RebuildStep; + + RebuildState(RefCount sd); + ~RebuildState(); + + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + int n_read; + /* FILE *log;*/ + Fs::Ufs::UFSSwapLogParser *LogParser; + int curlvl1; + int curlvl2; + + struct { + unsigned int need_to_validate:1; + unsigned int clean:1; + unsigned int init:1; + } flags; + int in_dir; + int done; + int fn; + + dirent_t *entry; + DIR *td; + char fullpath[MAXPATHLEN]; + char fullfilename[MAXPATHLEN]; + + struct _store_rebuild_data counts; + +private: + CBDATA_CLASS2(RebuildState); + void rebuildFromDirectory(); + void rebuildFromSwapLog(); + void rebuildStep(); + void undoAdd(); + int getNextFile(sfileno *, int *size); + StoreEntry *currentEntry() const; + void currentEntry(StoreEntry *); + StoreEntry *e; + bool fromLog; + bool _done; + /// \bug (callback) should be hidden behind a proper human readable name + void (callback)(void *cbdata); + void *cbdata; +}; + + +} /* namespace Ufs */ +} /* namespace Fs */ + +#endif /* SQUID_FS_UFS_REBUILDSTATE_H */ diff --git a/src/fs/ufs/StoreFSufs.cc b/src/fs/ufs/StoreFSufs.cc index 18fe2f6025..c1162d13b8 100644 --- a/src/fs/ufs/StoreFSufs.cc +++ b/src/fs/ufs/StoreFSufs.cc @@ -1,6 +1,4 @@ /* - * $Id$ - * * DEBUG: section 47 Store Directory Routines * AUTHOR: Robert Collins * @@ -41,13 +39,11 @@ #endif #include "fs/ufs/StoreFSufs.h" +#include "fs/ufs/UFSSwapDir.h" #if 0 #include "DiskIO/DiskIOModule.h" #endif -/** \todo FIXME: break UFSSwapDir out so we don't build all the extras */ -#include "fs/ufs/ufscommon.h" - /* Unused variable: */ -StoreFSufs *UfsInstance_foo = NULL; +Fs::Ufs::StoreFSufs *UfsInstance_foo = NULL; diff --git a/src/fs/ufs/StoreFSufs.h b/src/fs/ufs/StoreFSufs.h index 9b5d288420..bab13bcd43 100644 --- a/src/fs/ufs/StoreFSufs.h +++ b/src/fs/ufs/StoreFSufs.h @@ -37,10 +37,14 @@ \ingroup FileSystems */ -class DiskIOModule; - #include "StoreFileSystem.h" +class DiskIOModule; + +namespace Fs +{ +namespace Ufs +{ /** \ingroup UFS, FileSystems * @@ -52,7 +56,6 @@ class DiskIOModule; template class StoreFSufs : public StoreFileSystem { - public: static StoreFileSystem &GetInstance(); StoreFSufs(char const *DefaultModuleType, char const *label); @@ -108,4 +111,7 @@ StoreFSufs::setup() initialised = true; } +} /* namespace Ufs */ +} /* namespace Fs */ + #endif /* SQUID_STOREFSUFS_H */ diff --git a/src/fs/ufs/StoreSearchUFS.cc b/src/fs/ufs/StoreSearchUFS.cc new file mode 100644 index 0000000000..a9ec0bdcdc --- /dev/null +++ b/src/fs/ufs/StoreSearchUFS.cc @@ -0,0 +1,90 @@ +/* + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "cbdata.h" +#include "StoreSearchUFS.h" +#include "UFSSwapDir.h" + +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,StoreSearchUFS); + +Fs::Ufs::StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : + sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), + current (NULL), _done (false) +{} + +Fs::Ufs::StoreSearchUFS::~StoreSearchUFS() +{ + walker->Done(walker); + walker = NULL; +} + +void +Fs::Ufs::StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) +{ + next(); + aCallback(aCallbackArgs); +} + +bool +Fs::Ufs::StoreSearchUFS::next() +{ + /* the walker API doesn't make sense. the store entries referred to are already readwrite + * from their hash table entries + */ + + if (walker) + current = const_cast(walker->Next(walker)); + + if (current == NULL) + _done = true; + + return current != NULL; +} + +bool +Fs::Ufs::StoreSearchUFS::error() const +{ + return false; +} + +bool +Fs::Ufs::StoreSearchUFS::isDone() const +{ + return _done; +} + +StoreEntry * +Fs::Ufs::StoreSearchUFS::currentItem() +{ + return current; +} diff --git a/src/fs/ufs/StoreSearchUFS.h b/src/fs/ufs/StoreSearchUFS.h new file mode 100644 index 0000000000..c8d5d017ad --- /dev/null +++ b/src/fs/ufs/StoreSearchUFS.h @@ -0,0 +1,83 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_STORESEARCHUFS_H +#define SQUID_FS_UFS_STORESEARCHUFS_H + +#include "StoreSearch.h" +#include "UFSSwapDir.h" + +namespace Fs +{ +namespace Ufs +{ + +/// \ingroup UFS +class StoreSearchUFS : public StoreSearch +{ +public: + StoreSearchUFS(RefCount sd); + virtual ~StoreSearchUFS(); + + /** \todo Iterator API - garh, wrong place */ + /** + * callback the client when a new StoreEntry is available + * or an error occurs + */ + virtual void next(void (callback)(void *cbdata), void *cbdata); + + /** + \retval true if a new StoreEntry is immediately available + \retval false if a new StoreEntry is NOT immediately available + */ + virtual bool next(); + + virtual bool error() const; + virtual bool isDone() const; + virtual StoreEntry *currentItem(); + + RefCount sd; + RemovalPolicyWalker *walker; + +private: + CBDATA_CLASS2(StoreSearchUFS); + /// \bug (callback) should be hidden behind a proper human readable name + void (callback)(void *cbdata); + void *cbdata; + StoreEntry * current; + bool _done; + + StoreSearchUFS(StoreSearchUFS const &); //disabled + StoreSearchUFS& operator=(StoreSearchUFS const &); //disabled + StoreSearchUFS(); //disabled +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_STORESEARCHUFS_H */ diff --git a/src/fs/ufs/store_io_ufs.cc b/src/fs/ufs/UFSStoreState.cc similarity index 76% rename from src/fs/ufs/store_io_ufs.cc rename to src/fs/ufs/UFSStoreState.cc index 2e99844748..cfe8592e6f 100644 --- a/src/fs/ufs/store_io_ufs.cc +++ b/src/fs/ufs/UFSStoreState.cc @@ -35,7 +35,6 @@ #include "squid-old.h" #include "Store.h" -#include "ufscommon.h" #include "Generic.h" #include "DiskIO/DiskFile.h" #include "DiskIO/DiskIOStrategy.h" @@ -43,63 +42,27 @@ #include "DiskIO/WriteRequest.h" #include "SwapDir.h" +#include "UFSStrategy.h" +#include "UFSStoreState.h" -bool -UFSStrategy::shedLoad() -{ - return io->shedLoad(); -} - -int -UFSStrategy::load() -{ - return io->load(); -} - -UFSStrategy::UFSStrategy (DiskIOStrategy *anIO) : io(anIO) -{} - -UFSStrategy::~UFSStrategy () -{ - delete io; -} - -StoreIOState::Pointer -UFSStrategy::createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * aCallback, void *callback_data) const -{ - return new UFSStoreState (SD, e, aCallback, callback_data); -} - -DiskFile::Pointer -UFSStrategy::newFile (char const *path) -{ - return io->newFile(path); -} - - -void -UFSStrategy::unlinkFile(char const *path) -{ - io->unlinkFile(path); -} -CBDATA_CLASS_INIT(UFSStoreState); +CBDATA_NAMESPACED_CLASS_INIT(Fs::Ufs,UFSStoreState); void * -UFSStoreState::operator new (size_t) +Fs::Ufs::UFSStoreState::operator new (size_t) { CBDATA_INIT_TYPE(UFSStoreState); return cbdataAlloc(UFSStoreState); } void -UFSStoreState::operator delete (void *address) +Fs::Ufs::UFSStoreState::operator delete (void *address) { cbdataFree(address); } void -UFSStoreState::ioCompletedNotification() +Fs::Ufs::UFSStoreState::ioCompletedNotification() { if (opening) { opening = false; @@ -137,7 +100,7 @@ UFSStoreState::ioCompletedNotification() } void -UFSStoreState::openDone() +Fs::Ufs::UFSStoreState::openDone() { if (closing) debugs(0, DBG_CRITICAL, HERE << "already closing in openDone()!?"); @@ -162,7 +125,7 @@ UFSStoreState::openDone() } void -UFSStoreState::closeCompleted() +Fs::Ufs::UFSStoreState::closeCompleted() { assert (closing); debugs(79, 3, "UFSStoreState::closeCompleted: dirno " << swap_dirn << @@ -190,7 +153,7 @@ UFSStoreState::closeCompleted() * when it is safe to actually signal the lower layer for closing. */ void -UFSStoreState::close(int) +Fs::Ufs::UFSStoreState::close(int) { debugs(79, 3, "UFSStoreState::close: dirno " << swap_dirn << ", fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); @@ -198,7 +161,7 @@ UFSStoreState::close(int) } void -UFSStoreState::read_(char *buf, size_t size, off_t aOffset, STRCB * aCallback, void *aCallbackData) +Fs::Ufs::UFSStoreState::read_(char *buf, size_t size, off_t aOffset, STRCB * aCallback, void *aCallbackData) { assert(read.callback == NULL); assert(read.callback_data == NULL); @@ -232,7 +195,7 @@ UFSStoreState::read_(char *buf, size_t size, off_t aOffset, STRCB * aCallback, v * code simpler and always go through the pending_writes queue. */ void -UFSStoreState::write(char const *buf, size_t size, off_t aOffset, FREE * free_func) +Fs::Ufs::UFSStoreState::write(char const *buf, size_t size, off_t aOffset, FREE * free_func) { debugs(79, 3, "UFSStoreState::write: dirn " << swap_dirn << ", fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen); @@ -256,7 +219,7 @@ UFSStoreState::write(char const *buf, size_t size, off_t aOffset, FREE * free_fu * called by drainWriteQueue(). */ void -UFSStoreState::doWrite() +Fs::Ufs::UFSStoreState::doWrite() { debugs(79, 3, HERE << this << " UFSStoreState::doWrite"); @@ -300,7 +263,7 @@ UFSStoreState::doWrite() } void -UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount result) +Fs::Ufs::UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount result) { assert (result.getRaw()); reading = false; @@ -343,7 +306,7 @@ UFSStoreState::readCompleted(const char *buf, int len, int errflag, RefCount writeRequest) +Fs::Ufs::UFSStoreState::writeCompleted(int errflag, size_t len, RefCount writeRequest) { debugs(79, 3, HERE << "dirno " << swap_dirn << ", fileno " << std::setfill('0') << std::hex << std::uppercase << std::setw(8) << swap_filen << @@ -369,7 +332,7 @@ UFSStoreState::writeCompleted(int errflag, size_t len, RefCount wr } void -UFSStoreState::doCloseCallback(int errflag) +Fs::Ufs::UFSStoreState::doCloseCallback(int errflag) { debugs(79, 3, "storeUfsIOCallback: errflag=" << errflag); /* @@ -398,7 +361,7 @@ UFSStoreState::doCloseCallback(int errflag) /* ============= THE REAL UFS CODE ================ */ -UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_) : opening (false), creating (false), closing (false), reading(false), writing(false), pending_reads(NULL), pending_writes (NULL) +Fs::Ufs::UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_) : opening (false), creating (false), closing (false), reading(false), writing(false), pending_reads(NULL), pending_writes (NULL) { swap_filen = anEntry->swap_filen; swap_dirn = SD->index; @@ -410,14 +373,14 @@ UFSStoreState::UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callba flags.try_closing = false; } -UFSStoreState::~UFSStoreState() +Fs::Ufs::UFSStoreState::~UFSStoreState() { assert(pending_reads == NULL); assert(pending_writes == NULL); } void -UFSStoreState::freePending() +Fs::Ufs::UFSStoreState::freePending() { _queued_read *qr; @@ -440,7 +403,7 @@ UFSStoreState::freePending() } bool -UFSStoreState::kickReadQueue() +Fs::Ufs::UFSStoreState::kickReadQueue() { _queued_read *q = (_queued_read *)linklistShift(&pending_reads); @@ -465,7 +428,7 @@ UFSStoreState::kickReadQueue() } void -UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_, void *callback_data_) +Fs::Ufs::UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_, void *callback_data_) { debugs(79, 3, "UFSStoreState::queueRead: queueing read"); assert(opening); @@ -484,7 +447,7 @@ UFSStoreState::queueRead(char *buf, size_t size, off_t aOffset, STRCB *callback_ * drainWriteQueue() is a loop around doWrite(). */ void -UFSStoreState::drainWriteQueue() +Fs::Ufs::UFSStoreState::drainWriteQueue() { /* * DPW 2007-04-12 @@ -521,7 +484,7 @@ UFSStoreState::drainWriteQueue() * or will remember to do the close for us. */ void -UFSStoreState::tryClosing() +Fs::Ufs::UFSStoreState::tryClosing() { debugs(79,3,HERE << this << " tryClosing()" << " closing = " << closing << @@ -541,7 +504,7 @@ UFSStoreState::tryClosing() } void -UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * free_func) +Fs::Ufs::UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * free_func) { debugs(79, 3, HERE << this << " UFSStoreState::queueWrite: queueing write of size " << size); @@ -554,109 +517,3 @@ UFSStoreState::queueWrite(char const *buf, size_t size, off_t aOffset, FREE * fr linklistPush(&pending_writes, q); } -StoreIOState::Pointer -UFSStrategy::open(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, - StoreIOState::STIOCB * aCallback, void *callback_data) -{ - assert (((UFSSwapDir *)SD)->IO == this); - debugs(79, 3, "UFSStrategy::open: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e->swap_filen); - - /* to consider: make createstate a private UFSStrategy call */ - StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); - - sio->mode |= O_RDONLY; - - UFSStoreState *state = dynamic_cast (sio.getRaw()); - - assert (state); - - char *path = ((UFSSwapDir *)SD)->fullPath(e->swap_filen, NULL); - - DiskFile::Pointer myFile = newFile (path); - - if (myFile.getRaw() == NULL) - return NULL; - - state->theFile = myFile; - - state->opening = true; - - myFile->open (sio->mode, 0644, state); - - if (myFile->error()) - return NULL; - - return sio; -} - -StoreIOState::Pointer -UFSStrategy::create(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, - StoreIOState::STIOCB * aCallback, void *callback_data) -{ - assert (((UFSSwapDir *)SD)->IO == this); - /* Allocate a number */ - sfileno filn = ((UFSSwapDir *)SD)->mapBitAllocate(); - debugs(79, 3, "UFSStrategy::create: fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn); - - /* Shouldn't we handle a 'bitmap full' error here? */ - - StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); - - sio->mode |= O_WRONLY | O_CREAT | O_TRUNC; - - sio->swap_filen = filn; - - UFSStoreState *state = dynamic_cast (sio.getRaw()); - - assert (state); - - char *path = ((UFSSwapDir *)SD)->fullPath(filn, NULL); - - DiskFile::Pointer myFile = newFile (path); - - if (myFile.getRaw() == NULL) { - ((UFSSwapDir *)SD)->mapBitReset (filn); - return NULL; - } - - state->theFile = myFile; - - state->creating = true; - - myFile->create (state->mode, 0644, state); - - if (myFile->error()) { - ((UFSSwapDir *)SD)->mapBitReset (filn); - return NULL; - } - - /* now insert into the replacement policy */ - ((UFSSwapDir *)SD)->replacementAdd(e); - - return sio; -} - -int -UFSStrategy::callback() -{ - return io->callback(); -} - -void -UFSStrategy::init() -{ - io->init(); -} - -void -UFSStrategy::sync() -{ - io->sync(); -} - -void -UFSStrategy::statfs(StoreEntry & sentry)const -{ - io->statfs(sentry); -} - diff --git a/src/fs/ufs/UFSStoreState.h b/src/fs/ufs/UFSStoreState.h new file mode 100644 index 0000000000..bf078b8d89 --- /dev/null +++ b/src/fs/ufs/UFSStoreState.h @@ -0,0 +1,129 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_UFSSTORESTATE_H +#define SQUID_FS_UFS_UFSSTORESTATE_H + +#include "DiskIO/IORequestor.h" +#include "StoreIOState.h" + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSStoreState : public StoreIOState, public IORequestor +{ +public: + void * operator new (size_t); + void operator delete (void *); + UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_); + ~UFSStoreState(); + virtual void close(int how); + virtual void closeCompleted(); + // protected: + virtual void ioCompletedNotification(); + virtual void readCompleted(const char *buf, int len, int errflag, RefCount); + virtual void writeCompleted(int errflag, size_t len, RefCount); + RefCount theFile; + bool opening; + bool creating; + bool closing; + bool reading; + bool writing; + void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); + void write(char const *buf, size_t size, off_t offset, FREE * free_func); + +protected: + virtual void doCloseCallback (int errflag); + + class _queued_read + { + public: + MEMPROXY_CLASS(UFSStoreState::_queued_read); + char *buf; + size_t size; + off_t offset; + STRCB *callback; + void *callback_data; + + }; + + class _queued_write + { + public: + MEMPROXY_CLASS(UFSStoreState::_queued_write); + char const *buf; + size_t size; + off_t offset; + FREE *free_func; + + }; + + /** \todo These should be in the IO strategy */ + + struct { + /** + * DPW 2006-05-24 + * the write_draining flag is used to avoid recursion inside + * the UFSStoreState::drainWriteQueue() method. + */ + bool write_draining; + /** + * DPW 2006-05-24 + * The try_closing flag is set by UFSStoreState::tryClosing() + * when UFSStoreState wants to close the file, but cannot + * because of pending I/Os. If set, UFSStoreState will + * try to close again in the I/O callbacks. + */ + bool try_closing; + } flags; + link_list *pending_reads; + link_list *pending_writes; + void queueRead(char *, size_t, off_t, STRCB *, void *); + void queueWrite(char const *, size_t, off_t, FREE *); + bool kickReadQueue(); + void drainWriteQueue(); + void tryClosing(); + char *read_buf; + +private: + void openDone(); + void freePending(); + void doWrite(); + CBDATA_CLASS(UFSStoreState); +}; + +MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_read); +MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_write); + +} //namespace Ufs +} //namespace Fs + +#endif /* SQUID_FS_UFS_UFSSTORESTATE_H */ diff --git a/src/fs/ufs/UFSStrategy.cc b/src/fs/ufs/UFSStrategy.cc new file mode 100644 index 0000000000..78fd0c8898 --- /dev/null +++ b/src/fs/ufs/UFSStrategy.cc @@ -0,0 +1,183 @@ +/* + * DEBUG: section 47 Store Directory Routines + * AUTHOR: Robert Collins + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#include "squid.h" + +#include "DiskIO/DiskIOStrategy.h" +#include "UFSStrategy.h" +#include "UFSStoreState.h" +#include "UFSSwapDir.h" + +bool +Fs::Ufs::UFSStrategy::shedLoad() +{ + return io->shedLoad(); +} + +int +Fs::Ufs::UFSStrategy::load() +{ + return io->load(); +} + +Fs::Ufs::UFSStrategy::UFSStrategy (DiskIOStrategy *anIO) : io(anIO) +{} + +Fs::Ufs::UFSStrategy::~UFSStrategy () +{ + delete io; +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * aCallback, void *callback_data) const +{ + return new Fs::Ufs::UFSStoreState (SD, e, aCallback, callback_data); +} + +DiskFile::Pointer +Fs::Ufs::UFSStrategy::newFile (char const *path) +{ + return io->newFile(path); +} + +void +Fs::Ufs::UFSStrategy::unlinkFile(char const *path) +{ + io->unlinkFile(path); +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::open(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, + StoreIOState::STIOCB * aCallback, void *callback_data) +{ + assert (((UFSSwapDir *)SD)->IO == this); + debugs(79, 3, HERE << "fileno "<< std::setfill('0') << std::hex + << std::uppercase << std::setw(8) << e->swap_filen); + + /* to consider: make createstate a private UFSStrategy call */ + StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); + + sio->mode |= O_RDONLY; + + Fs::Ufs::UFSStoreState *state = dynamic_cast (sio.getRaw()); + + assert (state); + + char *path = ((UFSSwapDir *)SD)->fullPath(e->swap_filen, NULL); + + DiskFile::Pointer myFile = newFile (path); + + if (myFile.getRaw() == NULL) + return NULL; + + state->theFile = myFile; + + state->opening = true; + + myFile->open (sio->mode, 0644, state); + + if (myFile->error()) + return NULL; + + return sio; +} + +StoreIOState::Pointer +Fs::Ufs::UFSStrategy::create(SwapDir * SD, StoreEntry * e, StoreIOState::STFNCB * file_callback, + StoreIOState::STIOCB * aCallback, void *callback_data) +{ + assert (((UFSSwapDir *)SD)->IO == this); + /* Allocate a number */ + sfileno filn = ((UFSSwapDir *)SD)->mapBitAllocate(); + debugs(79, 3, HERE << "fileno "<< std::setfill('0') << + std::hex << std::uppercase << std::setw(8) << filn); + + /* Shouldn't we handle a 'bitmap full' error here? */ + + StoreIOState::Pointer sio = createState (SD, e, aCallback, callback_data); + + sio->mode |= O_WRONLY | O_CREAT | O_TRUNC; + + sio->swap_filen = filn; + + Fs::Ufs::UFSStoreState *state = dynamic_cast (sio.getRaw()); + + assert (state); + + char *path = ((UFSSwapDir *)SD)->fullPath(filn, NULL); + + DiskFile::Pointer myFile = newFile (path); + + if (myFile.getRaw() == NULL) { + ((UFSSwapDir *)SD)->mapBitReset (filn); + return NULL; + } + + state->theFile = myFile; + + state->creating = true; + + myFile->create (state->mode, 0644, state); + + if (myFile->error()) { + ((UFSSwapDir *)SD)->mapBitReset (filn); + return NULL; + } + + /* now insert into the replacement policy */ + ((UFSSwapDir *)SD)->replacementAdd(e); + + return sio; +} + +int +Fs::Ufs::UFSStrategy::callback() +{ + return io->callback(); +} + +void +Fs::Ufs::UFSStrategy::init() +{ + io->init(); +} + +void +Fs::Ufs::UFSStrategy::sync() +{ + io->sync(); +} + +void +Fs::Ufs::UFSStrategy::statfs(StoreEntry & sentry)const +{ + io->statfs(sentry); +} diff --git a/src/fs/ufs/UFSStrategy.h b/src/fs/ufs/UFSStrategy.h new file mode 100644 index 0000000000..6e6b1d43af --- /dev/null +++ b/src/fs/ufs/UFSStrategy.h @@ -0,0 +1,90 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#ifndef SQUID_FS_UFS_UFSSTRATEGY_H +#define SQUID_FS_UFS_UFSSTRATEGY_H + +#include "DiskIO/DiskFile.h" +#include "StoreIOState.h" + +class Swapdir; +class StoreEntry; +class DiskIOStrategy; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSStrategy +{ +public: + UFSStrategy (DiskIOStrategy *); + virtual ~UFSStrategy (); + virtual bool shedLoad(); + + virtual int load(); + + StoreIOState::Pointer createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * callback, void *callback_data) const; + /* UFS specific */ + virtual RefCount newFile (char const *path); + StoreIOState::Pointer open(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, + StoreIOState::STIOCB *, void *); + StoreIOState::Pointer create(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, + StoreIOState::STIOCB *, void *); + + virtual void unlinkFile (char const *); + virtual void sync(); + + virtual int callback(); + + /** Init per-instance logic */ + virtual void init(); + + /** cachemgr output on the IO instance stats */ + virtual void statfs(StoreEntry & sentry)const; + + /** The io strategy in use */ + DiskIOStrategy *io; + +protected: + + friend class UFSSwapDir; + +private: + UFSStrategy(); //disabled + UFSStrategy(UFSStrategy const &); //disabled + UFSStrategy &operator=(UFSStrategy const &); //disabled + +}; + +} //namespace Ufs +} //namespace Fs + +#endif /* SQUID_FS_UFS_UFSSTRATEGY_H */ diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/UFSSwapDir.cc similarity index 73% rename from src/fs/ufs/store_dir_ufs.cc rename to src/fs/ufs/UFSSwapDir.cc index 9c6fa87822..c4bfba5a03 100644 --- a/src/fs/ufs/store_dir_ufs.cc +++ b/src/fs/ufs/UFSSwapDir.cc @@ -1,9 +1,6 @@ - /* - * $Id$ - * * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels + * AUTHOR: Robert Collins * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- @@ -30,37 +27,107 @@ * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * */ #include "squid-old.h" -#include "Store.h" -#include "fde.h" -#include "ufscommon.h" -#include "StoreSwapLogData.h" + +#define CLEAN_BUF_SZ 16384 + #include "ConfigOption.h" -#include "DiskIO/DiskIOStrategy.h" #include "DiskIO/DiskIOModule.h" #include "FileMap.h" +#include "fde.h" #include "Parsing.h" +#include "protos.h" +#include "RebuildState.h" #include "SquidMath.h" +#include "DiskIO/DiskIOStrategy.h" +#include "StoreSearchUFS.h" +#include "StoreSwapLogData.h" #include "SquidTime.h" #include "StatCounters.h" -#include "SwapDir.h" -#include "swap_log_op.h" +#include "UFSSwapDir.h" -int UFSSwapDir::NumberOfUFSDirs = 0; -int *UFSSwapDir::UFSDirToGlobalDirMapping = NULL; -/* - * storeUfsDirCheckObj - * - * This routine is called by storeDirSelectSwapDir to see if the given - * object is able to be stored on this filesystem. UFS filesystems will - * happily store anything as long as the LRU time isn't too small. - */ +int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0; +int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = NULL; + +class UFSCleanLog : public SwapDir::CleanLog +{ + +public: + UFSCleanLog(SwapDir *); + /** Get the next entry that is a candidate for clean log writing + */ + virtual const StoreEntry *nextEntry(); + /** "write" an entry to the clean log file. + */ + virtual void write(StoreEntry const &); + char *cur; + char *newLog; + char *cln; + char *outbuf; + off_t outbuf_offset; + int fd; + RemovalPolicyWalker *walker; + SwapDir *sd; +}; + +UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : + cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL), + outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) +{} + +const StoreEntry * +UFSCleanLog::nextEntry() +{ + const StoreEntry *entry = NULL; + + if (walker) + entry = walker->Next(walker); + + return entry; +} + +void +UFSCleanLog::write(StoreEntry const &e) +{ + StoreSwapLogData s; + static size_t ss = sizeof(StoreSwapLogData); + s.op = (char) SWAP_LOG_ADD; + s.swap_filen = e.swap_filen; + s.timestamp = e.timestamp; + s.lastref = e.lastref; + s.expires = e.expires; + s.lastmod = e.lastmod; + s.swap_file_sz = e.swap_file_sz; + s.refcount = e.refcount; + s.flags = e.flags; + memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); + s.finalize(); + memcpy(outbuf + outbuf_offset, &s, ss); + outbuf_offset += ss; + /* buffered write */ + + if (outbuf_offset + ss >= CLEAN_BUF_SZ) { + if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { + /* XXX This error handling should probably move up to the caller */ + debugs(50, DBG_CRITICAL, HERE << newLog << ": write: " << xstrerror()); + debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); + file_close(fd); + fd = -1; + unlink(newLog); + sd->cleanLog = NULL; + delete this; + return; + } + + outbuf_offset = 0; + } +} + bool -UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const +Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const { if (!SwapDir::canStore(e, diskSpaceNeeded, load)) return false; @@ -72,11 +139,24 @@ UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) co return true; } +static void +FreeObject(void *address) +{ + StoreSwapLogData *anObject = static_cast (address); + delete anObject; +} -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ +static QS rev_int_sort; +static int +rev_int_sort(const void *A, const void *B) +{ + const int *i1 = (const int *)A; + const int *i2 = (const int *)B; + return *i2 - *i1; +} void -UFSSwapDir::parseSizeL1L2() +Fs::Ufs::UFSSwapDir::parseSizeL1L2() { int i = GetInteger(); if (i <= 0) @@ -89,7 +169,7 @@ UFSSwapDir::parseSizeL1L2() if (size == maxSize()) debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); else - debugs(3, 1, "Cache dir '" << path << "' size changed to " << i << " MB"); + debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB"); } max_size = size; @@ -105,26 +185,15 @@ UFSSwapDir::parseSizeL1L2() fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); } -/* - * storeUfsDirReconfigure - * - * This routine is called when the given swapdir needs reconfiguring - */ - void -UFSSwapDir::reconfigure() +Fs::Ufs::UFSSwapDir::reconfigure() { parseSizeL1L2(); parseOptions(1); } -/* - * storeUfsDirParse - * - * Called when a *new* fs is being setup. - */ void -UFSSwapDir::parse (int anIndex, char *aPath) +Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath) { index = anIndex; path = xstrdup(aPath); @@ -138,7 +207,7 @@ UFSSwapDir::parse (int anIndex, char *aPath) } void -UFSSwapDir::changeIO(DiskIOModule *module) +Fs::Ufs::UFSSwapDir::changeIO(DiskIOModule *module) { DiskIOStrategy *anIO = module->createStrategy(); safe_free(ioType); @@ -159,7 +228,7 @@ UFSSwapDir::changeIO(DiskIOModule *module) } bool -UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) +Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) { if (strcmp(option, "IOEngine") != 0) return false; @@ -182,13 +251,13 @@ UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig } void -UFSSwapDir::optionIODump(StoreEntry * e) const +Fs::Ufs::UFSSwapDir::optionIODump(StoreEntry * e) const { storeAppendPrintf(e, " IOEngine=%s", ioType); } ConfigOption * -UFSSwapDir::getOptionTree() const +Fs::Ufs::UFSSwapDir::getOptionTree() const { ConfigOption *parentResult = SwapDir::getOptionTree(); @@ -209,13 +278,10 @@ UFSSwapDir::getOptionTree() const return result; } -/* - * Initial setup / end destruction - */ void -UFSSwapDir::init() +Fs::Ufs::UFSSwapDir::init() { - debugs(47, 3, "Initialising UFS SwapDir engine."); + debugs(47, 3, HERE << "Initialising UFS SwapDir engine."); /* Parsing must be finished by now - force to NULL, don't delete */ currentIOOptions = NULL; static int started_clean_event = 0; @@ -241,22 +307,22 @@ UFSSwapDir::init() } void -UFSSwapDir::create() +Fs::Ufs::UFSSwapDir::create() { debugs(47, 3, "Creating swap space in " << path); createDirectory(path, 0); createSwapSubDirs(); } -UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) +Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) { /* modulename is only set to disk modules that are built, by configure, * so the Find call should never return NULL here. */ - IO = new UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); + IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); } -UFSSwapDir::~UFSSwapDir() +Fs::Ufs::UFSSwapDir::~UFSSwapDir() { if (swaplog_fd > -1) { file_close(swaplog_fd); @@ -274,33 +340,28 @@ UFSSwapDir::~UFSSwapDir() } void -UFSSwapDir::dumpEntry(StoreEntry &e) const +Fs::Ufs::UFSSwapDir::dumpEntry(StoreEntry &e) const { - debugs(47, 0, "UFSSwapDir::dumpEntry: FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - debugs(47, 0, "UFSSwapDir::dumpEntry: PATH " << fullPath(e.swap_filen, NULL) ); + debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); + debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL) ); e.dump(0); } -/* - * UFSSwapDir::doubleCheck - * - * This is called by storeCleanup() if -S was given on the command line. - */ bool -UFSSwapDir::doubleCheck(StoreEntry & e) +Fs::Ufs::UFSSwapDir::doubleCheck(StoreEntry & e) { struct stat sb; if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { - debugs(47, 0, "UFSSwapDir::doubleCheck: MISSING SWAP FILE"); + debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file"); dumpEntry(e); return true; } if ((off_t)e.swap_file_sz != sb.st_size) { - debugs(47, 0, "UFSSwapDir::doubleCheck: SIZE MISMATCH"); - debugs(47, 0, "UFSSwapDir::doubleCheck: ENTRY SIZE: " << e.swap_file_sz << ", FILE SIZE: " << sb.st_size); + debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: " + << e.swap_file_sz << ", file size: " << sb.st_size); dumpEntry(e); return true; } @@ -309,7 +370,7 @@ UFSSwapDir::doubleCheck(StoreEntry & e) } void -UFSSwapDir::statfs(StoreEntry & sentry) const +Fs::Ufs::UFSSwapDir::statfs(StoreEntry & sentry) const { int totl_kb = 0; int free_kb = 0; @@ -352,7 +413,7 @@ UFSSwapDir::statfs(StoreEntry & sentry) const } void -UFSSwapDir::maintain() +Fs::Ufs::UFSSwapDir::maintain() { /* We can't delete objects while rebuilding swap */ @@ -379,7 +440,7 @@ UFSSwapDir::maintain() * This is kinda cheap, but so we need this priority hack? */ - debugs(47, 3, "storeMaintainSwapSpace: f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); + debugs(47, 3, HERE << "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); walker = repl->PurgeInit(repl, max_scan); @@ -393,7 +454,7 @@ UFSSwapDir::maintain() e = walker->Next(walker); if (!e) - break; /* no more objects */ + break; /* no more objects */ ++removed; @@ -401,35 +462,26 @@ UFSSwapDir::maintain() } walker->Done(walker); - debugs(47, (removed ? 2 : 3), "UFSSwapDir::maintain: " << path << + debugs(47, (removed ? 2 : 3), HERE << path << " removed " << removed << "/" << max_remove << " f=" << std::setprecision(4) << f << " max_scan=" << max_scan); } -/* - * UFSSwapDir::reference - * - * This routine is called whenever an object is referenced, so we can - * maintain replacement information within the storage fs. - */ void -UFSSwapDir::reference(StoreEntry &e) +Fs::Ufs::UFSSwapDir::reference(StoreEntry &e) { - debugs(47, 3, "UFSSwapDir::reference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); + debugs(47, 3, HERE << "referencing " << &e << " " << + e.swap_dirn << "/" << e.swap_filen); if (repl->Referenced) repl->Referenced(repl, &e, &e.repl); } -/* - * UFSSwapDir::dereference - * This routine is called whenever the last reference to an object is - * removed, to maintain replacement information within the storage fs. - */ bool -UFSSwapDir::dereference(StoreEntry & e) +Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e) { - debugs(47, 3, "UFSSwapDir::dereference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); + debugs(47, 3, HERE << "dereferencing " << &e << " " << + e.swap_dirn << "/" << e.swap_filen); if (repl->Dereferenced) repl->Dereferenced(repl, &e, &e.repl); @@ -438,31 +490,31 @@ UFSSwapDir::dereference(StoreEntry & e) } StoreIOState::Pointer -UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) +Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) { return IO->create (this, &e, file_callback, aCallback, callback_data); } StoreIOState::Pointer -UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) +Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) { return IO->open (this, &e, file_callback, aCallback, callback_data); } int -UFSSwapDir::mapBitTest(sfileno filn) +Fs::Ufs::UFSSwapDir::mapBitTest(sfileno filn) { return map->testBit(filn); } void -UFSSwapDir::mapBitSet(sfileno filn) +Fs::Ufs::UFSSwapDir::mapBitSet(sfileno filn) { map->setBit(filn); } void -UFSSwapDir::mapBitReset(sfileno filn) +Fs::Ufs::UFSSwapDir::mapBitReset(sfileno filn) { /* * We have to test the bit before calling clearBit as @@ -477,7 +529,7 @@ UFSSwapDir::mapBitReset(sfileno filn) } int -UFSSwapDir::mapBitAllocate() +Fs::Ufs::UFSSwapDir::mapBitAllocate() { int fn; fn = map->allocate(suggest); @@ -487,7 +539,7 @@ UFSSwapDir::mapBitAllocate() } char * -UFSSwapDir::swapSubDir(int subdirn)const +Fs::Ufs::UFSSwapDir::swapSubDir(int subdirn)const { LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); assert(0 <= subdirn && subdirn < l1); @@ -496,7 +548,7 @@ UFSSwapDir::swapSubDir(int subdirn)const } int -UFSSwapDir::createDirectory(const char *aPath, int should_exist) +Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist) { int created = 0; @@ -505,12 +557,19 @@ UFSSwapDir::createDirectory(const char *aPath, int should_exist) if (0 == ::stat(aPath, &st)) { if (S_ISDIR(st.st_mode)) { - debugs(47, (should_exist ? 3 : 1), aPath << " exists"); + debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists"); } else { fatalf("Swap directory %s is not a directory.", aPath); } + +#if _SQUID_MSWIN_ + + } else if (0 == mkdir(aPath)) { +#else + } else if (0 == mkdir(aPath, 0755)) { - debugs(47, (should_exist ? 1 : 3), aPath << " created"); +#endif + debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created"); created = 1; } else { fatalf("Failed to make swap directory %s: %s", @@ -521,31 +580,26 @@ UFSSwapDir::createDirectory(const char *aPath, int should_exist) } bool -UFSSwapDir::pathIsDirectory(const char *aPath)const +Fs::Ufs::UFSSwapDir::pathIsDirectory(const char *aPath)const { struct stat sb; if (::stat(aPath, &sb) < 0) { - debugs(47, 0, "" << aPath << ": " << xstrerror()); + debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerror()); return false; } if (S_ISDIR(sb.st_mode) == 0) { - debugs(47, 0, "" << aPath << " is not a directory"); + debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory"); return false; } return true; } -/* - * This function is called by commonUfsDirInit(). If this returns < 0, - * then Squid exits, complains about swap directories not - * existing, and instructs the admin to run 'squid -z' - */ bool -UFSSwapDir::verifyCacheDirs() +Fs::Ufs::UFSSwapDir::verifyCacheDirs() { if (!pathIsDirectory(path)) return true; @@ -561,7 +615,7 @@ UFSSwapDir::verifyCacheDirs() } void -UFSSwapDir::createSwapSubDirs() +Fs::Ufs::UFSSwapDir::createSwapSubDirs() { LOCAL_ARRAY(char, name, MAXPATHLEN); @@ -575,7 +629,7 @@ UFSSwapDir::createSwapSubDirs() else should_exist = 1; - debugs(47, 1, "Making directories in " << name); + debugs(47, DBG_IMPORTANT, "Making directories in " << name); for (int k = 0; k < l2; ++k) { snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); @@ -585,7 +639,7 @@ UFSSwapDir::createSwapSubDirs() } char * -UFSSwapDir::logFile(char const *ext) const +Fs::Ufs::UFSSwapDir::logFile(char const *ext) const { LOCAL_ARRAY(char, lpath, MAXPATHLEN); LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); @@ -622,18 +676,18 @@ UFSSwapDir::logFile(char const *ext) const } void -UFSSwapDir::openLog() +Fs::Ufs::UFSSwapDir::openLog() { char *logPath; logPath = logFile(); swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); if (swaplog_fd < 0) { - debugs(50, 1, "" << logPath << ": " << xstrerror()); - fatal("commonUfsDirOpenSwapLog: Failed to open swap log."); + debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerror()); + fatal("UFSSwapDir::openLog: Failed to open swap log."); } - debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd); + debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd); if (0 == NumberOfUFSDirs) assert(NULL == UFSDirToGlobalDirMapping); @@ -644,9 +698,9 @@ UFSSwapDir::openLog() } void -UFSSwapDir::closeLog() +Fs::Ufs::UFSSwapDir::closeLog() { - if (swaplog_fd < 0) /* not open */ + if (swaplog_fd < 0) /* not open */ return; file_close(swaplog_fd); @@ -664,21 +718,19 @@ UFSSwapDir::closeLog() } bool -UFSSwapDir::validL1(int anInt) const +Fs::Ufs::UFSSwapDir::validL1(int anInt) const { return anInt < l1; } bool -UFSSwapDir::validL2(int anInt) const +Fs::Ufs::UFSSwapDir::validL2(int anInt) const { return anInt < l2; } -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ StoreEntry * -UFSSwapDir::addDiskRestore(const cache_key * key, +Fs::Ufs::UFSSwapDir::addDiskRestore(const cache_key * key, sfileno file_number, uint64_t swap_file_sz, time_t expires, @@ -690,7 +742,7 @@ UFSSwapDir::addDiskRestore(const cache_key * key, int clean) { StoreEntry *e = NULL; - debugs(47, 5, "commonUfsAddDiskRestore: " << storeKeyText(key) << + debugs(47, 5, HERE << storeKeyText(key) << ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); /* if you call this you'd better be sure file_number is not * already in use! */ @@ -716,13 +768,13 @@ UFSSwapDir::addDiskRestore(const cache_key * key, mapBitSet(e->swap_filen); cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); ++n_disk_objects; - e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ + e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ replacementAdd (e); return e; } void -UFSSwapDir::undoAddDiskRestore(StoreEntry *e) +Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e) { debugs(47, 5, HERE << *e); replacementRemove(e); // checks swap_dirn so do it before we invalidate it @@ -735,14 +787,14 @@ UFSSwapDir::undoAddDiskRestore(StoreEntry *e) } void -UFSSwapDir::rebuild() +Fs::Ufs::UFSSwapDir::rebuild() { ++StoreController::store_dirs_rebuilding; - eventAdd("storeRebuild", RebuildState::RebuildStep, new RebuildState(this), 0.0, 1); + eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1); } void -UFSSwapDir::closeTmpSwapLog() +Fs::Ufs::UFSSwapDir::closeTmpSwapLog() { char *swaplog_path = xstrdup(logFile(NULL)); char *new_path = xstrdup(logFile(".new")); @@ -750,25 +802,25 @@ UFSSwapDir::closeTmpSwapLog() file_close(swaplog_fd); if (xrename(new_path, swaplog_path) < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); + debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); } fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); if (fd < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); + debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); fatalf("Failed to open swap log %s", swaplog_path); } safe_free(swaplog_path); safe_free(new_path); swaplog_fd = fd; - debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd); + debugs(47, 3, HERE << "Cache Dir #" << index << " log opened on FD " << fd); } FILE * -UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) +Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) { char *swaplog_path = xstrdup(logFile(NULL)); char *clean_path = xstrdup(logFile(".last-clean")); @@ -781,7 +833,7 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) int fd; if (::stat(swaplog_path, &log_sb) < 0) { - debugs(47, 1, "Cache Dir #" << index << ": No log file"); + debugs(47, DBG_IMPORTANT, HERE << "Cache Dir #" << index << ": No log file"); safe_free(swaplog_path); safe_free(clean_path); safe_free(new_path); @@ -798,8 +850,8 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); if (fd < 0) { - debugs(50, 1, "" << new_path << ": " << xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); + debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerror()); + fatal("UFSSwapDir::openTmpSwapLog: Failed to open swap log."); } swaplog_fd = fd; @@ -820,7 +872,7 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) fp = fopen(swaplog_path, "rb"); if (fp == NULL) { - debugs(50, 0, "" << swaplog_path << ": " << xstrerror()); + debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerror()); fatal("Failed to open swap log for reading"); } @@ -844,37 +896,13 @@ UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) return fp; } -class UFSCleanLog : public SwapDir::CleanLog -{ - -public: - UFSCleanLog(SwapDir *); - virtual const StoreEntry *nextEntry(); - virtual void write(StoreEntry const &); - char *cur; - char *newLog; - char *cln; - char *outbuf; - off_t outbuf_offset; - int fd; - RemovalPolicyWalker *walker; - SwapDir *sd; -}; - -#define CLEAN_BUF_SZ 16384 - - -UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), - outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) -{} - /* * Begin the process to write clean cache state. For AUFS this means * opening some log files and allocating write buffers. Return 0 if * we succeed, and assign the 'func' and 'data' return pointers. */ int -UFSSwapDir::writeCleanStart() +Fs::Ufs::UFSSwapDir::writeCleanStart() { UFSCleanLog *state = new UFSCleanLog(this); StoreSwapLogHeader header; @@ -905,7 +933,7 @@ UFSSwapDir::writeCleanStart() state->walker = repl->WalkInit(repl); ::unlink(state->cln); - debugs(47, 3, "storeDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); + debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd); #if HAVE_FCHMOD if (::stat(state->cur, &sb) == 0) @@ -918,62 +946,8 @@ UFSSwapDir::writeCleanStart() return 0; } -/* - * Get the next entry that is a candidate for clean log writing - */ -const StoreEntry * -UFSCleanLog::nextEntry() -{ - const StoreEntry *entry = NULL; - - if (walker) - entry = walker->Next(walker); - - return entry; -} - -/* - * "write" an entry to the clean log file. - */ -void -UFSCleanLog::write(StoreEntry const &e) -{ - StoreSwapLogData s; - static size_t ss = sizeof(StoreSwapLogData); - s.op = (char) SWAP_LOG_ADD; - s.swap_filen = e.swap_filen; - s.timestamp = e.timestamp; - s.lastref = e.lastref; - s.expires = e.expires; - s.lastmod = e.lastmod; - s.swap_file_sz = e.swap_file_sz; - s.refcount = e.refcount; - s.flags = e.flags; - memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); - s.finalize(); - memcpy(outbuf + outbuf_offset, &s, ss); - outbuf_offset += ss; - /* buffered write */ - - if (outbuf_offset + ss >= CLEAN_BUF_SZ) { - if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { - /* XXX This error handling should probably move up to the caller */ - debugs(50, 0, "storeDirWriteCleanLogs: " << newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(fd); - fd = -1; - unlink(newLog); - sd->cleanLog = NULL; - delete this; - return; - } - - outbuf_offset = 0; - } -} - void -UFSSwapDir::writeCleanDone() +Fs::Ufs::UFSSwapDir::writeCleanDone() { UFSCleanLog *state = (UFSCleanLog *)cleanLog; int fd; @@ -987,8 +961,8 @@ UFSSwapDir::writeCleanDone() state->walker->Done(state->walker); if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debugs(50, 0, "storeDirWriteCleanLogs: " << state->newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); + debugs(50, DBG_CRITICAL, HERE << state->newLog << ": write: " << xstrerror()); + debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); file_close(state->fd); state->fd = -1; ::unlink(state->newLog); @@ -1038,126 +1012,8 @@ UFSSwapDir::writeCleanDone() cleanLog = NULL; } -static void -FreeObject(void *address) -{ - StoreSwapLogData *anObject = static_cast (address); - delete anObject; -} - -void -UFSSwapDir::logEntry(const StoreEntry & e, int op) const -{ - StoreSwapLogData *s = new StoreSwapLogData; - s->op = (char) op; - s->swap_filen = e.swap_filen; - s->timestamp = e.timestamp; - s->lastref = e.lastref; - s->expires = e.expires; - s->lastmod = e.lastmod; - s->swap_file_sz = e.swap_file_sz; - s->refcount = e.refcount; - s->flags = e.flags; - memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); - s->finalize(); - file_write(swaplog_fd, - -1, - s, - sizeof(StoreSwapLogData), - NULL, - NULL, - FreeObject); -} - -static QS rev_int_sort; -static int -rev_int_sort(const void *A, const void *B) -{ - const int *i1 = (const int *)A; - const int *i2 = (const int *)B; - return *i2 - *i1; -} - -int -UFSSwapDir::DirClean(int swap_index) -{ - DIR *dir_pointer = NULL; - - LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); - LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); - - int files[20]; - int swapfileno; - int fn; /* same as swapfileno, but with dirn bits set */ - int n = 0; - int k = 0; - int N0, N1, N2; - int D0, D1, D2; - UFSSwapDir *SD; - N0 = NumberOfUFSDirs; - D0 = UFSDirToGlobalDirMapping[swap_index % N0]; - SD = dynamic_cast(INDEXSD(D0)); - assert (SD); - N1 = SD->l1; - D1 = (swap_index / N0) % N1; - N2 = SD->l2; - D2 = ((swap_index / N0) / N1) % N2; - snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", - SD->path, D1, D2); - debugs(36, 3, "storeDirClean: Cleaning directory " << p1); - dir_pointer = opendir(p1); - - if (dir_pointer == NULL) { - if (errno == ENOENT) { - debugs(36, 0, "storeDirClean: WARNING: Creating " << p1); - if (mkdir(p1, 0777) == 0) - return 0; - } - - debugs(50, 0, "storeDirClean: " << p1 << ": " << xstrerror()); - safeunlink(p1, 1); - return 0; - } - - dirent_t *de; - while ((de = readdir(dir_pointer)) != NULL && k < 20) { - if (sscanf(de->d_name, "%X", &swapfileno) != 1) - continue; - - fn = swapfileno; /* XXX should remove this cruft ! */ - - if (SD->validFileno(fn, 1)) - if (SD->mapBitTest(fn)) - if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) - continue; - - files[k] = swapfileno; - ++k; - } - - closedir(dir_pointer); - - if (k == 0) - return 0; - - qsort(files, k, sizeof(int), rev_int_sort); - - if (k > 10) - k = 10; - - for (n = 0; n < k; ++n) { - debugs(36, 3, "storeDirClean: Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); - snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); - safeunlink(p2, 0); - ++statCounter.swap.files_cleaned; - } - - debugs(36, 3, "Cleaned " << k << " unused files from " << p1); - return k; -} - void -UFSSwapDir::CleanEvent(void *unused) +Fs::Ufs::UFSSwapDir::CleanEvent(void *unused) { static int swap_index = 0; int i; @@ -1213,22 +1069,20 @@ UFSSwapDir::CleanEvent(void *unused) 15.0 * exp(-0.25 * n), 1); } -int -UFSSwapDir::IsUFSDir(SwapDir * sd) +bool +Fs::Ufs::UFSSwapDir::IsUFSDir(SwapDir * sd) { UFSSwapDir *mySD = dynamic_cast(sd); - return mySD ? 1 : 0 ; + return (mySD != 0) ; } /* - * Does swapfile number 'fn' belong in cachedir #F0, - * level1 dir #F1, level2 dir #F2? * XXX: this is broken - it assumes all cache dirs use the same * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - * if not UFSSwapDir return 0; */ -int -UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) +bool +Fs::Ufs::UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) { int D1, D2; int L1, L2; @@ -1257,9 +1111,8 @@ UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) return 1; } - int -UFSSwapDir::validFileno(sfileno filn, int flag) const +Fs::Ufs::UFSSwapDir::validFileno(sfileno filn, int flag) const { if (filn < 0) return 0; @@ -1275,19 +1128,10 @@ UFSSwapDir::validFileno(sfileno filn, int flag) const return 1; } - - -/* - * UFSSwapDir::unlinkFile - * - * This routine unlinks a file and pulls it out of the bitmap. - * It used to be in commonUfsUnlink(), however an interface change - * forced this bit of code here. Eeek. - */ void -UFSSwapDir::unlinkFile(sfileno f) +Fs::Ufs::UFSSwapDir::unlinkFile(sfileno f) { - debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') << + debugs(79, 3, HERE << "unlinking fileno " << std::setfill('0') << std::hex << std::uppercase << std::setw(8) << f << " '" << fullPath(f,NULL) << "'"); /* commonUfsDirMapBitReset(this, f); */ @@ -1295,16 +1139,16 @@ UFSSwapDir::unlinkFile(sfileno f) } bool -UFSSwapDir::unlinkdUseful() const +Fs::Ufs::UFSSwapDir::unlinkdUseful() const { // unlinkd may be useful only in workers return IamWorkerProcess() && IO->io->unlinkdUseful(); } void -UFSSwapDir::unlink(StoreEntry & e) +Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e) { - debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<< + debugs(79, 3, HERE << "dirno " << index << ", fileno "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); if (e.swap_status == SWAPOUT_DONE) { cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); @@ -1315,21 +1159,16 @@ UFSSwapDir::unlink(StoreEntry & e) UFSSwapDir::unlinkFile(e.swap_filen); } -/* - * Add and remove the given StoreEntry from the replacement policy in - * use. - */ - void -UFSSwapDir::replacementAdd(StoreEntry * e) +Fs::Ufs::UFSSwapDir::replacementAdd(StoreEntry * e) { - debugs(47, 4, "UFSSwapDir::replacementAdd: added node " << e << " to dir " << index); + debugs(47, 4, HERE << "added node " << e << " to dir " << index); repl->Add(repl, e, &e->repl); } void -UFSSwapDir::replacementRemove(StoreEntry * e) +Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e) { StorePointer SD; @@ -1340,20 +1179,20 @@ UFSSwapDir::replacementRemove(StoreEntry * e) assert (dynamic_cast(SD.getRaw()) == this); - debugs(47, 4, "UFSSwapDir::replacementRemove: remove node " << e << " from dir " << index); + debugs(47, 4, HERE << "remove node " << e << " from dir " << index); repl->Remove(repl, e, &e->repl); } void -UFSSwapDir::dump(StoreEntry & entry) const +Fs::Ufs::UFSSwapDir::dump(StoreEntry & entry) const { storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2); dumpOptions(&entry); } char * -UFSSwapDir::fullPath(sfileno filn, char *fullpath) const +Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const { LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); int L1 = l1; @@ -1374,84 +1213,138 @@ UFSSwapDir::fullPath(sfileno filn, char *fullpath) const } int -UFSSwapDir::callback() +Fs::Ufs::UFSSwapDir::callback() { return IO->callback(); } void -UFSSwapDir::sync() +Fs::Ufs::UFSSwapDir::sync() { IO->sync(); } void -UFSSwapDir::swappedOut(const StoreEntry &e) +Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e) { cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); ++n_disk_objects; } StoreSearch * -UFSSwapDir::search(String const url, HttpRequest *request) +Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *request) { if (url.size()) fatal ("Cannot search by url yet\n"); - return new StoreSearchUFS (this); -} - -CBDATA_CLASS_INIT(StoreSearchUFS); -StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), current (NULL), _done (false) -{} - -/* do not link -StoreSearchUFS::StoreSearchUFS(StoreSearchUFS const &); -*/ - -StoreSearchUFS::~StoreSearchUFS() -{ - walker->Done(walker); - walker = NULL; + return new Fs::Ufs::StoreSearchUFS (this); } void -StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) +Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const { - next(); - aCallback(aCallbackArgs); + StoreSwapLogData *s = new StoreSwapLogData; + s->op = (char) op; + s->swap_filen = e.swap_filen; + s->timestamp = e.timestamp; + s->lastref = e.lastref; + s->expires = e.expires; + s->lastmod = e.lastmod; + s->swap_file_sz = e.swap_file_sz; + s->refcount = e.refcount; + s->flags = e.flags; + memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); + s->finalize(); + file_write(swaplog_fd, + -1, + s, + sizeof(StoreSwapLogData), + NULL, + NULL, + FreeObject); } -bool -StoreSearchUFS::next() +int +Fs::Ufs::UFSSwapDir::DirClean(int swap_index) { - /* the walker API doesn't make sense. the store entries referred to are already readwrite - * from their hash table entries - */ + DIR *dir_pointer = NULL; - if (walker) - current = const_cast(walker->Next(walker)); + LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); + LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); - if (current == NULL) - _done = true; + int files[20]; + int swapfileno; + int fn; /* same as swapfileno, but with dirn bits set */ + int n = 0; + int k = 0; + int N0, N1, N2; + int D0, D1, D2; + UFSSwapDir *SD; + N0 = NumberOfUFSDirs; + D0 = UFSDirToGlobalDirMapping[swap_index % N0]; + SD = dynamic_cast(INDEXSD(D0)); + assert (SD); + N1 = SD->l1; + D1 = (swap_index / N0) % N1; + N2 = SD->l2; + D2 = ((swap_index / N0) / N1) % N2; + snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", + SD->path, D1, D2); + debugs(36, 3, HERE << "Cleaning directory " << p1); + dir_pointer = opendir(p1); - return current != NULL; -} + if (dir_pointer == NULL) { + if (errno == ENOENT) { + debugs(36, DBG_CRITICAL, HERE << "WARNING: Creating " << p1); +#if _SQUID_MSWIN_ -bool -StoreSearchUFS::error() const -{ - return false; -} + if (mkdir(p1) == 0) +#else -bool -StoreSearchUFS::isDone() const -{ - return _done; -} + if (mkdir(p1, 0777) == 0) +#endif -StoreEntry * -StoreSearchUFS::currentItem() -{ - return current; + return 0; + } + + debugs(50, DBG_CRITICAL, HERE << p1 << ": " << xstrerror()); + safeunlink(p1, 1); + return 0; + } + + dirent_t *de; + while ((de = readdir(dir_pointer)) != NULL && k < 20) { + if (sscanf(de->d_name, "%X", &swapfileno) != 1) + continue; + + fn = swapfileno; /* XXX should remove this cruft ! */ + + if (SD->validFileno(fn, 1)) + if (SD->mapBitTest(fn)) + if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) + continue; + + files[k] = swapfileno; + ++k; + } + + closedir(dir_pointer); + + if (k == 0) + return 0; + + qsort(files, k, sizeof(int), rev_int_sort); + + if (k > 10) + k = 10; + + for (n = 0; n < k; ++n) { + debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); + snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); + safeunlink(p2, 0); + ++statCounter.swap.files_cleaned; + } + + debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1); + return k; } diff --git a/src/fs/ufs/UFSSwapDir.h b/src/fs/ufs/UFSSwapDir.h new file mode 100644 index 0000000000..34bdeb0ee3 --- /dev/null +++ b/src/fs/ufs/UFSSwapDir.h @@ -0,0 +1,191 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#ifndef SQUID_FS_UFS_UFSSWAPDIR_H +#define SQUID_FS_UFS_UFSSWAPDIR_H + +#include "SquidString.h" +#include "Store.h" +#include "StoreIOState.h" +#include "StoreSearch.h" +#include "SwapDir.h" +#include "swap_log_op.h" +#include "UFSStrategy.h" + +class HttpRequest; +class ConfigOptionVector; +class FileMap; +class DiskIOModule; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSSwapDir : public SwapDir +{ +public: + static bool IsUFSDir(SwapDir* sd); + static int DirClean(int swap_index); + /** check whether swapfile belongs to the specified cachedir/l1dir/l2dir + * + * \param cachedir the number of the cachedir which is being tested + * \param level1dir level-1 dir in the cachedir + * \param level2dir level-2 dir + */ + static bool FilenoBelongsHere(int fn, int cachedir, int level1dir, int level2dir); + + UFSSwapDir(char const *aType, const char *aModuleType); + /** Initial setup / end destruction */ + virtual void init(); + /** Create a new SwapDir (-z command-line option) */ + virtual void create(); + virtual void dump(StoreEntry &) const; + ~UFSSwapDir(); + virtual StoreSearch *search(String const url, HttpRequest *); + /** double-check swap during rebuild (-S command-line option) + * + * called by storeCleanup if needed + */ + virtual bool doubleCheck(StoreEntry &); + virtual bool unlinkdUseful() const; + /** unlink a file, and remove its entry from the filemap */ + virtual void unlink(StoreEntry &); + virtual void statfs(StoreEntry &)const; + virtual void maintain(); + /** check whether this filesystem can store the given object + * + * UFS filesystems will happily store anything as long as + * the LRU time isn't too small + */ + virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; + /** reference an object + * + * This routine is called whenever an object is referenced, so we can + * maintain replacement information within the storage fs. + */ + virtual void reference(StoreEntry &); + /** de-reference an object + * + * This routine is called whenever the last reference to an object is + * removed, to maintain replacement information within the storage fs. + */ + virtual bool dereference(StoreEntry &); + virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); + virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); + virtual void openLog(); + virtual void closeLog(); + virtual int writeCleanStart(); + virtual void writeCleanDone(); + virtual void logEntry(const StoreEntry & e, int op) const; + virtual void parse(int index, char *path); ///parse configuration and setup new SwapDir + virtual void reconfigure(); ///reconfigure the SwapDir + virtual int callback(); + virtual void sync(); + virtual void swappedOut(const StoreEntry &e); + virtual uint64_t currentSize() const { return cur_size; } + virtual uint64_t currentCount() const { return n_disk_objects; } + + void unlinkFile(sfileno f); + // move down when unlink is a virtual method + //protected: + Fs::Ufs::UFSStrategy *IO; + char *fullPath(sfileno, char *) const; + /* temp */ + void closeTmpSwapLog(); + FILE *openTmpSwapLog(int *clean_flag, int *zero_flag); + char *swapSubDir(int subdirn) const; + int mapBitTest(sfileno filn); + void mapBitReset(sfileno filn); + void mapBitSet(sfileno filn); + /** Add a new object to the cache with empty memory copy and pointer to disk + * + * This method is used to rebuild a store from disk + */ + StoreEntry *addDiskRestore(const cache_key * key, + sfileno file_number, + uint64_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + uint32_t refcount, + uint16_t flags, + int clean); + /// Undo the effects of UFSSwapDir::addDiskRestore(). + void undoAddDiskRestore(StoreEntry *e); + int validFileno(sfileno filn, int flag) const; + int mapBitAllocate(); + virtual ConfigOption *getOptionTree() const; + + void *fsdata; + + bool validL2(int) const; + bool validL1(int) const; + + /** Add and remove the given StoreEntry from the replacement policy in use */ + void replacementAdd(StoreEntry *e); + void replacementRemove(StoreEntry *e); + +protected: + FileMap *map; + int suggest; + int l1; + int l2; + +private: + void parseSizeL1L2(); + static int NumberOfUFSDirs; + static int * UFSDirToGlobalDirMapping; + bool pathIsDirectory(const char *path)const; + int swaplog_fd; + static EVH CleanEvent; + /** Verify that the the CacheDir exists + * + * If this returns < 0, then Squid exits, complains about swap + * directories not existing, and instructs the admin to run 'squid -z' + * Called by UFSSwapDir::init + */ + bool verifyCacheDirs(); + void rebuild(); + int createDirectory(const char *path, int); + void createSwapSubDirs(); + void dumpEntry(StoreEntry &) const; + char *logFile(char const *ext = NULL)const; + void changeIO(DiskIOModule *); + bool optionIOParse(char const *option, const char *value, int reconfiguring); + void optionIODump(StoreEntry * e) const; + mutable ConfigOptionVector *currentIOOptions; + char const *ioType; + uint64_t cur_size; ///< currently used space in the storage area + uint64_t n_disk_objects; ///< total number of objects stored +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_UFSSWAPDIR_H */ diff --git a/src/fs/ufs/UFSSwapLogParser.cc b/src/fs/ufs/UFSSwapLogParser.cc new file mode 100644 index 0000000000..619e2ffbec --- /dev/null +++ b/src/fs/ufs/UFSSwapLogParser.cc @@ -0,0 +1,199 @@ +/* + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + * + */ + +#include "squid.h" +#include "md5.h" +#include "StoreSwapLogData.h" +#include "swap_log_op.h" +#include "UFSSwapLogParser.h" + +/// Parse a swap header entry created on a system with 32-bit size_t and sfileno +/// this is typical of 32-bit systems without large file support +/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still. +class UFSSwapLogParser_v1_32bs:public Fs::Ufs::UFSSwapLogParser +{ +public: + /// version 1 cache swap.state entry with 32-bit size_t (swap_file_sz) + /// time_t an sfileno have no variation from the v1 baseline format + struct StoreSwapLogDataOld { + char op; + sfileno swap_filen; + time_t timestamp; + time_t lastref; + time_t expires; + time_t lastmod; + uint32_t swap_file_sz; + uint16_t refcount; + uint16_t flags; + unsigned char key[SQUID_MD5_DIGEST_LENGTH]; + }; + UFSSwapLogParser_v1_32bs(FILE *fp):Fs::Ufs::UFSSwapLogParser(fp) { + record_size = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); + } + /// Convert the on-disk 32-bit format to our current format while reading + bool ReadRecord(StoreSwapLogData &swapData) { + UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld readData; + int bytes = sizeof(UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld); + + assert(log); + + if (fread(&readData, bytes, 1, log) != 1) { + return false; + } + swapData.op = readData.op; + swapData.swap_filen = readData.swap_filen; + swapData.timestamp = readData.timestamp; + swapData.lastref = readData.lastref; + swapData.expires = readData.expires; + swapData.lastmod = readData.lastmod; + swapData.swap_file_sz = readData.swap_file_sz; + swapData.refcount = readData.refcount; + swapData.flags = readData.flags; + memcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); + return true; + } +}; + +/// swap.state v2 log parser +class UFSSwapLogParser_v2: public Fs::Ufs::UFSSwapLogParser +{ +public: + UFSSwapLogParser_v2(FILE *fp): Fs::Ufs::UFSSwapLogParser(fp) { + record_size = sizeof(StoreSwapLogData); + } + bool ReadRecord(StoreSwapLogData &swapData) { + assert(log); + return fread(&swapData, sizeof(StoreSwapLogData), 1, log) == 1; + } +}; + +Fs::Ufs::UFSSwapLogParser * +Fs::Ufs::UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) +{ + StoreSwapLogHeader header; + + assert(fp); + + if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) + return NULL; + + if (header.op != SWAP_LOG_VERSION) { + debugs(47, DBG_IMPORTANT, "Old swap file detected..."); + fseek(fp, 0, SEEK_SET); + return new UFSSwapLogParser_v1_32bs(fp); // Um. 32-bits except time_t, and can't determine that. + } + + debugs(47, 2, "Swap file version: " << header.version); + + if (header.version == 1) { + if (fseek(fp, header.record_size, SEEK_SET) != 0) + return NULL; + + debugs(47, DBG_IMPORTANT, "Rejecting swap file v1 to avoid cache " << + "index corruption. Forcing a full cache index rebuild. " << + "See Squid bug #3441."); + return NULL; + +#if UNUSED_CODE + // baseline + // 32-bit sfileno + // native time_t (hopefully 64-bit) + // 64-bit file size + if (header.record_size == sizeof(StoreSwapLogData)) { + debugs(47, DBG_IMPORTANT, "Version 1 of swap file with LFS support detected... "); + return new UFSSwapLogParser_v1(fp); + } + + // which means we have a 3-way grid of permutations to import (yuck!) + // 1) sfileno 32-bit / 64-bit (64-bit was broken) + // 2) time_t 32-bit / 64-bit + // 3) size_t 32-bit / 64-bit (32-bit was pre-LFS) + + // 32-bit systems... + // only LFS (size_t) differs from baseline + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bs::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (32-bit) swap file without LFS support detected... "); + return new UFSSwapLogParser_v1_32bs(fp); + } + // LFS (size_t) and timestamps (time_t) differs from baseline + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_32bst::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (32-bit) swap file with short timestamps and without LFS support detected... "); + return new UFSSwapLogParser_v1_32bst(fp); + } + // No downgrade for 64-bit timestamps to 32-bit. + + // 64-bit systems + // sfileno was 64-bit for a some builds + if (header.record_size == sizeof(struct UFSSwapLogParser_v1_64bfn::StoreSwapLogDataOld)) { + debugs(47, DBG_IMPORTANT, "Version 1 (64-bit) swap file with broken sfileno detected... "); + return new UFSSwapLogParser_v1_64bfn(fp); + } + // NP: 64-bit system with 32-bit size_t/time_t are not handled. + + debugs(47, DBG_IMPORTANT, "WARNING: The swap file has wrong format!... "); + debugs(47, DBG_IMPORTANT, "NOTE: Cannot safely downgrade caches to short (32-bit) timestamps."); + return NULL; +#endif + } + + if (header.version >= 2) { + if (!header.sane()) { + debugs(47, DBG_IMPORTANT, "ERROR: Corrupted v" << header.version << + " swap file header."); + return NULL; + } + + if (fseek(fp, header.record_size, SEEK_SET) != 0) + return NULL; + + if (header.version == 2) + return new UFSSwapLogParser_v2(fp); + } + + // TODO: v3: write to disk in network-order bytes for the larger fields? + + debugs(47, DBG_IMPORTANT, "Unknown swap file version: " << header.version); + return NULL; +} + +int +Fs::Ufs::UFSSwapLogParser::SwapLogEntries() +{ + struct stat sb; + + if (log_entries >= 0) + return log_entries; + + if (log && record_size && 0 == fstat(fileno(log), &sb)) { + log_entries = sb.st_size/record_size; + return log_entries; + } + + return 0; +} diff --git a/src/fs/ufs/ufscommon.cci b/src/fs/ufs/UFSSwapLogParser.h similarity index 65% rename from src/fs/ufs/ufscommon.cci rename to src/fs/ufs/UFSSwapLogParser.h index d54c3316e3..3163f760dd 100644 --- a/src/fs/ufs/ufscommon.cci +++ b/src/fs/ufs/UFSSwapLogParser.h @@ -1,9 +1,4 @@ /* - * $Id$ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- * @@ -32,3 +27,41 @@ * */ +#ifndef SQUID_FS_UFS_UFSSWAPLOGPARSER_H +#define SQUID_FS_UFS_UFSSWAPLOGPARSER_H + +#include + +class StoreSwapLogData; + +namespace Fs +{ +namespace Ufs +{ +/// \ingroup UFS +class UFSSwapLogParser +{ +public: + FILE *log; + int log_entries; + int record_size; + + UFSSwapLogParser(FILE *fp):log(fp),log_entries(-1), record_size(0) { + } + virtual ~UFSSwapLogParser() {}; + + static UFSSwapLogParser *GetUFSSwapLogParser(FILE *fp); + + virtual bool ReadRecord(StoreSwapLogData &swapData) = 0; + int SwapLogEntries(); + void Close() { + if (log) { + fclose(log); + log = NULL; + } + } +}; + +} //namespace Ufs +} //namespace Fs +#endif /* SQUID_FS_UFS_UFSSWAPLOGPARSER_H */ diff --git a/src/fs/ufs/ufscommon.h b/src/fs/ufs/ufscommon.h deleted file mode 100644 index 578af1f10f..0000000000 --- a/src/fs/ufs/ufscommon.h +++ /dev/null @@ -1,428 +0,0 @@ -/* - * $Id$ - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ -#ifndef SQUID_UFSCOMMON_H -#define SQUID_UFSCOMMON_H - - -#define DefaultLevelOneDirs 16 -#define DefaultLevelTwoDirs 256 -#define STORE_META_BUFSZ 4096 - -class UFSStrategy; -class ConfigOptionVector; -class DiskIOModule; -class StoreSearch; -class FileMap; - -#include "SwapDir.h" - -/// \ingroup UFS -class UFSSwapDir : public SwapDir -{ - -public: - static int IsUFSDir(SwapDir* sd); - static int DirClean(int swap_index); - static int FilenoBelongsHere(int fn, int F0, int F1, int F2); - - UFSSwapDir(char const *aType, const char *aModuleType); - virtual void init(); - virtual void create(); - virtual void dump(StoreEntry &) const; - ~UFSSwapDir(); - virtual StoreSearch *search(String const url, HttpRequest *); - virtual bool doubleCheck(StoreEntry &); - virtual bool unlinkdUseful() const; - virtual void unlink(StoreEntry &); - virtual void statfs(StoreEntry &)const; - virtual void maintain(); - virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const; - virtual void reference(StoreEntry &); - virtual bool dereference(StoreEntry &); - virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); - virtual void openLog(); - virtual void closeLog(); - virtual int writeCleanStart(); - virtual void writeCleanDone(); - virtual void logEntry(const StoreEntry & e, int op) const; - virtual void parse(int index, char *path); - virtual void reconfigure(); - virtual int callback(); - virtual void sync(); - virtual void swappedOut(const StoreEntry &e); - virtual uint64_t currentSize() const { return cur_size; } - virtual uint64_t currentCount() const { return n_disk_objects; } - - void unlinkFile(sfileno f); - // move down when unlink is a virtual method - //protected: - UFSStrategy *IO; - char *fullPath(sfileno, char *) const; - /* temp */ - void closeTmpSwapLog(); - FILE *openTmpSwapLog(int *clean_flag, int *zero_flag); - char *swapSubDir(int subdirn) const; - int mapBitTest(sfileno filn); - void mapBitReset(sfileno filn); - void mapBitSet(sfileno filn); - StoreEntry *addDiskRestore(const cache_key * key, - sfileno file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t flags, - int clean); - /// Undo the effects of UFSSwapDir::addDiskRestore(). - void undoAddDiskRestore(StoreEntry *e); - int validFileno(sfileno filn, int flag) const; - int mapBitAllocate(); - virtual ConfigOption *getOptionTree() const; - - void *fsdata; - - bool validL2(int) const; - bool validL1(int) const; - - void replacementAdd(StoreEntry *e); - void replacementRemove(StoreEntry *e); - -protected: - FileMap *map; - int suggest; - int l1; - int l2; - -private: - void parseSizeL1L2(); - static int NumberOfUFSDirs; - static int * UFSDirToGlobalDirMapping; - bool pathIsDirectory(const char *path)const; - int swaplog_fd; - static EVH CleanEvent; - bool verifyCacheDirs(); - void rebuild(); - int createDirectory(const char *path, int); - void createSwapSubDirs(); - void dumpEntry(StoreEntry &) const; - char *logFile(char const *ext = NULL)const; - void changeIO(DiskIOModule *); - bool optionIOParse(char const *option, const char *value, int reconfiguring); - void optionIODump(StoreEntry * e) const; - mutable ConfigOptionVector *currentIOOptions; - char const *ioType; - uint64_t cur_size; ///< currently used space in the storage area - uint64_t n_disk_objects; ///< total number of objects stored -}; - -#include "RefCount.h" -#include "DiskIO/IORequestor.h" - -/** - * UFS dir specific IO calls - * - \todo This should be whittled away. - * DiskIOModule should be providing the entire needed API. - */ - -class DiskIOStrategy; - -class DiskFile; - -/// \ingroup UFS -class UFSStrategy -{ - -public: - UFSStrategy (DiskIOStrategy *); - virtual ~UFSStrategy (); - /* Not implemented */ - UFSStrategy (UFSStrategy const &); - UFSStrategy &operator=(UFSStrategy const &); - - virtual bool shedLoad(); - - virtual int load(); - - StoreIOState::Pointer createState(SwapDir *SD, StoreEntry *e, StoreIOState::STIOCB * callback, void *callback_data) const; - /* UFS specific */ - virtual RefCount newFile (char const *path); - StoreIOState::Pointer open(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, - StoreIOState::STIOCB *, void *); - StoreIOState::Pointer create(SwapDir *, StoreEntry *, StoreIOState::STFNCB *, - StoreIOState::STIOCB *, void *); - - virtual void unlinkFile (char const *); - virtual void sync(); - - virtual int callback(); - - /** Init per-instance logic */ - virtual void init(); - - /** cachemgr output on the IO instance stats */ - virtual void statfs(StoreEntry & sentry)const; - - /** The io strategy in use */ - DiskIOStrategy *io; -protected: - - friend class UFSSwapDir; -}; - -/** Common ufs-store-dir logic */ - -class ReadRequest; - -/// \ingroup UFS -class UFSStoreState : public StoreIOState, public IORequestor -{ - -public: - void * operator new (size_t); - void operator delete (void *); - UFSStoreState(SwapDir * SD, StoreEntry * anEntry, STIOCB * callback_, void *callback_data_); - ~UFSStoreState(); - virtual void close(int how); - virtual void closeCompleted(); - // protected: - virtual void ioCompletedNotification(); - virtual void readCompleted(const char *buf, int len, int errflag, RefCount); - virtual void writeCompleted(int errflag, size_t len, RefCount); - RefCount theFile; - bool opening; - bool creating; - bool closing; - bool reading; - bool writing; - void read_(char *buf, size_t size, off_t offset, STRCB * callback, void *callback_data); - void write(char const *buf, size_t size, off_t offset, FREE * free_func); - -protected: - virtual void doCloseCallback (int errflag); - - class _queued_read - { - - public: - MEMPROXY_CLASS(UFSStoreState::_queued_read); - char *buf; - size_t size; - off_t offset; - STRCB *callback; - void *callback_data; - - }; - - class _queued_write - { - - public: - MEMPROXY_CLASS(UFSStoreState::_queued_write); - char const *buf; - size_t size; - off_t offset; - FREE *free_func; - - }; - - /** \todo These should be in the IO strategy */ - - struct { - /** - * DPW 2006-05-24 - * the write_draining flag is used to avoid recursion inside - * the UFSStoreState::drainWriteQueue() method. - */ - bool write_draining; - /** - * DPW 2006-05-24 - * The try_closing flag is set by UFSStoreState::tryClosing() - * when UFSStoreState wants to close the file, but cannot - * because of pending I/Os. If set, UFSStoreState will - * try to close again in the I/O callbacks. - */ - bool try_closing; - } flags; - link_list *pending_reads; - link_list *pending_writes; - void queueRead(char *, size_t, off_t, STRCB *, void *); - void queueWrite(char const *, size_t, off_t, FREE *); - bool kickReadQueue(); - void drainWriteQueue(); - void tryClosing(); - char *read_buf; - -private: - void openDone(); - void freePending(); - void doWrite(); - - CBDATA_CLASS(UFSStoreState); -}; - -MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_read); -MEMPROXY_CLASS_INLINE(UFSStoreState::_queued_write); - - -#include "StoreSearch.h" - -/// \ingroup UFS -class StoreSearchUFS : public StoreSearch -{ - -public: - StoreSearchUFS(RefCount sd); - StoreSearchUFS(StoreSearchUFS const &); - virtual ~StoreSearchUFS(); - - /** \todo Iterator API - garh, wrong place */ - /** - * callback the client when a new StoreEntry is available - * or an error occurs - */ - virtual void next(void (callback)(void *cbdata), void *cbdata); - - /** - \retval true if a new StoreEntry is immediately available - \retval false if a new StoreEntry is NOT immediately available - */ - virtual bool next(); - - virtual bool error() const; - virtual bool isDone() const; - virtual StoreEntry *currentItem(); - - RefCount sd; - RemovalPolicyWalker *walker; - -private: - /// \bug (callback) should be hidden behind a proper human readable name - void (callback)(void *cbdata); - void *cbdata; - StoreEntry * current; - bool _done; - - CBDATA_CLASS2(StoreSearchUFS); -}; - - -class StoreSwapLogData; - -/// \ingroup UFS -class UFSSwapLogParser -{ - -public: - FILE *log; - int log_entries; - int record_size; - - UFSSwapLogParser(FILE *fp):log(fp),log_entries(-1), record_size(0) { - } - virtual ~UFSSwapLogParser() {}; - - static UFSSwapLogParser *GetUFSSwapLogParser(FILE *fp); - - virtual bool ReadRecord(StoreSwapLogData &swapData) = 0; - int SwapLogEntries(); - void Close() { - if (log) { - fclose(log); - log = NULL; - } - } -}; - - -/// \ingroup UFS -class RebuildState : public RefCountable -{ - -public: - static EVH RebuildStep; - - RebuildState(RefCount sd); - ~RebuildState(); - - virtual bool error() const; - virtual bool isDone() const; - virtual StoreEntry *currentItem(); - - RefCount sd; - int n_read; - /* FILE *log;*/ - UFSSwapLogParser *LogParser; - int curlvl1; - int curlvl2; - - struct { - unsigned int need_to_validate:1; - unsigned int clean:1; - unsigned int init:1; - } flags; - int in_dir; - int done; - int fn; - - dirent_t *entry; - DIR *td; - char fullpath[MAXPATHLEN]; - char fullfilename[MAXPATHLEN]; - - struct _store_rebuild_data counts; - -private: - void rebuildFromDirectory(); - void rebuildFromSwapLog(); - void rebuildStep(); - void undoAdd(); - int getNextFile(sfileno *, int *size); - StoreEntry *currentEntry() const; - void currentEntry(StoreEntry *); - StoreEntry *e; - bool fromLog; - bool _done; - /// \bug (callback) should be hidden behind a proper human readable name - void (callback)(void *cbdata); - void *cbdata; - - CBDATA_CLASS2(RebuildState); -}; - -#if _USE_INLINE_ -#include "ufscommon.cci" -#endif - -#endif /* SQUID_UFSCOMMON_H */ diff --git a/src/tests/testCoss.cc b/src/tests/testCoss.cc index 3fd5f9945e..ef68737619 100644 --- a/src/tests/testCoss.cc +++ b/src/tests/testCoss.cc @@ -4,7 +4,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #include "fs/coss/CossSwapDir.h" #include "Mem.h" #include "MemObject.h" diff --git a/src/tests/testDiskIO.cc b/src/tests/testDiskIO.cc index 343b296dc8..ca3f1ff2b0 100644 --- a/src/tests/testDiskIO.cc +++ b/src/tests/testDiskIO.cc @@ -5,7 +5,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #if 0 // AYJ: COSS in Squid-3 is disabled. #include "fs/coss/CossSwapDir.h" #endif diff --git a/src/tests/testNull.cc b/src/tests/testNull.cc index a987615b13..a8d511fbf8 100644 --- a/src/tests/testNull.cc +++ b/src/tests/testNull.cc @@ -4,7 +4,6 @@ #include "Store.h" #include "SwapDir.h" #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" #include "fs/null/store_null.h" #include "Mem.h" #include "MemObject.h" diff --git a/src/tests/testUfs.cc b/src/tests/testUfs.cc index 34e129135a..c85f1beee5 100644 --- a/src/tests/testUfs.cc +++ b/src/tests/testUfs.cc @@ -1,15 +1,16 @@ #define SQUID_UNIT_TEST 1 #include "squid.h" -#include "testUfs.h" -#include "Store.h" -#include "SwapDir.h" + #include "DiskIO/DiskIOModule.h" -#include "fs/ufs/ufscommon.h" -#include "Mem.h" -#include "MemObject.h" #include "HttpHeader.h" #include "HttpReply.h" +#include "Mem.h" +#include "MemObject.h" #include "testStoreSupport.h" +#include "testUfs.h" +#include "Store.h" +#include "SwapDir.h" +#include "fs/ufs/UFSSwapDir.h" #if HAVE_STDEXCEPT #include @@ -19,7 +20,7 @@ CPPUNIT_TEST_SUITE_REGISTRATION( testUfs ); -typedef RefCount SwapDirPointer; +typedef RefCount SwapDirPointer; extern REMOVALPOLICYCREATE createRemovalPolicy_lru; /* XXX fails with --enable-removal-policies=heap */ static void @@ -89,9 +90,9 @@ testUfs::testUfsSearch() Store::Root(new StoreController); - SwapDirPointer aStore (new UFSSwapDir("ufs", "Blocking")); + SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking")); - aStore->IO = new UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy()); + aStore->IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy()); addSwapDir(aStore); @@ -232,7 +233,7 @@ testUfs::testUfsDefaultEngine() CPPUNIT_ASSERT(!store_table); // or StoreHashIndex ctor will abort below Store::Root(new StoreController); - SwapDirPointer aStore (new UFSSwapDir("ufs", "Blocking")); + SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking")); addSwapDir(aStore); commonInit(); Config.replPolicy = new RemovalPolicySettings;