From: Automatic source maintenance Date: Wed, 30 May 2012 00:24:55 +0000 (-0600) Subject: Bootstrapped X-Git-Tag: SQUID_3_2_0_18~44 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7037e95cad4cff5d374d560e67a4b0b4c4ffa020;p=thirdparty%2Fsquid.git Bootstrapped --- diff --git a/src/fs/ufs/store_dir_ufs.cc b/src/fs/ufs/store_dir_ufs.cc index 228ebcd034..e69de29bb2 100644 --- a/src/fs/ufs/store_dir_ufs.cc +++ b/src/fs/ufs/store_dir_ufs.cc @@ -1,1456 +0,0 @@ - -/* - * $Id$ - * - * DEBUG: section 47 Store Directory Routines - * AUTHOR: Duane Wessels - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#include "squid-old.h" -#include "Store.h" -#include "fde.h" -#include "ufscommon.h" -#include "StoreSwapLogData.h" -#include "ConfigOption.h" -#include "DiskIO/DiskIOStrategy.h" -#include "DiskIO/DiskIOModule.h" -#include "FileMap.h" -#include "Parsing.h" -#include "SquidMath.h" -#include "SquidTime.h" -#include "StatCounters.h" -#include "SwapDir.h" -#include "swap_log_op.h" - -int UFSSwapDir::NumberOfUFSDirs = 0; -int *UFSSwapDir::UFSDirToGlobalDirMapping = NULL; - -/* - * storeUfsDirCheckObj - * - * This routine is called by storeDirSelectSwapDir to see if the given - * object is able to be stored on this filesystem. UFS filesystems will - * happily store anything as long as the LRU time isn't too small. - */ -bool -UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const -{ - if (!SwapDir::canStore(e, diskSpaceNeeded, load)) - return false; - - if (IO->shedLoad()) - return false; - - load = IO->load(); - return true; -} - - -/* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ - -void -UFSSwapDir::parseSizeL1L2() -{ - int i = GetInteger(); - if (i <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); - - const uint64_t size = static_cast(i) << 20; // MBytes to Bytes - - /* just reconfigure it */ - if (reconfiguring) { - if (size == maxSize()) - debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); - else - debugs(3, 1, "Cache dir '" << path << "' size changed to " << i << " MB"); - } - - max_size = size; - - l1 = GetInteger(); - - if (l1 <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value"); - - l2 = GetInteger(); - - if (l2 <= 0) - fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); -} - -/* - * storeUfsDirReconfigure - * - * This routine is called when the given swapdir needs reconfiguring - */ - -void -UFSSwapDir::reconfigure() -{ - parseSizeL1L2(); - parseOptions(1); -} - -/* - * storeUfsDirParse - * - * Called when a *new* fs is being setup. - */ -void -UFSSwapDir::parse (int anIndex, char *aPath) -{ - index = anIndex; - path = xstrdup(aPath); - - parseSizeL1L2(); - - /* Initialise replacement policy stuff */ - repl = createRemovalPolicy(Config.replPolicy); - - parseOptions(0); -} - -void -UFSSwapDir::changeIO(DiskIOModule *module) -{ - DiskIOStrategy *anIO = module->createStrategy(); - safe_free(ioType); - ioType = xstrdup(module->type()); - - delete IO->io; - IO->io = anIO; - /* Change the IO Options */ - - if (currentIOOptions && currentIOOptions->options.size() > 2) - delete currentIOOptions->options.pop_back(); - - /* TODO: factor out these 4 lines */ - ConfigOption *ioOptions = IO->io->getOptionTree(); - - if (ioOptions) - currentIOOptions->options.push_back(ioOptions); -} - -bool -UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) -{ - if (strcmp(option, "IOEngine") != 0) - return false; - - if (isaReconfig) - /* silently ignore this */ - return true; - - if (!value) - self_destruct(); - - DiskIOModule *module = DiskIOModule::Find(value); - - if (!module) - self_destruct(); - - changeIO(module); - - return true; -} - -void -UFSSwapDir::optionIODump(StoreEntry * e) const -{ - storeAppendPrintf(e, " IOEngine=%s", ioType); -} - -ConfigOption * -UFSSwapDir::getOptionTree() const -{ - ConfigOption *parentResult = SwapDir::getOptionTree(); - - if (currentIOOptions == NULL) - currentIOOptions = new ConfigOptionVector(); - - currentIOOptions->options.push_back(parentResult); - - currentIOOptions->options.push_back(new ConfigOptionAdapter(*const_cast(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump)); - - if (ConfigOption *ioOptions = IO->io->getOptionTree()) - currentIOOptions->options.push_back(ioOptions); - - ConfigOption* result = currentIOOptions; - - currentIOOptions = NULL; - - return result; -} - -/* - * Initial setup / end destruction - */ -void -UFSSwapDir::init() -{ - debugs(47, 3, "Initialising UFS SwapDir engine."); - /* Parsing must be finished by now - force to NULL, don't delete */ - currentIOOptions = NULL; - static int started_clean_event = 0; - static const char *errmsg = - "\tFailed to verify one of the swap directories, Check cache.log\n" - "\tfor details. Run 'squid -z' to create swap directories\n" - "\tif needed, or if running Squid for the first time."; - IO->init(); - - if (verifyCacheDirs()) - fatal(errmsg); - - openLog(); - - rebuild(); - - if (!started_clean_event) { - eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1); - started_clean_event = 1; - } - - (void) storeDirGetBlkSize(path, &fs.blksize); -} - -void -UFSSwapDir::create() -{ - debugs(47, 3, "Creating swap space in " << path); - createDirectory(path, 0); - createSwapSubDirs(); -} - -UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) -{ - /* modulename is only set to disk modules that are built, by configure, - * so the Find call should never return NULL here. - */ - IO = new UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); -} - -UFSSwapDir::~UFSSwapDir() -{ - if (swaplog_fd > -1) { - file_close(swaplog_fd); - swaplog_fd = -1; - } - - delete map; - - if (IO) - delete IO; - - IO = NULL; - - safe_free(ioType); -} - -void -UFSSwapDir::dumpEntry(StoreEntry &e) const -{ - debugs(47, 0, "UFSSwapDir::dumpEntry: FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - debugs(47, 0, "UFSSwapDir::dumpEntry: PATH " << fullPath(e.swap_filen, NULL) ); - e.dump(0); -} - -/* - * UFSSwapDir::doubleCheck - * - * This is called by storeCleanup() if -S was given on the command line. - */ -bool -UFSSwapDir::doubleCheck(StoreEntry & e) -{ - - struct stat sb; - - if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { - debugs(47, 0, "UFSSwapDir::doubleCheck: MISSING SWAP FILE"); - dumpEntry(e); - return true; - } - - if ((off_t)e.swap_file_sz != sb.st_size) { - debugs(47, 0, "UFSSwapDir::doubleCheck: SIZE MISMATCH"); - debugs(47, 0, "UFSSwapDir::doubleCheck: ENTRY SIZE: " << e.swap_file_sz << ", FILE SIZE: " << sb.st_size); - dumpEntry(e); - return true; - } - - return false; -} - -void -UFSSwapDir::statfs(StoreEntry & sentry) const -{ - int totl_kb = 0; - int free_kb = 0; - int totl_in = 0; - int free_in = 0; - int x; - storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); - storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); - storeAppendPrintf(&sentry, "Maximum Size: %"PRIu64" KB\n", maxSize() >> 10); - storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); - storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", - Math::doublePercent(currentSize(), maxSize())); - storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", - map->numFilesInMap(), map->capacity(), - Math::intPercent(map->numFilesInMap(), map->capacity())); - x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in); - - if (0 == x) { - storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", - totl_kb - free_kb, - totl_kb, - Math::intPercent(totl_kb - free_kb, totl_kb)); - storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", - totl_in - free_in, - totl_in, - Math::intPercent(totl_in - free_in, totl_in)); - } - - storeAppendPrintf(&sentry, "Flags:"); - - if (flags.selected) - storeAppendPrintf(&sentry, " SELECTED"); - - if (flags.read_only) - storeAppendPrintf(&sentry, " READ-ONLY"); - - storeAppendPrintf(&sentry, "\n"); - - IO->statfs(sentry); -} - -void -UFSSwapDir::maintain() -{ - /* We can't delete objects while rebuilding swap */ - - /* XXX FIXME each store should start maintaining as it comes online. */ - - if (StoreController::store_dirs_rebuilding) - return; - - StoreEntry *e = NULL; - - int removed = 0; - - RemovalPurgeWalker *walker; - - double f = (double) (currentSize() - minSize()) / (maxSize() - minSize()); - - f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; - - int max_scan = (int) (f * 400.0 + 100.0); - - int max_remove = (int) (f * 70.0 + 10.0); - - /* - * This is kinda cheap, but so we need this priority hack? - */ - - debugs(47, 3, "storeMaintainSwapSpace: f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); - - walker = repl->PurgeInit(repl, max_scan); - - while (1) { - if (currentSize() < minSize()) - break; - - if (removed >= max_remove) - break; - - e = walker->Next(walker); - - if (!e) - break; /* no more objects */ - - removed++; - - e->release(); - } - - walker->Done(walker); - debugs(47, (removed ? 2 : 3), "UFSSwapDir::maintain: " << path << - " removed " << removed << "/" << max_remove << " f=" << - std::setprecision(4) << f << " max_scan=" << max_scan); -} - -/* - * UFSSwapDir::reference - * - * This routine is called whenever an object is referenced, so we can - * maintain replacement information within the storage fs. - */ -void -UFSSwapDir::reference(StoreEntry &e) -{ - debugs(47, 3, "UFSSwapDir::reference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); - - if (repl->Referenced) - repl->Referenced(repl, &e, &e.repl); -} - -/* - * UFSSwapDir::dereference - * This routine is called whenever the last reference to an object is - * removed, to maintain replacement information within the storage fs. - */ -bool -UFSSwapDir::dereference(StoreEntry & e) -{ - debugs(47, 3, "UFSSwapDir::dereference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); - - if (repl->Dereferenced) - repl->Dereferenced(repl, &e, &e.repl); - - return true; // keep e in the global store_table -} - -StoreIOState::Pointer -UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) -{ - return IO->create (this, &e, file_callback, aCallback, callback_data); -} - -StoreIOState::Pointer -UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) -{ - return IO->open (this, &e, file_callback, aCallback, callback_data); -} - -int -UFSSwapDir::mapBitTest(sfileno filn) -{ - return map->testBit(filn); -} - -void -UFSSwapDir::mapBitSet(sfileno filn) -{ - map->setBit(filn); -} - -void -UFSSwapDir::mapBitReset(sfileno filn) -{ - /* - * We have to test the bit before calling clearBit as - * it doesn't do bounds checking and blindly assumes - * filn is a valid file number, but it might not be because - * the map is dynamic in size. Also clearing an already clear - * bit puts the map counter of-of-whack. - */ - - if (map->testBit(filn)) - map->clearBit(filn); -} - -int -UFSSwapDir::mapBitAllocate() -{ - int fn; - fn = map->allocate(suggest); - map->setBit(fn); - suggest = fn + 1; - return fn; -} - -char * -UFSSwapDir::swapSubDir(int subdirn)const -{ - LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); - assert(0 <= subdirn && subdirn < l1); - snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn); - return fullfilename; -} - -int -UFSSwapDir::createDirectory(const char *aPath, int should_exist) -{ - int created = 0; - - struct stat st; - getCurrentTime(); - - if (0 == ::stat(aPath, &st)) { - if (S_ISDIR(st.st_mode)) { - debugs(47, (should_exist ? 3 : 1), aPath << " exists"); - } else { - fatalf("Swap directory %s is not a directory.", aPath); - } - -#if _SQUID_MSWIN_ - - } else if (0 == mkdir(aPath)) { -#else - - } else if (0 == mkdir(aPath, 0755)) { -#endif - debugs(47, (should_exist ? 1 : 3), aPath << " created"); - created = 1; - } else { - fatalf("Failed to make swap directory %s: %s", - aPath, xstrerror()); - } - - return created; -} - -bool -UFSSwapDir::pathIsDirectory(const char *aPath)const -{ - - struct stat sb; - - if (::stat(aPath, &sb) < 0) { - debugs(47, 0, "" << aPath << ": " << xstrerror()); - return false; - } - - if (S_ISDIR(sb.st_mode) == 0) { - debugs(47, 0, "" << aPath << " is not a directory"); - return false; - } - - return true; -} - -/* - * This function is called by commonUfsDirInit(). If this returns < 0, - * then Squid exits, complains about swap directories not - * existing, and instructs the admin to run 'squid -z' - */ -bool -UFSSwapDir::verifyCacheDirs() -{ - if (!pathIsDirectory(path)) - return true; - - for (int j = 0; j < l1; j++) { - char const *aPath = swapSubDir(j); - - if (!pathIsDirectory(aPath)) - return true; - } - - return false; -} - -void -UFSSwapDir::createSwapSubDirs() -{ - LOCAL_ARRAY(char, name, MAXPATHLEN); - - for (int i = 0; i < l1; i++) { - snprintf(name, MAXPATHLEN, "%s/%02X", path, i); - - int should_exist; - - if (createDirectory(name, 0)) - should_exist = 0; - else - should_exist = 1; - - debugs(47, 1, "Making directories in " << name); - - for (int k = 0; k < l2; k++) { - snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); - createDirectory(name, should_exist); - } - } -} - -char * -UFSSwapDir::logFile(char const *ext) const -{ - LOCAL_ARRAY(char, lpath, MAXPATHLEN); - LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); - LOCAL_ARRAY(char, digit, 32); - char *pathtmp2; - - if (Config.Log.swap) { - xstrncpy(pathtmp, path, MAXPATHLEN - 64); - pathtmp2 = pathtmp; - - while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) - *pathtmp2 = '.'; - - while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') - pathtmp[strlen(pathtmp) - 1] = '\0'; - - for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); - snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); - - if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) { - strcat(lpath, "."); - snprintf(digit, 32, "%02d", index); - strncat(lpath, digit, 3); - } - } else { - xstrncpy(lpath, path, MAXPATHLEN - 64); - strcat(lpath, "/swap.state"); - } - - if (ext) - strncat(lpath, ext, 16); - - return lpath; -} - -void -UFSSwapDir::openLog() -{ - char *logPath; - logPath = logFile(); - swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); - - if (swaplog_fd < 0) { - debugs(50, 1, "" << logPath << ": " << xstrerror()); - fatal("commonUfsDirOpenSwapLog: Failed to open swap log."); - } - - debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd); - - if (0 == NumberOfUFSDirs) - assert(NULL == UFSDirToGlobalDirMapping); - - ++NumberOfUFSDirs; - - assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured); -} - -void -UFSSwapDir::closeLog() -{ - if (swaplog_fd < 0) /* not open */ - return; - - file_close(swaplog_fd); - - debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd); - - swaplog_fd = -1; - - --NumberOfUFSDirs; - - assert(NumberOfUFSDirs >= 0); - - if (0 == NumberOfUFSDirs) - safe_free(UFSDirToGlobalDirMapping); -} - -bool -UFSSwapDir::validL1(int anInt) const -{ - return anInt < l1; -} - -bool -UFSSwapDir::validL2(int anInt) const -{ - return anInt < l2; -} - -/* Add a new object to the cache with empty memory copy and pointer to disk - * use to rebuild store from disk. */ -StoreEntry * -UFSSwapDir::addDiskRestore(const cache_key * key, - sfileno file_number, - uint64_t swap_file_sz, - time_t expires, - time_t timestamp, - time_t lastref, - time_t lastmod, - uint32_t refcount, - uint16_t newFlags, - int clean) -{ - StoreEntry *e = NULL; - debugs(47, 5, "commonUfsAddDiskRestore: " << storeKeyText(key) << - ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); - /* if you call this you'd better be sure file_number is not - * already in use! */ - e = new StoreEntry(); - e->store_status = STORE_OK; - e->setMemStatus(NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_filen = file_number; - e->swap_dirn = index; - e->swap_file_sz = swap_file_sz; - e->lock_count = 0; - e->lastref = lastref; - e->timestamp = timestamp; - e->expires = expires; - e->lastmod = lastmod; - e->refcount = refcount; - e->flags = newFlags; - EBIT_SET(e->flags, ENTRY_CACHABLE); - EBIT_CLR(e->flags, RELEASE_REQUEST); - EBIT_CLR(e->flags, KEY_PRIVATE); - e->ping_status = PING_NONE; - EBIT_CLR(e->flags, ENTRY_VALIDATED); - mapBitSet(e->swap_filen); - cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); - ++n_disk_objects; - e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ - replacementAdd (e); - return e; -} - -void -UFSSwapDir::rebuild() -{ - ++StoreController::store_dirs_rebuilding; - eventAdd("storeRebuild", RebuildState::RebuildStep, new RebuildState(this), 0.0, 1); -} - -void -UFSSwapDir::closeTmpSwapLog() -{ - char *swaplog_path = xstrdup(logFile(NULL)); - char *new_path = xstrdup(logFile(".new")); - int fd; - file_close(swaplog_fd); - - if (xrename(new_path, swaplog_path) < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); - fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); - } - - fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); - - if (fd < 0) { - debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); - fatalf("Failed to open swap log %s", swaplog_path); - } - - safe_free(swaplog_path); - safe_free(new_path); - swaplog_fd = fd; - debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd); -} - -FILE * -UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) -{ - char *swaplog_path = xstrdup(logFile(NULL)); - char *clean_path = xstrdup(logFile(".last-clean")); - char *new_path = xstrdup(logFile(".new")); - - struct stat log_sb; - - struct stat clean_sb; - FILE *fp; - int fd; - - if (::stat(swaplog_path, &log_sb) < 0) { - debugs(47, 1, "Cache Dir #" << index << ": No log file"); - safe_free(swaplog_path); - safe_free(clean_path); - safe_free(new_path); - return NULL; - } - - *zero_flag = log_sb.st_size == 0 ? 1 : 0; - /* close the existing write-only FD */ - - if (swaplog_fd >= 0) - file_close(swaplog_fd); - - /* open a write-only FD for the new log */ - fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - - if (fd < 0) { - debugs(50, 1, "" << new_path << ": " << xstrerror()); - fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); - } - - swaplog_fd = fd; - - { - const StoreSwapLogHeader header; - MemBuf buf; - buf.init(header.record_size, header.record_size); - buf.append(reinterpret_cast(&header), sizeof(header)); - // Pad to keep in sync with UFSSwapDir::writeCleanStart(). - memset(buf.space(), 0, header.gapSize()); - buf.appended(header.gapSize()); - file_write(swaplog_fd, -1, buf.content(), buf.contentSize(), - NULL, NULL, buf.freeFunc()); - } - - /* open a read-only stream of the old log */ - fp = fopen(swaplog_path, "rb"); - - if (fp == NULL) { - debugs(50, 0, "" << swaplog_path << ": " << xstrerror()); - fatal("Failed to open swap log for reading"); - } - - memset(&clean_sb, '\0', sizeof(struct stat)); - - if (::stat(clean_path, &clean_sb) < 0) - *clean_flag = 0; - else if (clean_sb.st_mtime < log_sb.st_mtime) - *clean_flag = 0; - else - *clean_flag = 1; - - safeunlink(clean_path, 1); - - safe_free(swaplog_path); - - safe_free(clean_path); - - safe_free(new_path); - - return fp; -} - -class UFSCleanLog : public SwapDir::CleanLog -{ - -public: - UFSCleanLog(SwapDir *); - virtual const StoreEntry *nextEntry(); - virtual void write(StoreEntry const &); - char *cur; - char *newLog; - char *cln; - char *outbuf; - off_t outbuf_offset; - int fd; - RemovalPolicyWalker *walker; - SwapDir *sd; -}; - -#define CLEAN_BUF_SZ 16384 - - -UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), - outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) -{} - -/* - * Begin the process to write clean cache state. For AUFS this means - * opening some log files and allocating write buffers. Return 0 if - * we succeed, and assign the 'func' and 'data' return pointers. - */ -int -UFSSwapDir::writeCleanStart() -{ - UFSCleanLog *state = new UFSCleanLog(this); - StoreSwapLogHeader header; -#if HAVE_FCHMOD - - struct stat sb; -#endif - - cleanLog = NULL; - state->newLog = xstrdup(logFile(".clean")); - state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); - - if (state->fd < 0) { - xfree(state->newLog); - delete state; - return -1; - } - - state->cur = xstrdup(logFile(NULL)); - state->cln = xstrdup(logFile(".last-clean")); - state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); - state->outbuf_offset = 0; - /*copy the header */ - memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); - // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog(). - memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize()); - state->outbuf_offset += header.record_size; - - state->walker = repl->WalkInit(repl); - ::unlink(state->cln); - debugs(47, 3, "storeDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); -#if HAVE_FCHMOD - - if (::stat(state->cur, &sb) == 0) - fchmod(state->fd, sb.st_mode); - -#endif - - - cleanLog = state; - return 0; -} - -/* - * Get the next entry that is a candidate for clean log writing - */ -const StoreEntry * -UFSCleanLog::nextEntry() -{ - const StoreEntry *entry = NULL; - - if (walker) - entry = walker->Next(walker); - - return entry; -} - -/* - * "write" an entry to the clean log file. - */ -void -UFSCleanLog::write(StoreEntry const &e) -{ - StoreSwapLogData s; - static size_t ss = sizeof(StoreSwapLogData); - s.op = (char) SWAP_LOG_ADD; - s.swap_filen = e.swap_filen; - s.timestamp = e.timestamp; - s.lastref = e.lastref; - s.expires = e.expires; - s.lastmod = e.lastmod; - s.swap_file_sz = e.swap_file_sz; - s.refcount = e.refcount; - s.flags = e.flags; - memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); - s.finalize(); - memcpy(outbuf + outbuf_offset, &s, ss); - outbuf_offset += ss; - /* buffered write */ - - if (outbuf_offset + ss >= CLEAN_BUF_SZ) { - if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { - /* XXX This error handling should probably move up to the caller */ - debugs(50, 0, "storeDirWriteCleanLogs: " << newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(fd); - fd = -1; - unlink(newLog); - sd->cleanLog = NULL; - delete this; - return; - } - - outbuf_offset = 0; - } -} - -void -UFSSwapDir::writeCleanDone() -{ - UFSCleanLog *state = (UFSCleanLog *)cleanLog; - int fd; - - if (NULL == state) - return; - - if (state->fd < 0) - return; - - state->walker->Done(state->walker); - - if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { - debugs(50, 0, "storeDirWriteCleanLogs: " << state->newLog << ": write: " << xstrerror()); - debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); - file_close(state->fd); - state->fd = -1; - ::unlink(state->newLog); - } - - safe_free(state->outbuf); - /* - * You can't rename open files on Microsoft "operating systems" - * so we have to close before renaming. - */ - closeLog(); - /* save the fd value for a later test */ - fd = state->fd; - /* rename */ - - if (state->fd >= 0) { -#if _SQUID_OS2_ || _SQUID_WINDOWS_ - file_close(state->fd); - state->fd = -1; -#endif - - xrename(state->newLog, state->cur); - } - - /* touch a timestamp file if we're not still validating */ - if (StoreController::store_dirs_rebuilding) - (void) 0; - else if (fd < 0) - (void) 0; - else - file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); - - /* close */ - safe_free(state->cur); - - safe_free(state->newLog); - - safe_free(state->cln); - - if (state->fd >= 0) - file_close(state->fd); - - state->fd = -1; - - delete state; - - cleanLog = NULL; -} - -static void -FreeObject(void *address) -{ - StoreSwapLogData *anObject = static_cast (address); - delete anObject; -} - -void -UFSSwapDir::logEntry(const StoreEntry & e, int op) const -{ - StoreSwapLogData *s = new StoreSwapLogData; - s->op = (char) op; - s->swap_filen = e.swap_filen; - s->timestamp = e.timestamp; - s->lastref = e.lastref; - s->expires = e.expires; - s->lastmod = e.lastmod; - s->swap_file_sz = e.swap_file_sz; - s->refcount = e.refcount; - s->flags = e.flags; - memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); - s->finalize(); - file_write(swaplog_fd, - -1, - s, - sizeof(StoreSwapLogData), - NULL, - NULL, - FreeObject); -} - -static QS rev_int_sort; -static int -rev_int_sort(const void *A, const void *B) -{ - const int *i1 = (const int *)A; - const int *i2 = (const int *)B; - return *i2 - *i1; -} - -int -UFSSwapDir::DirClean(int swap_index) -{ - DIR *dir_pointer = NULL; - - LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); - LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); - - int files[20]; - int swapfileno; - int fn; /* same as swapfileno, but with dirn bits set */ - int n = 0; - int k = 0; - int N0, N1, N2; - int D0, D1, D2; - UFSSwapDir *SD; - N0 = NumberOfUFSDirs; - D0 = UFSDirToGlobalDirMapping[swap_index % N0]; - SD = dynamic_cast(INDEXSD(D0)); - assert (SD); - N1 = SD->l1; - D1 = (swap_index / N0) % N1; - N2 = SD->l2; - D2 = ((swap_index / N0) / N1) % N2; - snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", - SD->path, D1, D2); - debugs(36, 3, "storeDirClean: Cleaning directory " << p1); - dir_pointer = opendir(p1); - - if (dir_pointer == NULL) { - if (errno == ENOENT) { - debugs(36, 0, "storeDirClean: WARNING: Creating " << p1); -#if _SQUID_MSWIN_ - - if (mkdir(p1) == 0) -#else - - if (mkdir(p1, 0777) == 0) -#endif - - return 0; - } - - debugs(50, 0, "storeDirClean: " << p1 << ": " << xstrerror()); - safeunlink(p1, 1); - return 0; - } - - dirent_t *de; - while ((de = readdir(dir_pointer)) != NULL && k < 20) { - if (sscanf(de->d_name, "%X", &swapfileno) != 1) - continue; - - fn = swapfileno; /* XXX should remove this cruft ! */ - - if (SD->validFileno(fn, 1)) - if (SD->mapBitTest(fn)) - if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) - continue; - - files[k++] = swapfileno; - } - - closedir(dir_pointer); - - if (k == 0) - return 0; - - qsort(files, k, sizeof(int), rev_int_sort); - - if (k > 10) - k = 10; - - for (n = 0; n < k; n++) { - debugs(36, 3, "storeDirClean: Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); - snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); - safeunlink(p2, 0); - ++statCounter.swap.files_cleaned; - } - - debugs(36, 3, "Cleaned " << k << " unused files from " << p1); - return k; -} - -void -UFSSwapDir::CleanEvent(void *unused) -{ - static int swap_index = 0; - int i; - int j = 0; - int n = 0; - /* - * Assert that there are UFS cache_dirs configured, otherwise - * we should never be called. - */ - assert(NumberOfUFSDirs); - - if (NULL == UFSDirToGlobalDirMapping) { - SwapDir *sd; - /* - * Initialize the little array that translates UFS cache_dir - * number into the Config.cacheSwap.swapDirs array index. - */ - UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); - - for (i = 0, n = 0; i < Config.cacheSwap.n_configured; i++) { - /* This is bogus, the controller should just clean each instance once */ - sd = dynamic_cast (INDEXSD(i)); - - if (!UFSSwapDir::IsUFSDir(sd)) - continue; - - UFSSwapDir *usd = dynamic_cast(sd); - - assert (usd); - - UFSDirToGlobalDirMapping[n++] = i; - - j += (usd->l1 * usd->l2); - } - - assert(n == NumberOfUFSDirs); - /* - * Start the commonUfsDirClean() swap_index with a random - * value. j equals the total number of UFS level 2 - * swap directories - */ - swap_index = (int) (squid_random() % j); - } - - /* if the rebuild is finished, start cleaning directories. */ - if (0 == StoreController::store_dirs_rebuilding) { - n = DirClean(swap_index); - swap_index++; - } - - eventAdd("storeDirClean", CleanEvent, NULL, - 15.0 * exp(-0.25 * n), 1); -} - -int -UFSSwapDir::IsUFSDir(SwapDir * sd) -{ - UFSSwapDir *mySD = dynamic_cast(sd); - return mySD ? 1 : 0 ; -} - -/* - * Does swapfile number 'fn' belong in cachedir #F0, - * level1 dir #F1, level2 dir #F2? - * XXX: this is broken - it assumes all cache dirs use the same - * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - - * if not UFSSwapDir return 0; - */ -int -UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) -{ - int D1, D2; - int L1, L2; - int filn = fn; - assert(F0 < Config.cacheSwap.n_configured); - assert (UFSSwapDir::IsUFSDir (dynamic_cast(INDEXSD(F0)))); - UFSSwapDir *sd = dynamic_cast(INDEXSD(F0)); - - if (!sd) - return 0; - - L1 = sd->l1; - - L2 = sd->l2; - - D1 = ((filn / L2) / L2) % L1; - - if (F1 != D1) - return 0; - - D2 = (filn / L2) % L2; - - if (F2 != D2) - return 0; - - return 1; -} - - -int -UFSSwapDir::validFileno(sfileno filn, int flag) const -{ - if (filn < 0) - return 0; - - /* - * If flag is set it means out-of-range file number should - * be considered invalid. - */ - if (flag) - if (filn > map->capacity()) - return 0; - - return 1; -} - - - -/* - * UFSSwapDir::unlinkFile - * - * This routine unlinks a file and pulls it out of the bitmap. - * It used to be in commonUfsUnlink(), however an interface change - * forced this bit of code here. Eeek. - */ -void -UFSSwapDir::unlinkFile(sfileno f) -{ - debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') << - std::hex << std::uppercase << std::setw(8) << f << " '" << - fullPath(f,NULL) << "'"); - /* commonUfsDirMapBitReset(this, f); */ - IO->unlinkFile(fullPath(f,NULL)); -} - -bool -UFSSwapDir::unlinkdUseful() const -{ - // unlinkd may be useful only in workers - return IamWorkerProcess() && IO->io->unlinkdUseful(); -} - -void -UFSSwapDir::unlink(StoreEntry & e) -{ - debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - if (e.swap_status == SWAPOUT_DONE && EBIT_TEST(e.flags, ENTRY_VALIDATED)) { - cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); - --n_disk_objects; - } - replacementRemove(&e); - mapBitReset(e.swap_filen); - UFSSwapDir::unlinkFile(e.swap_filen); -} - -/* - * Add and remove the given StoreEntry from the replacement policy in - * use. - */ - -void -UFSSwapDir::replacementAdd(StoreEntry * e) -{ - debugs(47, 4, "UFSSwapDir::replacementAdd: added node " << e << " to dir " << index); - repl->Add(repl, e, &e->repl); -} - - -void -UFSSwapDir::replacementRemove(StoreEntry * e) -{ - StorePointer SD; - - if (e->swap_dirn < 0) - return; - - SD = INDEXSD(e->swap_dirn); - - assert (dynamic_cast(SD.getRaw()) == this); - - debugs(47, 4, "UFSSwapDir::replacementRemove: remove node " << e << " from dir " << index); - - repl->Remove(repl, e, &e->repl); -} - -void -UFSSwapDir::dump(StoreEntry & entry) const -{ - storeAppendPrintf(&entry, " %"PRIu64" %d %d", maxSize() >> 20, l1, l2); - dumpOptions(&entry); -} - -char * -UFSSwapDir::fullPath(sfileno filn, char *fullpath) const -{ - LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); - int L1 = l1; - int L2 = l2; - - if (!fullpath) - fullpath = fullfilename; - - fullpath[0] = '\0'; - - snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X", - path, - ((filn / L2) / L2) % L1, - (filn / L2) % L2, - filn); - - return fullpath; -} - -int -UFSSwapDir::callback() -{ - return IO->callback(); -} - -void -UFSSwapDir::sync() -{ - IO->sync(); -} - -void -UFSSwapDir::swappedOut(const StoreEntry &e) -{ - cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); - ++n_disk_objects; -} - -StoreSearch * -UFSSwapDir::search(String const url, HttpRequest *request) -{ - if (url.size()) - fatal ("Cannot search by url yet\n"); - - return new StoreSearchUFS (this); -} - -CBDATA_CLASS_INIT(StoreSearchUFS); -StoreSearchUFS::StoreSearchUFS(RefCount aSwapDir) : sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), current (NULL), _done (false) -{} - -/* do not link -StoreSearchUFS::StoreSearchUFS(StoreSearchUFS const &); -*/ - -StoreSearchUFS::~StoreSearchUFS() -{ - walker->Done(walker); - walker = NULL; -} - -void -StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) -{ - next(); - aCallback(aCallbackArgs); -} - -bool -StoreSearchUFS::next() -{ - /* the walker API doesn't make sense. the store entries referred to are already readwrite - * from their hash table entries - */ - - if (walker) - current = const_cast(walker->Next(walker)); - - if (current == NULL) - _done = true; - - return current != NULL; -} - -bool -StoreSearchUFS::error() const -{ - return false; -} - -bool -StoreSearchUFS::isDone() const -{ - return _done; -} - -StoreEntry * -StoreSearchUFS::currentItem() -{ - return current; -}