src/fs/ufs/UFSSwapDir.cc

   1 /*
   2  * Copyright (C) 1996-2023 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 47    Store Directory Routines */
  10
  11 #define CLEAN_BUF_SZ 16384
  12
  13 #include "squid.h"
  14 #include "base/Random.h"
  15 #include "cache_cf.h"
  16 #include "ConfigOption.h"
  17 #include "DiskIO/DiskIOModule.h"
  18 #include "DiskIO/DiskIOStrategy.h"
  19 #include "fde.h"
  20 #include "FileMap.h"
  21 #include "fs_io.h"
  22 #include "globals.h"
  23 #include "Parsing.h"
  24 #include "RebuildState.h"
  25 #include "SquidConfig.h"
  26 #include "SquidMath.h"
  27 #include "StatCounters.h"
  28 #include "store_key_md5.h"
  29 #include "StoreSearchUFS.h"
  30 #include "StoreSwapLogData.h"
  31 #include "tools.h"
  32 #include "UFSSwapDir.h"
  33
  34 #include <cerrno>
  35 #include <cmath>
  36 #if HAVE_SYS_STAT_H
  37 #include <sys/stat.h>
  38 #endif
  39
  40 int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0;
  41 int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = nullptr;
  42
  43 class UFSCleanLog : public SwapDir::CleanLog
  44 {
  45
  46 public:
  47     UFSCleanLog(SwapDir *aSwapDir) : sd(aSwapDir) {}
  48
  49     /// Get the next entry that is a candidate for clean log writing
  50     const StoreEntry *nextEntry() override;
  51
  52     /// "write" an entry to the clean log file.
  53     void write(StoreEntry const &) override;
  54
  55     SBuf cur;
  56     SBuf newLog;
  57     SBuf cln;
  58     char *outbuf = nullptr;
  59     off_t outbuf_offset = 0;
  60     int fd = -1;
  61     RemovalPolicyWalker *walker = nullptr;
  62     SwapDir *sd = nullptr;
  63 };
  64
  65 const StoreEntry *
  66 UFSCleanLog::nextEntry()
  67 {
  68     const StoreEntry *entry = nullptr;
  69
  70     if (walker)
  71         entry = walker->Next(walker);
  72
  73     return entry;
  74 }
  75
  76 void
  77 UFSCleanLog::write(StoreEntry const &e)
  78 {
  79     StoreSwapLogData s;
  80     static size_t ss = sizeof(StoreSwapLogData);
  81     s.op = (char) SWAP_LOG_ADD;
  82     s.swap_filen = e.swap_filen;
  83     s.timestamp = e.timestamp;
  84     s.lastref = e.lastref;
  85     s.expires = e.expires;
  86     s.lastmod = e.lastModified();
  87     s.swap_file_sz = e.swap_file_sz;
  88     s.refcount = e.refcount;
  89     s.flags = e.flags;
  90     memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
  91     s.finalize();
  92     memcpy(outbuf + outbuf_offset, &s, ss);
  93     outbuf_offset += ss;
  94     /* buffered write */
  95
  96     if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
  97         if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) {
  98             int xerrno = errno;
  99             /* XXX This error handling should probably move up to the caller */
 100             debugs(50, DBG_CRITICAL, MYNAME << newLog << ": write: " << xstrerr(xerrno));
 101             debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
 102             file_close(fd);
 103             fd = -1;
 104             unlink(newLog.c_str());
 105             sd->cleanLog = nullptr;
 106             delete this;
 107             return;
 108         }
 109
 110         outbuf_offset = 0;
 111     }
 112 }
 113
 114 bool
 115 Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
 116 {
 117     if (!SwapDir::canStore(e, diskSpaceNeeded, load))
 118         return false;
 119
 120     if (IO->shedLoad())
 121         return false;
 122
 123     load = IO->load();
 124     return true;
 125 }
 126
 127 static void
 128 FreeObject(void *address)
 129 {
 130     StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address);
 131     delete anObject;
 132 }
 133
 134 static int
 135 rev_int_sort(const void *A, const void *B)
 136 {
 137     const int *i1 = (const int *)A;
 138     const int *i2 = (const int *)B;
 139     return *i2 - *i1;
 140 }
 141
 142 void
 143 Fs::Ufs::UFSSwapDir::parseSizeL1L2()
 144 {
 145     int i = GetInteger();
 146     if (i <= 0)
 147         fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
 148
 149     const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes
 150
 151     /* just reconfigure it */
 152     if (reconfiguring) {
 153         if (size == maxSize())
 154             debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB");
 155         else
 156             debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB");
 157     }
 158
 159     max_size = size;
 160
 161     l1 = GetInteger();
 162
 163     if (l1 <= 0)
 164         fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value");
 165
 166     l2 = GetInteger();
 167
 168     if (l2 <= 0)
 169         fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value");
 170 }
 171
 172 void
 173 Fs::Ufs::UFSSwapDir::reconfigure()
 174 {
 175     parseSizeL1L2();
 176     parseOptions(1);
 177 }
 178
 179 void
 180 Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath)
 181 {
 182     index = anIndex;
 183     path = xstrdup(aPath);
 184
 185     parseSizeL1L2();
 186
 187     /* Initialise replacement policy stuff */
 188     repl = createRemovalPolicy(Config.replPolicy);
 189
 190     parseOptions(0);
 191 }
 192
 193 void
 194 Fs::Ufs::UFSSwapDir::changeIO(DiskIOModule *module)
 195 {
 196     DiskIOStrategy *anIO = module->createStrategy();
 197     safe_free(ioType);
 198     ioType = xstrdup(module->type());
 199
 200     delete IO->io;
 201     IO->io = anIO;
 202     /* Change the IO Options */
 203
 204     if (currentIOOptions && currentIOOptions->options.size() > 2) {
 205         delete currentIOOptions->options.back();
 206         currentIOOptions->options.pop_back();
 207     }
 208
 209     /* TODO: factor out these 4 lines */
 210     ConfigOption *ioOptions = IO->io->getOptionTree();
 211
 212     if (currentIOOptions && ioOptions)
 213         currentIOOptions->options.push_back(ioOptions);
 214 }
 215
 216 bool
 217 Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig)
 218 {
 219     if (strcmp(option, "IOEngine") != 0)
 220         return false;
 221
 222     if (isaReconfig)
 223         /* silently ignore this */
 224         return true;
 225
 226     if (!value) {
 227         self_destruct();
 228         return false;
 229     }
 230
 231     DiskIOModule *module = DiskIOModule::Find(value);
 232
 233     if (!module) {
 234         self_destruct();
 235         return false;
 236     }
 237
 238     changeIO(module);
 239
 240     return true;
 241 }
 242
 243 void
 244 Fs::Ufs::UFSSwapDir::optionIODump(StoreEntry * e) const
 245 {
 246     storeAppendPrintf(e, " IOEngine=%s", ioType);
 247 }
 248
 249 ConfigOption *
 250 Fs::Ufs::UFSSwapDir::getOptionTree() const
 251 {
 252     ConfigOption *parentResult = SwapDir::getOptionTree();
 253
 254     if (currentIOOptions == nullptr)
 255         currentIOOptions = new ConfigOptionVector();
 256
 257     currentIOOptions->options.push_back(parentResult);
 258
 259     currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump));
 260
 261     if (ConfigOption *ioOptions = IO->io->getOptionTree())
 262         currentIOOptions->options.push_back(ioOptions);
 263
 264     ConfigOption* result = currentIOOptions;
 265
 266     currentIOOptions = nullptr;
 267
 268     return result;
 269 }
 270
 271 void
 272 Fs::Ufs::UFSSwapDir::init()
 273 {
 274     debugs(47, 3, "Initialising UFS SwapDir engine.");
 275     /* Parsing must be finished by now - force to NULL, don't delete */
 276     currentIOOptions = nullptr;
 277     static int started_clean_event = 0;
 278     static const char *errmsg =
 279         "\tFailed to verify one of the swap directories, Check cache.log\n"
 280         "\tfor details.  Run 'squid -z' to create swap directories\n"
 281         "\tif needed, or if running Squid for the first time.";
 282     IO->init();
 283
 284     if (verifyCacheDirs())
 285         fatal(errmsg);
 286
 287     openLog();
 288
 289     rebuild();
 290
 291     if (!started_clean_event) {
 292         eventAdd("UFS storeDirClean", CleanEvent, nullptr, 15.0, 1);
 293         started_clean_event = 1;
 294     }
 295
 296     (void) fsBlockSize(path, &fs.blksize);
 297 }
 298
 299 void
 300 Fs::Ufs::UFSSwapDir::create()
 301 {
 302     debugs(47, 3, "Creating swap space in " << path);
 303     createDirectory(path, 0);
 304     createSwapSubDirs();
 305 }
 306
 307 Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
 308     SwapDir(aType),
 309     IO(nullptr),
 310     fsdata(nullptr),
 311     map(new FileMap()),
 312     suggest(0),
 313     l1(16),
 314     l2(256),
 315     swaplog_fd(-1),
 316     currentIOOptions(new ConfigOptionVector()),
 317     ioType(xstrdup(anIOType)),
 318     cur_size(0),
 319     n_disk_objects(0),
 320     rebuilding_(false)
 321 {
 322     /* modulename is only set to disk modules that are built, by configure,
 323      * so the Find call should never return NULL here.
 324      */
 325     IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy());
 326 }
 327
 328 Fs::Ufs::UFSSwapDir::~UFSSwapDir()
 329 {
 330     if (swaplog_fd > -1) {
 331         file_close(swaplog_fd);
 332         swaplog_fd = -1;
 333     }
 334     xfree(ioType);
 335     delete map;
 336     delete IO;
 337     delete currentIOOptions;
 338 }
 339
 340 void
 341 Fs::Ufs::UFSSwapDir::dumpEntry(StoreEntry &e) const
 342 {
 343     debugs(47, DBG_CRITICAL, "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
 344     debugs(47, DBG_CRITICAL, "PATH " << fullPath(e.swap_filen, nullptr)   );
 345     e.dump(0);
 346 }
 347
 348 bool
 349 Fs::Ufs::UFSSwapDir::doubleCheck(StoreEntry & e)
 350 {
 351
 352     struct stat sb;
 353
 354     if (::stat(fullPath(e.swap_filen, nullptr), &sb) < 0) {
 355         debugs(47, DBG_CRITICAL, "WARNING: Missing swap file");
 356         dumpEntry(e);
 357         return true;
 358     }
 359
 360     if ((off_t)e.swap_file_sz != sb.st_size) {
 361         debugs(47, DBG_CRITICAL, "WARNING: Size Mismatch. Entry size: "
 362                << e.swap_file_sz << ", file size: " << sb.st_size);
 363         dumpEntry(e);
 364         return true;
 365     }
 366
 367     return false;
 368 }
 369
 370 void
 371 Fs::Ufs::UFSSwapDir::statfs(StoreEntry & sentry) const
 372 {
 373     int totl_kb = 0;
 374     int free_kb = 0;
 375     int totl_in = 0;
 376     int free_in = 0;
 377     int x;
 378     storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
 379     storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
 380     storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10);
 381     storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0);
 382     storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
 383                       Math::doublePercent(currentSize(), maxSize()));
 384     storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
 385                       map->numFilesInMap(), map->capacity(),
 386                       Math::intPercent(map->numFilesInMap(), map->capacity()));
 387     x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
 388
 389     if (0 == x) {
 390         storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
 391                           totl_kb - free_kb,
 392                           totl_kb,
 393                           Math::intPercent(totl_kb - free_kb, totl_kb));
 394         storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
 395                           totl_in - free_in,
 396                           totl_in,
 397                           Math::intPercent(totl_in - free_in, totl_in));
 398     }
 399
 400     storeAppendPrintf(&sentry, "Flags:");
 401
 402     if (flags.selected)
 403         storeAppendPrintf(&sentry, " SELECTED");
 404
 405     if (flags.read_only)
 406         storeAppendPrintf(&sentry, " READ-ONLY");
 407
 408     storeAppendPrintf(&sentry, "\n");
 409
 410     IO->statfs(sentry);
 411 }
 412
 413 void
 414 Fs::Ufs::UFSSwapDir::maintain()
 415 {
 416     /* TODO: possible options for improvement;
 417      *
 418      * Note that too much aggression here is not good. It means that disk
 419      * controller is getting a long queue of removals to act on, along
 420      * with its regular I/O queue, and that client traffic is 'paused'
 421      * and growing the network I/O queue as well while the scan happens.
 422      * Possibly bad knock-on effects as Squid catches up on all that.
 423      *
 424      * Bug 2448 may have been a sign of what can wrong. At the least it
 425      * provides a test case for aggression effects in overflow conditions.
 426      *
 427      * - base removal limit on space saved, instead of count ?
 428      *
 429      * - base removal rate on a traffic speed counter ?
 430      *   as the purge took up more time out of the second it would grow to
 431      *   a graceful full pause
 432      *
 433      * - pass out a value to cause another event to be scheduled immediately
 434      *   instead of waiting a whole second more ?
 435      *   knock on; schedule less if all caches are under low-water
 436      *
 437      * - admin configurable removal rate or count ?
 438      *   the current numbers are arbitrary, config helps with experimental
 439      *   trials and future-proofing the install base.
 440      *   we also have this indirectly by shifting the relative positions
 441      *   of low-, high- water and the total capacity limit.
 442      */
 443
 444     // minSize() is swap_low_watermark in bytes
 445     const uint64_t lowWaterSz = minSize();
 446
 447     if (currentSize() < lowWaterSz) {
 448         debugs(47, 5, "space still available in " << path);
 449         return;
 450     }
 451
 452     /* We can't delete objects while rebuilding swap */
 453     /* XXX each store should start maintaining as it comes online. */
 454     if (StoreController::store_dirs_rebuilding) {
 455         // suppress the warnings, except once each minute
 456         static int64_t lastWarn = 0;
 457         int warnLevel = 3;
 458         if (lastWarn+60 < squid_curtime) {
 459             lastWarn = squid_curtime;
 460             warnLevel = DBG_IMPORTANT;
 461         }
 462         debugs(47, warnLevel, StoreController::store_dirs_rebuilding << " cache_dir still rebuilding. Skip GC for " << path);
 463         return;
 464     }
 465
 466     // maxSize() is cache_dir total size in bytes
 467     const uint64_t highWaterSz = ((maxSize() * Config.Swap.highWaterMark) / 100);
 468
 469     // f is percentage of 'gap' filled between low- and high-water.
 470     // Used to reduced purge rate when between water markers, and
 471     // to multiply it more aggressively the further above high-water
 472     // it reaches. But in a graceful linear growth curve.
 473     double f = 1.0;
 474     if (highWaterSz > lowWaterSz) {
 475         // might be equal. n/0 is bad.
 476         f = (double) (currentSize() - lowWaterSz) / (highWaterSz - lowWaterSz);
 477     }
 478
 479     // how deep to look for a single object that can be removed
 480     int max_scan = (int) (f * 400.0 + 100.0);
 481
 482     // try to purge only this many objects this cycle.
 483     int max_remove = (int) (f * 300.0 + 20.0);
 484
 485     /*
 486      * This is kinda cheap, but so we need this priority hack?
 487      */
 488     debugs(47, 3, "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove);
 489
 490     RemovalPurgeWalker *walker = repl->PurgeInit(repl, max_scan);
 491
 492     int removed = 0;
 493     // only purge while above low-water
 494     while (currentSize() >= lowWaterSz) {
 495
 496         // stop if we reached max removals for this cycle,
 497         // Bug 2448 may be from this not clearing enough,
 498         // but it predates the current algorithm so not sure
 499         if (removed >= max_remove)
 500             break;
 501
 502         StoreEntry *e = walker->Next(walker);
 503
 504         // stop if all objects are locked / in-use,
 505         // or the cache is empty
 506         if (!e)
 507             break;      /* no more objects */
 508
 509         ++removed;
 510
 511         e->release(true);
 512     }
 513
 514     walker->Done(walker);
 515     debugs(47, (removed ? 2 : 3), path <<
 516            " removed " << removed << "/" << max_remove << " f=" <<
 517            std::setprecision(4) << f << " max_scan=" << max_scan);
 518
 519     // what if cache is still over the high watermark ?
 520     // Store::Maintain() schedules another purge in 1 second.
 521 }
 522
 523 void
 524 Fs::Ufs::UFSSwapDir::reference(StoreEntry &e)
 525 {
 526     debugs(47, 3, "referencing " << &e << " " <<
 527            e.swap_dirn << "/" << e.swap_filen);
 528
 529     if (repl->Referenced)
 530         repl->Referenced(repl, &e, &e.repl);
 531 }
 532
 533 bool
 534 Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e)
 535 {
 536     debugs(47, 3, "dereferencing " << &e << " " <<
 537            e.swap_dirn << "/" << e.swap_filen);
 538
 539     if (repl->Dereferenced)
 540         repl->Dereferenced(repl, &e, &e.repl);
 541
 542     return true; // keep e in the global store_table
 543 }
 544
 545 StoreIOState::Pointer
 546 Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
 547 {
 548     return IO->create (this, &e, file_callback, aCallback, callback_data);
 549 }
 550
 551 StoreIOState::Pointer
 552 Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
 553 {
 554     return IO->open (this, &e, file_callback, aCallback, callback_data);
 555 }
 556
 557 int
 558 Fs::Ufs::UFSSwapDir::mapBitTest(sfileno filn)
 559 {
 560     return map->testBit(filn);
 561 }
 562
 563 void
 564 Fs::Ufs::UFSSwapDir::mapBitSet(sfileno filn)
 565 {
 566     map->setBit(filn);
 567 }
 568
 569 void
 570 Fs::Ufs::UFSSwapDir::mapBitReset(sfileno filn)
 571 {
 572     /*
 573      * We have to test the bit before calling clearBit as
 574      * it doesn't do bounds checking and blindly assumes
 575      * filn is a valid file number, but it might not be because
 576      * the map is dynamic in size.  Also clearing an already clear
 577      * bit puts the map counter of-of-whack.
 578      */
 579
 580     if (map->testBit(filn))
 581         map->clearBit(filn);
 582 }
 583
 584 int
 585 Fs::Ufs::UFSSwapDir::mapBitAllocate()
 586 {
 587     int fn;
 588     fn = map->allocate(suggest);
 589     map->setBit(fn);
 590     suggest = fn + 1;
 591     return fn;
 592 }
 593
 594 char *
 595 Fs::Ufs::UFSSwapDir::swapSubDir(int subdirn)const
 596 {
 597     LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
 598     assert(0 <= subdirn && subdirn < l1);
 599     snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn);
 600     return fullfilename;
 601 }
 602
 603 int
 604 Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist)
 605 {
 606     int created = 0;
 607
 608     struct stat st;
 609     getCurrentTime();
 610
 611     if (0 == ::stat(aPath, &st)) {
 612         if (S_ISDIR(st.st_mode)) {
 613             debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists");
 614         } else {
 615             fatalf("Swap directory %s is not a directory.", aPath);
 616         }
 617     } else if (0 == mkdir(aPath, 0755)) {
 618         debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
 619         created = 1;
 620     } else {
 621         int xerrno = errno;
 622         fatalf("Failed to make swap directory %s: %s", aPath, xstrerr(xerrno));
 623     }
 624
 625     return created;
 626 }
 627
 628 bool
 629 Fs::Ufs::UFSSwapDir::pathIsDirectory(const char *aPath)const
 630 {
 631
 632     struct stat sb;
 633
 634     if (::stat(aPath, &sb) < 0) {
 635         int xerrno = errno;
 636         debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerr(xerrno));
 637         return false;
 638     }
 639
 640     if (S_ISDIR(sb.st_mode) == 0) {
 641         debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory");
 642         return false;
 643     }
 644
 645     return true;
 646 }
 647
 648 bool
 649 Fs::Ufs::UFSSwapDir::verifyCacheDirs()
 650 {
 651     if (!pathIsDirectory(path))
 652         return true;
 653
 654     for (int j = 0; j < l1; ++j) {
 655         char const *aPath = swapSubDir(j);
 656
 657         if (!pathIsDirectory(aPath))
 658             return true;
 659     }
 660
 661     return false;
 662 }
 663
 664 void
 665 Fs::Ufs::UFSSwapDir::createSwapSubDirs()
 666 {
 667     LOCAL_ARRAY(char, name, MAXPATHLEN);
 668
 669     for (int i = 0; i < l1; ++i) {
 670         snprintf(name, MAXPATHLEN, "%s/%02X", path, i);
 671
 672         int should_exist;
 673
 674         if (createDirectory(name, 0))
 675             should_exist = 0;
 676         else
 677             should_exist = 1;
 678
 679         debugs(47, DBG_IMPORTANT, "Making directories in " << name);
 680
 681         for (int k = 0; k < l2; ++k) {
 682             snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k);
 683             createDirectory(name, should_exist);
 684         }
 685     }
 686 }
 687
 688 SBuf
 689 Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
 690 {
 691     SBuf lpath;
 692
 693     if (Config.Log.swap) {
 694         static char pathtmp[MAXPATHLEN];
 695         char *pathtmp2 = xstrncpy(pathtmp, path, MAXPATHLEN - 64);
 696
 697         // replace all '/' with '.'
 698         while ((pathtmp2 = strchr(pathtmp2, '/')))
 699             *pathtmp2 = '.';
 700
 701         // remove any trailing '.' characters
 702         int pos = strlen(pathtmp);
 703         while (pos && pathtmp[pos-1] == '.')
 704             pathtmp[--pos] = '\0';
 705
 706         // remove any prefix '.' characters
 707         for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
 708         // replace a '%s' (if any) in the config string
 709         // with the resulting pathtmp2 string
 710         lpath.appendf(Config.Log.swap, pathtmp2);
 711
 712         // is pathtmp2 was NOT injected, append numeric file extension
 713         if (lpath.cmp(Config.Log.swap) == 0) {
 714             lpath.append(".", 1);
 715             lpath.appendf("%02d", index);
 716         }
 717     } else {
 718         lpath.append(path);
 719         lpath.append("/swap.state", 11);
 720     }
 721
 722     lpath.append(ext); // may be nil, that is okay.
 723
 724     return lpath;
 725 }
 726
 727 void
 728 Fs::Ufs::UFSSwapDir::openLog()
 729 {
 730     if (!IamWorkerProcess())
 731         return;
 732
 733     assert(NumberOfUFSDirs || !UFSDirToGlobalDirMapping);
 734     ++NumberOfUFSDirs;
 735     assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
 736
 737     if (rebuilding_) { // we did not close the temporary log used for rebuilding
 738         assert(swaplog_fd >= 0);
 739         return;
 740     }
 741
 742     SBuf logPath(logFile());
 743     swaplog_fd = file_open(logPath.c_str(), O_WRONLY | O_CREAT | O_BINARY);
 744
 745     if (swaplog_fd < 0) {
 746         int xerrno = errno;
 747         debugs(50, DBG_IMPORTANT, "ERROR: opening swap log " << logPath << ": " << xstrerr(xerrno));
 748         fatal("UFSSwapDir::openLog: Failed to open swap log.");
 749     }
 750
 751     debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
 752 }
 753
 754 void
 755 Fs::Ufs::UFSSwapDir::closeLog()
 756 {
 757     if (swaplog_fd < 0) /* not open */
 758         return;
 759
 760     --NumberOfUFSDirs;
 761     assert(NumberOfUFSDirs >= 0);
 762     if (!NumberOfUFSDirs)
 763         safe_free(UFSDirToGlobalDirMapping);
 764
 765     if (rebuilding_) // we cannot close the temporary log used for rebuilding
 766         return;
 767
 768     file_close(swaplog_fd);
 769
 770     debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
 771
 772     swaplog_fd = -1;
 773 }
 774
 775 bool
 776 Fs::Ufs::UFSSwapDir::validL1(int anInt) const
 777 {
 778     return anInt < l1;
 779 }
 780
 781 bool
 782 Fs::Ufs::UFSSwapDir::validL2(int anInt) const
 783 {
 784     return anInt < l2;
 785 }
 786
 787 StoreEntry *
 788 Fs::Ufs::UFSSwapDir::addDiskRestore(const cache_key * key,
 789                                     sfileno file_number,
 790                                     uint64_t swap_file_sz,
 791                                     time_t expires,
 792                                     time_t timestamp,
 793                                     time_t lastref,
 794                                     time_t lastmod,
 795                                     uint32_t refcount,
 796                                     uint16_t newFlags,
 797                                     int)
 798 {
 799     StoreEntry *e = nullptr;
 800     debugs(47, 5, storeKeyText(key)  <<
 801            ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number);
 802     /* if you call this you'd better be sure file_number is not
 803      * already in use! */
 804     e = new StoreEntry();
 805     e->store_status = STORE_OK;
 806     e->setMemStatus(NOT_IN_MEMORY);
 807     e->attachToDisk(index, file_number, SWAPOUT_DONE);
 808     e->swap_file_sz = swap_file_sz;
 809     e->lastref = lastref;
 810     e->timestamp = timestamp;
 811     e->expires = expires;
 812     e->lastModified(lastmod);
 813     e->refcount = refcount;
 814     e->flags = newFlags;
 815     e->ping_status = PING_NONE;
 816     EBIT_CLR(e->flags, ENTRY_VALIDATED);
 817     mapBitSet(e->swap_filen);
 818     cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
 819     ++n_disk_objects;
 820     e->hashInsert(key);
 821     replacementAdd (e);
 822     return e;
 823 }
 824
 825 void
 826 Fs::Ufs::UFSSwapDir::rebuild()
 827 {
 828     eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1);
 829 }
 830
 831 void
 832 Fs::Ufs::UFSSwapDir::closeTmpSwapLog()
 833 {
 834     assert(rebuilding_);
 835     rebuilding_ = false;
 836
 837     SBuf swaplog_path(logFile()); // where the swaplog should be
 838     SBuf tmp_path(logFile(".new"));
 839
 840     file_close(swaplog_fd);
 841
 842     if (!FileRename(tmp_path, swaplog_path)) {
 843         fatalf("Failed to rename log file " SQUIDSBUFPH " to " SQUIDSBUFPH, SQUIDSBUFPRINT(tmp_path), SQUIDSBUFPRINT(swaplog_path));
 844     }
 845
 846     int fd = file_open(swaplog_path.c_str(), O_WRONLY | O_CREAT | O_BINARY);
 847
 848     if (fd < 0) {
 849         int xerrno = errno;
 850         debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerr(xerrno));
 851         fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
 852     }
 853
 854     swaplog_fd = fd;
 855     debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
 856 }
 857
 858 FILE *
 859 Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
 860 {
 861     assert(!rebuilding_);
 862
 863     SBuf swaplog_path(logFile());
 864     SBuf clean_path(logFile(".last-clean"));
 865     SBuf new_path(logFile(".new"));
 866
 867     struct stat log_sb;
 868
 869     struct stat clean_sb;
 870
 871     if (::stat(swaplog_path.c_str(), &log_sb) < 0) {
 872         debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
 873         return nullptr;
 874     }
 875
 876     *zero_flag = log_sb.st_size == 0 ? 1 : 0;
 877     /* close the existing write-only FD */
 878
 879     if (swaplog_fd >= 0)
 880         file_close(swaplog_fd);
 881
 882     /* open a write-only FD for the new log */
 883     int fd = file_open(new_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
 884     if (fd < 0) {
 885         int xerrno = errno;
 886         debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerr(xerrno));
 887         fatalf("Failed to open swap log " SQUIDSBUFPH, SQUIDSBUFPRINT(new_path));
 888     }
 889
 890     swaplog_fd = fd;
 891     rebuilding_ = true;
 892
 893     {
 894         const StoreSwapLogHeader header;
 895         MemBuf buf;
 896         buf.init(header.record_size, header.record_size);
 897         buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
 898         // Pad to keep in sync with UFSSwapDir::writeCleanStart().
 899         memset(buf.space(), 0, header.gapSize());
 900         buf.appended(header.gapSize());
 901         file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
 902                    nullptr, nullptr, buf.freeFunc());
 903     }
 904
 905     /* open a read-only stream of the old log */
 906     FILE *fp = fopen(swaplog_path.c_str(), "rb");
 907     if (!fp) {
 908         int xerrno = errno;
 909         debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerr(xerrno));
 910         fatalf("Failed to open swap log for reading " SQUIDSBUFPH, SQUIDSBUFPRINT(swaplog_path));
 911     }
 912
 913     memset(&clean_sb, '\0', sizeof(struct stat));
 914
 915     if (::stat(clean_path.c_str(), &clean_sb) < 0)
 916         *clean_flag = 0;
 917     else if (clean_sb.st_mtime < log_sb.st_mtime)
 918         *clean_flag = 0;
 919     else
 920         *clean_flag = 1;
 921
 922     safeunlink(clean_path.c_str(), 1);
 923
 924     return fp;
 925 }
 926
 927 /*
 928  * Begin the process to write clean cache state.  For AUFS this means
 929  * opening some log files and allocating write buffers.  Return 0 if
 930  * we succeed, and assign the 'func' and 'data' return pointers.
 931  */
 932 int
 933 Fs::Ufs::UFSSwapDir::writeCleanStart()
 934 {
 935     UFSCleanLog *state = new UFSCleanLog(this);
 936     StoreSwapLogHeader header;
 937 #if HAVE_FCHMOD
 938
 939     struct stat sb;
 940 #endif
 941
 942     cleanLog = nullptr;
 943     state->cur = logFile();
 944     state->newLog = logFile(".clean");
 945     state->fd = file_open(state->newLog.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
 946
 947     if (state->fd < 0) {
 948         delete state;
 949         return -1;
 950     }
 951
 952     state->cln = state->cur;
 953     state->cln.append(".last-clean");
 954     state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
 955     state->outbuf_offset = 0;
 956     /*copy the header */
 957     memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
 958     // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
 959     memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
 960     state->outbuf_offset += header.record_size;
 961
 962     state->walker = repl->WalkInit(repl);
 963     ::unlink(state->cln.c_str());
 964     debugs(47, 3, "opened " << state->newLog << ", FD " << state->fd);
 965 #if HAVE_FCHMOD
 966
 967     if (::stat(state->cur.c_str(), &sb) == 0)
 968         fchmod(state->fd, sb.st_mode);
 969
 970 #endif
 971
 972     cleanLog = state;
 973     return 0;
 974 }
 975
 976 void
 977 Fs::Ufs::UFSSwapDir::writeCleanDone()
 978 {
 979     UFSCleanLog *state = (UFSCleanLog *)cleanLog;
 980     int fd;
 981
 982     if (nullptr == state)
 983         return;
 984
 985     if (state->fd < 0)
 986         return;
 987
 988     state->walker->Done(state->walker);
 989
 990     if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
 991         int xerrno = errno;
 992         debugs(50, DBG_CRITICAL, MYNAME << state->newLog << ": write: " << xstrerr(xerrno));
 993         debugs(50, DBG_CRITICAL, MYNAME << "Current swap logfile not replaced.");
 994         file_close(state->fd);
 995         state->fd = -1;
 996         ::unlink(state->newLog.c_str());
 997     }
 998
 999     safe_free(state->outbuf);
1000     /*
1001      * You can't rename open files on Microsoft "operating systems"
1002      * so we have to close before renaming.
1003      */
1004     closeLog();
1005     /* save the fd value for a later test */
1006     fd = state->fd;
1007     /* rename */
1008
1009     if (state->fd >= 0) {
1010 #if _SQUID_OS2_ || _SQUID_WINDOWS_
1011         file_close(state->fd);
1012         state->fd = -1;
1013 #endif
1014
1015         FileRename(state->newLog, state->cur);
1016         // TODO handle rename errors
1017     }
1018
1019     /* touch a timestamp file if we're not still validating */
1020     if (StoreController::store_dirs_rebuilding)
1021         (void) 0;
1022     else if (fd < 0)
1023         (void) 0;
1024     else
1025         file_close(file_open(state->cln.c_str(), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
1026
1027     /* close */
1028     if (state->fd >= 0)
1029         file_close(state->fd);
1030
1031     state->fd = -1;
1032
1033     delete state;
1034
1035     cleanLog = nullptr;
1036 }
1037
1038 /// safely cleans a few unused files if possible
1039 int
1040 Fs::Ufs::UFSSwapDir::HandleCleanEvent()
1041 {
1042     static int swap_index = 0;
1043     int i;
1044     int j = 0;
1045     int n = 0;
1046
1047     if (!NumberOfUFSDirs)
1048         return 0; // probably in the middle of reconfiguration
1049
1050     if (nullptr == UFSDirToGlobalDirMapping) {
1051         SwapDir *sd;
1052         /*
1053          * Initialize the little array that translates UFS cache_dir
1054          * number into the Config.cacheSwap.swapDirs array index.
1055          */
1056         UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping));
1057
1058         for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) {
1059             /* This is bogus, the controller should just clean each instance once */
1060             sd = dynamic_cast <SwapDir *>(INDEXSD(i));
1061
1062             if (!UFSSwapDir::IsUFSDir(sd))
1063                 continue;
1064
1065             UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd);
1066
1067             assert (usd);
1068
1069             UFSDirToGlobalDirMapping[n] = i;
1070             ++n;
1071
1072             j += (usd->l1 * usd->l2);
1073         }
1074
1075         assert(n == NumberOfUFSDirs);
1076         /*
1077          * Start the commonUfsDirClean() swap_index with a random
1078          * value.  j equals the total number of UFS level 2
1079          * swap directories
1080          */
1081         static std::mt19937 mt(RandomSeed32());
1082         std::uniform_int_distribution<> dist(0, j);
1083         swap_index = dist(mt);
1084     }
1085
1086     /* if the rebuild is finished, start cleaning directories. */
1087     if (0 == StoreController::store_dirs_rebuilding) {
1088         n = DirClean(swap_index);
1089         ++swap_index;
1090     }
1091
1092     return n;
1093 }
1094
1095 void
1096 Fs::Ufs::UFSSwapDir::CleanEvent(void *)
1097 {
1098     const int n = HandleCleanEvent();
1099     eventAdd("storeDirClean", CleanEvent, nullptr,
1100              15.0 * exp(-0.25 * n), 1);
1101 }
1102
1103 bool
1104 Fs::Ufs::UFSSwapDir::IsUFSDir(SwapDir * sd)
1105 {
1106     UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd);
1107     return (mySD != nullptr) ;
1108 }
1109
1110 /*
1111  * XXX: this is broken - it assumes all cache dirs use the same
1112  * l1 and l2 scheme. -RBC 20021215. Partial fix is in place -
1113  * if not UFSSwapDir return 0;
1114  */
1115 bool
1116 Fs::Ufs::UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2)
1117 {
1118     int D1, D2;
1119     int L1, L2;
1120     int filn = fn;
1121     assert(F0 < Config.cacheSwap.n_configured);
1122     assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0))));
1123     UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0));
1124
1125     if (!sd)
1126         return 0;
1127
1128     L1 = sd->l1;
1129
1130     L2 = sd->l2;
1131
1132     D1 = ((filn / L2) / L2) % L1;
1133
1134     if (F1 != D1)
1135         return 0;
1136
1137     D2 = (filn / L2) % L2;
1138
1139     if (F2 != D2)
1140         return 0;
1141
1142     return 1;
1143 }
1144
1145 int
1146 Fs::Ufs::UFSSwapDir::validFileno(sfileno filn, int flag) const
1147 {
1148     if (filn < 0)
1149         return 0;
1150
1151     /*
1152      * If flag is set it means out-of-range file number should
1153      * be considered invalid.
1154      */
1155     if (flag)
1156         if (filn > map->capacity())
1157             return 0;
1158
1159     return 1;
1160 }
1161
1162 void
1163 Fs::Ufs::UFSSwapDir::unlinkFile(sfileno f)
1164 {
1165     debugs(79, 3, "unlinking fileno " <<  std::setfill('0') <<
1166            std::hex << std::uppercase << std::setw(8) << f << " '" <<
1167            fullPath(f,nullptr) << "'");
1168     /* commonUfsDirMapBitReset(this, f); */
1169     IO->unlinkFile(fullPath(f,nullptr));
1170 }
1171
1172 bool
1173 Fs::Ufs::UFSSwapDir::unlinkdUseful() const
1174 {
1175     // unlinkd may be useful only in workers
1176     return IamWorkerProcess() && IO->io->unlinkdUseful();
1177 }
1178
1179 void
1180 Fs::Ufs::UFSSwapDir::evictCached(StoreEntry & e)
1181 {
1182     debugs(79, 3, e);
1183     if (e.locked()) // somebody else may still be using this file
1184         return; // nothing to do: our get() always returns nil
1185
1186     if (!e.hasDisk())
1187         return; // see evictIfFound()
1188
1189     // Since these fields grow only after swap out ends successfully,
1190     // do not decrement them for e.swappingOut() and e.swapoutFailed().
1191     if (e.swappedOut()) {
1192         cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
1193         --n_disk_objects;
1194     }
1195     replacementRemove(&e);
1196     mapBitReset(e.swap_filen);
1197     UFSSwapDir::unlinkFile(e.swap_filen);
1198     e.detachFromDisk();
1199 }
1200
1201 void
1202 Fs::Ufs::UFSSwapDir::evictIfFound(const cache_key *)
1203 {
1204     // UFS disk entries always have (attached) StoreEntries so if we got here,
1205     // the entry is not cached on disk and there is nothing for us to do.
1206 }
1207
1208 void
1209 Fs::Ufs::UFSSwapDir::replacementAdd(StoreEntry * e)
1210 {
1211     debugs(47, 4, "added node " << e << " to dir " << index);
1212     repl->Add(repl, e, &e->repl);
1213 }
1214
1215 void
1216 Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e)
1217 {
1218     assert(e->hasDisk());
1219
1220     SwapDirPointer SD = INDEXSD(e->swap_dirn);
1221
1222     assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
1223
1224     debugs(47, 4, "remove node " << e << " from dir " << index);
1225
1226     repl->Remove(repl, e, &e->repl);
1227 }
1228
1229 void
1230 Fs::Ufs::UFSSwapDir::dump(StoreEntry & entry) const
1231 {
1232     storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2);
1233     dumpOptions(&entry);
1234 }
1235
1236 char *
1237 Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const
1238 {
1239     LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
1240     int L1 = l1;
1241     int L2 = l2;
1242
1243     if (!fullpath)
1244         fullpath = fullfilename;
1245
1246     fullpath[0] = '\0';
1247
1248     snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X",
1249              path,
1250              ((filn / L2) / L2) % L1,
1251              (filn / L2) % L2,
1252              filn);
1253
1254     return fullpath;
1255 }
1256
1257 int
1258 Fs::Ufs::UFSSwapDir::callback()
1259 {
1260     return IO->callback();
1261 }
1262
1263 void
1264 Fs::Ufs::UFSSwapDir::sync()
1265 {
1266     IO->sync();
1267 }
1268
1269 void
1270 Fs::Ufs::UFSSwapDir::finalizeSwapoutSuccess(const StoreEntry &e)
1271 {
1272     cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
1273     ++n_disk_objects;
1274 }
1275
1276 void
1277 Fs::Ufs::UFSSwapDir::finalizeSwapoutFailure(StoreEntry &entry)
1278 {
1279     debugs(47, 5, entry);
1280     // rely on the expected eventual StoreEntry::release(), evictCached(), or
1281     // a similar call to call unlink(), detachFromDisk(), etc. for the entry.
1282 }
1283
1284 void
1285 Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const
1286 {
1287     if (swaplog_fd < 0) {
1288         debugs(36, 5, "cannot log " << e << " in the middle of reconfiguration");
1289         return;
1290     }
1291
1292     StoreSwapLogData *s = new StoreSwapLogData;
1293     s->op = (char) op;
1294     s->swap_filen = e.swap_filen;
1295     s->timestamp = e.timestamp;
1296     s->lastref = e.lastref;
1297     s->expires = e.expires;
1298     s->lastmod = e.lastModified();
1299     s->swap_file_sz = e.swap_file_sz;
1300     s->refcount = e.refcount;
1301     s->flags = e.flags;
1302     memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
1303     s->finalize();
1304     file_write(swaplog_fd,
1305                -1,
1306                s,
1307                sizeof(StoreSwapLogData),
1308                nullptr,
1309                nullptr,
1310                FreeObject);
1311 }
1312
1313 int
1314 Fs::Ufs::UFSSwapDir::DirClean(int swap_index)
1315 {
1316     DIR *dir_pointer = nullptr;
1317     int files[20];
1318     int swapfileno;
1319     int fn;         /* same as swapfileno, but with dirn bits set */
1320     int n = 0;
1321     int k = 0;
1322     int N0, N1, N2;
1323     int D0, D1, D2;
1324     UFSSwapDir *SD;
1325     N0 = NumberOfUFSDirs;
1326     D0 = UFSDirToGlobalDirMapping[swap_index % N0];
1327     SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0));
1328     assert (SD);
1329     N1 = SD->l1;
1330     D1 = (swap_index / N0) % N1;
1331     N2 = SD->l2;
1332     D2 = ((swap_index / N0) / N1) % N2;
1333
1334     SBuf p1;
1335     p1.appendf("%s/%02X/%02X", SD->path, D1, D2);
1336     debugs(36, 3, "Cleaning directory " << p1);
1337     dir_pointer = opendir(p1.c_str());
1338
1339     if (!dir_pointer) {
1340         int xerrno = errno;
1341         if (xerrno == ENOENT) {
1342             debugs(36, DBG_CRITICAL, MYNAME << "WARNING: Creating " << p1);
1343             if (mkdir(p1.c_str(), 0777) == 0)
1344                 return 0;
1345         }
1346
1347         debugs(50, DBG_CRITICAL, MYNAME << p1 << ": " << xstrerr(xerrno));
1348         safeunlink(p1.c_str(), 1);
1349         return 0;
1350     }
1351
1352     dirent_t *de;
1353     while ((de = readdir(dir_pointer)) != nullptr && k < 20) {
1354         if (sscanf(de->d_name, "%X", &swapfileno) != 1)
1355             continue;
1356
1357         fn = swapfileno;    /* XXX should remove this cruft ! */
1358
1359         if (SD->validFileno(fn, 1))
1360             if (SD->mapBitTest(fn))
1361                 if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2))
1362                     continue;
1363
1364         files[k] = swapfileno;
1365         ++k;
1366     }
1367
1368     closedir(dir_pointer);
1369
1370     if (k == 0)
1371         return 0;
1372
1373     qsort(files, k, sizeof(int), rev_int_sort);
1374
1375     if (k > 10)
1376         k = 10;
1377
1378     for (n = 0; n < k; ++n) {
1379         debugs(36, 3, "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
1380         SBuf p2(p1);
1381         p2.appendf("/%08X", files[n]);
1382         safeunlink(p2.c_str(), 0);
1383         ++statCounter.swap.files_cleaned;
1384     }
1385
1386     debugs(36, 3, "Cleaned " << k << " unused files from " << p1);
1387     return k;
1388 }
1389