src/fs/ufs/UFSSwapDir.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 47    Store Directory Routines */
  10
  11 #define CLEAN_BUF_SZ 16384
  12
  13 #include "squid.h"
  14 #include "cache_cf.h"
  15 #include "ConfigOption.h"
  16 #include "disk.h"
  17 #include "DiskIO/DiskIOModule.h"
  18 #include "DiskIO/DiskIOStrategy.h"
  19 #include "fde.h"
  20 #include "FileMap.h"
  21 #include "globals.h"
  22 #include "Parsing.h"
  23 #include "RebuildState.h"
  24 #include "SquidConfig.h"
  25 #include "SquidMath.h"
  26 #include "SquidTime.h"
  27 #include "StatCounters.h"
  28 #include "store_key_md5.h"
  29 #include "StoreSearchUFS.h"
  30 #include "StoreSwapLogData.h"
  31 #include "tools.h"
  32 #include "UFSSwapDir.h"
  33
  34 #include <cerrno>
  35 #include <cmath>
  36 #include <random>
  37 #if HAVE_SYS_STAT_H
  38 #include <sys/stat.h>
  39 #endif
  40
  41 int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0;
  42 int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = NULL;
  43
  44 class UFSCleanLog : public SwapDir::CleanLog
  45 {
  46
  47 public:
  48     UFSCleanLog(SwapDir *);
  49     /** Get the next entry that is a candidate for clean log writing
  50      */
  51     virtual const StoreEntry *nextEntry();
  52     /** "write" an entry to the clean log file.
  53      */
  54     virtual void write(StoreEntry const &);
  55     char *cur;
  56     char *newLog;
  57     char *cln;
  58     char *outbuf;
  59     off_t outbuf_offset;
  60     int fd;
  61     RemovalPolicyWalker *walker;
  62     SwapDir *sd;
  63 };
  64
  65 UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) :
  66     cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL),
  67     outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir)
  68 {}
  69
  70 const StoreEntry *
  71 UFSCleanLog::nextEntry()
  72 {
  73     const StoreEntry *entry = NULL;
  74
  75     if (walker)
  76         entry = walker->Next(walker);
  77
  78     return entry;
  79 }
  80
  81 void
  82 UFSCleanLog::write(StoreEntry const &e)
  83 {
  84     StoreSwapLogData s;
  85     static size_t ss = sizeof(StoreSwapLogData);
  86     s.op = (char) SWAP_LOG_ADD;
  87     s.swap_filen = e.swap_filen;
  88     s.timestamp = e.timestamp;
  89     s.lastref = e.lastref;
  90     s.expires = e.expires;
  91     s.lastmod = e.lastmod;
  92     s.swap_file_sz = e.swap_file_sz;
  93     s.refcount = e.refcount;
  94     s.flags = e.flags;
  95     memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
  96     s.finalize();
  97     memcpy(outbuf + outbuf_offset, &s, ss);
  98     outbuf_offset += ss;
  99     /* buffered write */
 100
 101     if (outbuf_offset + ss >= CLEAN_BUF_SZ) {
 102         if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) {
 103             /* XXX This error handling should probably move up to the caller */
 104             debugs(50, DBG_CRITICAL, HERE << newLog << ": write: " << xstrerror());
 105             debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced.");
 106             file_close(fd);
 107             fd = -1;
 108             unlink(newLog);
 109             sd->cleanLog = NULL;
 110             delete this;
 111             return;
 112         }
 113
 114         outbuf_offset = 0;
 115     }
 116 }
 117
 118 bool
 119 Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
 120 {
 121     if (!SwapDir::canStore(e, diskSpaceNeeded, load))
 122         return false;
 123
 124     if (IO->shedLoad())
 125         return false;
 126
 127     load = IO->load();
 128     return true;
 129 }
 130
 131 static void
 132 FreeObject(void *address)
 133 {
 134     StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address);
 135     delete anObject;
 136 }
 137
 138 static QS rev_int_sort;
 139 static int
 140 rev_int_sort(const void *A, const void *B)
 141 {
 142     const int *i1 = (const int *)A;
 143     const int *i2 = (const int *)B;
 144     return *i2 - *i1;
 145 }
 146
 147 void
 148 Fs::Ufs::UFSSwapDir::parseSizeL1L2()
 149 {
 150     int i = GetInteger();
 151     if (i <= 0)
 152         fatal("UFSSwapDir::parseSizeL1L2: invalid size value");
 153
 154     const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes
 155
 156     /* just reconfigure it */
 157     if (reconfiguring) {
 158         if (size == maxSize())
 159             debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB");
 160         else
 161             debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB");
 162     }
 163
 164     max_size = size;
 165
 166     l1 = GetInteger();
 167
 168     if (l1 <= 0)
 169         fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value");
 170
 171     l2 = GetInteger();
 172
 173     if (l2 <= 0)
 174         fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value");
 175 }
 176
 177 void
 178 Fs::Ufs::UFSSwapDir::reconfigure()
 179 {
 180     parseSizeL1L2();
 181     parseOptions(1);
 182 }
 183
 184 void
 185 Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath)
 186 {
 187     index = anIndex;
 188     path = xstrdup(aPath);
 189
 190     parseSizeL1L2();
 191
 192     /* Initialise replacement policy stuff */
 193     repl = createRemovalPolicy(Config.replPolicy);
 194
 195     parseOptions(0);
 196 }
 197
 198 void
 199 Fs::Ufs::UFSSwapDir::changeIO(DiskIOModule *module)
 200 {
 201     DiskIOStrategy *anIO = module->createStrategy();
 202     safe_free(ioType);
 203     ioType = xstrdup(module->type());
 204
 205     delete IO->io;
 206     IO->io = anIO;
 207     /* Change the IO Options */
 208
 209     if (currentIOOptions && currentIOOptions->options.size() > 2) {
 210         delete currentIOOptions->options.back();
 211         currentIOOptions->options.pop_back();
 212     }
 213
 214     /* TODO: factor out these 4 lines */
 215     ConfigOption *ioOptions = IO->io->getOptionTree();
 216
 217     if (currentIOOptions && ioOptions)
 218         currentIOOptions->options.push_back(ioOptions);
 219 }
 220
 221 bool
 222 Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig)
 223 {
 224     if (strcmp(option, "IOEngine") != 0)
 225         return false;
 226
 227     if (isaReconfig)
 228         /* silently ignore this */
 229         return true;
 230
 231     if (!value)
 232         self_destruct();
 233
 234     DiskIOModule *module = DiskIOModule::Find(value);
 235
 236     if (!module)
 237         self_destruct();
 238
 239     changeIO(module);
 240
 241     return true;
 242 }
 243
 244 void
 245 Fs::Ufs::UFSSwapDir::optionIODump(StoreEntry * e) const
 246 {
 247     storeAppendPrintf(e, " IOEngine=%s", ioType);
 248 }
 249
 250 ConfigOption *
 251 Fs::Ufs::UFSSwapDir::getOptionTree() const
 252 {
 253     ConfigOption *parentResult = SwapDir::getOptionTree();
 254
 255     if (currentIOOptions == NULL)
 256         currentIOOptions = new ConfigOptionVector();
 257
 258     currentIOOptions->options.push_back(parentResult);
 259
 260     currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump));
 261
 262     if (ConfigOption *ioOptions = IO->io->getOptionTree())
 263         currentIOOptions->options.push_back(ioOptions);
 264
 265     ConfigOption* result = currentIOOptions;
 266
 267     currentIOOptions = NULL;
 268
 269     return result;
 270 }
 271
 272 void
 273 Fs::Ufs::UFSSwapDir::init()
 274 {
 275     debugs(47, 3, HERE << "Initialising UFS SwapDir engine.");
 276     /* Parsing must be finished by now - force to NULL, don't delete */
 277     currentIOOptions = NULL;
 278     static int started_clean_event = 0;
 279     static const char *errmsg =
 280         "\tFailed to verify one of the swap directories, Check cache.log\n"
 281         "\tfor details.  Run 'squid -z' to create swap directories\n"
 282         "\tif needed, or if running Squid for the first time.";
 283     IO->init();
 284
 285     if (verifyCacheDirs())
 286         fatal(errmsg);
 287
 288     openLog();
 289
 290     rebuild();
 291
 292     if (!started_clean_event) {
 293         eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1);
 294         started_clean_event = 1;
 295     }
 296
 297     (void) storeDirGetBlkSize(path, &fs.blksize);
 298 }
 299
 300 void
 301 Fs::Ufs::UFSSwapDir::create()
 302 {
 303     debugs(47, 3, "Creating swap space in " << path);
 304     createDirectory(path, 0);
 305     createSwapSubDirs();
 306 }
 307
 308 Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) :
 309     SwapDir(aType),
 310     IO(NULL),
 311     fsdata(NULL),
 312     map(new FileMap()),
 313     suggest(0),
 314     l1(16),
 315     l2(256),
 316     swaplog_fd(-1),
 317     currentIOOptions(new ConfigOptionVector()),
 318     ioType(xstrdup(anIOType)),
 319     cur_size(0),
 320     n_disk_objects(0)
 321 {
 322     /* modulename is only set to disk modules that are built, by configure,
 323      * so the Find call should never return NULL here.
 324      */
 325     IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy());
 326 }
 327
 328 Fs::Ufs::UFSSwapDir::~UFSSwapDir()
 329 {
 330     if (swaplog_fd > -1) {
 331         file_close(swaplog_fd);
 332         swaplog_fd = -1;
 333     }
 334     xfree(ioType);
 335     delete map;
 336     delete IO;
 337     delete currentIOOptions;
 338 }
 339
 340 void
 341 Fs::Ufs::UFSSwapDir::dumpEntry(StoreEntry &e) const
 342 {
 343     debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
 344     debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL)   );
 345     e.dump(0);
 346 }
 347
 348 bool
 349 Fs::Ufs::UFSSwapDir::doubleCheck(StoreEntry & e)
 350 {
 351
 352     struct stat sb;
 353
 354     if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) {
 355         debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file");
 356         dumpEntry(e);
 357         return true;
 358     }
 359
 360     if ((off_t)e.swap_file_sz != sb.st_size) {
 361         debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: "
 362                << e.swap_file_sz << ", file size: " << sb.st_size);
 363         dumpEntry(e);
 364         return true;
 365     }
 366
 367     return false;
 368 }
 369
 370 void
 371 Fs::Ufs::UFSSwapDir::statfs(StoreEntry & sentry) const
 372 {
 373     int totl_kb = 0;
 374     int free_kb = 0;
 375     int totl_in = 0;
 376     int free_in = 0;
 377     int x;
 378     storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1);
 379     storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2);
 380     storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10);
 381     storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0);
 382     storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n",
 383                       Math::doublePercent(currentSize(), maxSize()));
 384     storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
 385                       map->numFilesInMap(), map->capacity(),
 386                       Math::intPercent(map->numFilesInMap(), map->capacity()));
 387     x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
 388
 389     if (0 == x) {
 390         storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
 391                           totl_kb - free_kb,
 392                           totl_kb,
 393                           Math::intPercent(totl_kb - free_kb, totl_kb));
 394         storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n",
 395                           totl_in - free_in,
 396                           totl_in,
 397                           Math::intPercent(totl_in - free_in, totl_in));
 398     }
 399
 400     storeAppendPrintf(&sentry, "Flags:");
 401
 402     if (flags.selected)
 403         storeAppendPrintf(&sentry, " SELECTED");
 404
 405     if (flags.read_only)
 406         storeAppendPrintf(&sentry, " READ-ONLY");
 407
 408     storeAppendPrintf(&sentry, "\n");
 409
 410     IO->statfs(sentry);
 411 }
 412
 413 void
 414 Fs::Ufs::UFSSwapDir::maintain()
 415 {
 416     /* We can't delete objects while rebuilding swap */
 417
 418     /* XXX FIXME each store should start maintaining as it comes online. */
 419
 420     if (StoreController::store_dirs_rebuilding)
 421         return;
 422
 423     StoreEntry *e = NULL;
 424
 425     int removed = 0;
 426
 427     RemovalPurgeWalker *walker;
 428
 429     double f = (double) (currentSize() - minSize()) / (maxSize() - minSize());
 430
 431     f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f;
 432
 433     int max_scan = (int) (f * 400.0 + 100.0);
 434
 435     int max_remove = (int) (f * 70.0 + 10.0);
 436
 437     /*
 438      * This is kinda cheap, but so we need this priority hack?
 439      */
 440
 441     debugs(47, 3, HERE << "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove  );
 442
 443     walker = repl->PurgeInit(repl, max_scan);
 444
 445     while (1) {
 446         if (currentSize() < minSize())
 447             break;
 448
 449         if (removed >= max_remove)
 450             break;
 451
 452         e = walker->Next(walker);
 453
 454         if (!e)
 455             break;      /* no more objects */
 456
 457         ++removed;
 458
 459         e->release();
 460     }
 461
 462     walker->Done(walker);
 463     debugs(47, (removed ? 2 : 3), HERE << path <<
 464            " removed " << removed << "/" << max_remove << " f=" <<
 465            std::setprecision(4) << f << " max_scan=" << max_scan);
 466 }
 467
 468 void
 469 Fs::Ufs::UFSSwapDir::reference(StoreEntry &e)
 470 {
 471     debugs(47, 3, HERE << "referencing " << &e << " " <<
 472            e.swap_dirn << "/" << e.swap_filen);
 473
 474     if (repl->Referenced)
 475         repl->Referenced(repl, &e, &e.repl);
 476 }
 477
 478 bool
 479 Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e, bool)
 480 {
 481     debugs(47, 3, HERE << "dereferencing " << &e << " " <<
 482            e.swap_dirn << "/" << e.swap_filen);
 483
 484     if (repl->Dereferenced)
 485         repl->Dereferenced(repl, &e, &e.repl);
 486
 487     return true; // keep e in the global store_table
 488 }
 489
 490 StoreIOState::Pointer
 491 Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
 492 {
 493     return IO->create (this, &e, file_callback, aCallback, callback_data);
 494 }
 495
 496 StoreIOState::Pointer
 497 Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data)
 498 {
 499     return IO->open (this, &e, file_callback, aCallback, callback_data);
 500 }
 501
 502 int
 503 Fs::Ufs::UFSSwapDir::mapBitTest(sfileno filn)
 504 {
 505     return map->testBit(filn);
 506 }
 507
 508 void
 509 Fs::Ufs::UFSSwapDir::mapBitSet(sfileno filn)
 510 {
 511     map->setBit(filn);
 512 }
 513
 514 void
 515 Fs::Ufs::UFSSwapDir::mapBitReset(sfileno filn)
 516 {
 517     /*
 518      * We have to test the bit before calling clearBit as
 519      * it doesn't do bounds checking and blindly assumes
 520      * filn is a valid file number, but it might not be because
 521      * the map is dynamic in size.  Also clearing an already clear
 522      * bit puts the map counter of-of-whack.
 523      */
 524
 525     if (map->testBit(filn))
 526         map->clearBit(filn);
 527 }
 528
 529 int
 530 Fs::Ufs::UFSSwapDir::mapBitAllocate()
 531 {
 532     int fn;
 533     fn = map->allocate(suggest);
 534     map->setBit(fn);
 535     suggest = fn + 1;
 536     return fn;
 537 }
 538
 539 char *
 540 Fs::Ufs::UFSSwapDir::swapSubDir(int subdirn)const
 541 {
 542     LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
 543     assert(0 <= subdirn && subdirn < l1);
 544     snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn);
 545     return fullfilename;
 546 }
 547
 548 int
 549 Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist)
 550 {
 551     int created = 0;
 552
 553     struct stat st;
 554     getCurrentTime();
 555
 556     if (0 == ::stat(aPath, &st)) {
 557         if (S_ISDIR(st.st_mode)) {
 558             debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists");
 559         } else {
 560             fatalf("Swap directory %s is not a directory.", aPath);
 561         }
 562     } else if (0 == mkdir(aPath, 0755)) {
 563         debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created");
 564         created = 1;
 565     } else {
 566         fatalf("Failed to make swap directory %s: %s",
 567                aPath, xstrerror());
 568     }
 569
 570     return created;
 571 }
 572
 573 bool
 574 Fs::Ufs::UFSSwapDir::pathIsDirectory(const char *aPath)const
 575 {
 576
 577     struct stat sb;
 578
 579     if (::stat(aPath, &sb) < 0) {
 580         debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerror());
 581         return false;
 582     }
 583
 584     if (S_ISDIR(sb.st_mode) == 0) {
 585         debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory");
 586         return false;
 587     }
 588
 589     return true;
 590 }
 591
 592 bool
 593 Fs::Ufs::UFSSwapDir::verifyCacheDirs()
 594 {
 595     if (!pathIsDirectory(path))
 596         return true;
 597
 598     for (int j = 0; j < l1; ++j) {
 599         char const *aPath = swapSubDir(j);
 600
 601         if (!pathIsDirectory(aPath))
 602             return true;
 603     }
 604
 605     return false;
 606 }
 607
 608 void
 609 Fs::Ufs::UFSSwapDir::createSwapSubDirs()
 610 {
 611     LOCAL_ARRAY(char, name, MAXPATHLEN);
 612
 613     for (int i = 0; i < l1; ++i) {
 614         snprintf(name, MAXPATHLEN, "%s/%02X", path, i);
 615
 616         int should_exist;
 617
 618         if (createDirectory(name, 0))
 619             should_exist = 0;
 620         else
 621             should_exist = 1;
 622
 623         debugs(47, DBG_IMPORTANT, "Making directories in " << name);
 624
 625         for (int k = 0; k < l2; ++k) {
 626             snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k);
 627             createDirectory(name, should_exist);
 628         }
 629     }
 630 }
 631
 632 char *
 633 Fs::Ufs::UFSSwapDir::logFile(char const *ext) const
 634 {
 635     LOCAL_ARRAY(char, lpath, MAXPATHLEN);
 636     LOCAL_ARRAY(char, pathtmp, MAXPATHLEN);
 637     LOCAL_ARRAY(char, digit, 32);
 638     char *pathtmp2;
 639
 640     if (Config.Log.swap) {
 641         xstrncpy(pathtmp, path, MAXPATHLEN - 64);
 642         pathtmp2 = pathtmp;
 643
 644         while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL)
 645             *pathtmp2 = '.';
 646
 647         while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.')
 648             pathtmp[strlen(pathtmp) - 1] = '\0';
 649
 650         for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2);
 651         snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2);
 652
 653         if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) {
 654             strcat(lpath, ".");
 655             snprintf(digit, 32, "%02d", index);
 656             strncat(lpath, digit, 3);
 657         }
 658     } else {
 659         xstrncpy(lpath, path, MAXPATHLEN - 64);
 660         strcat(lpath, "/swap.state");
 661     }
 662
 663     if (ext)
 664         strncat(lpath, ext, 16);
 665
 666     return lpath;
 667 }
 668
 669 void
 670 Fs::Ufs::UFSSwapDir::openLog()
 671 {
 672     char *logPath;
 673     logPath = logFile();
 674     swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY);
 675
 676     if (swaplog_fd < 0) {
 677         debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerror());
 678         fatal("UFSSwapDir::openLog: Failed to open swap log.");
 679     }
 680
 681     debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd);
 682
 683     if (0 == NumberOfUFSDirs)
 684         assert(NULL == UFSDirToGlobalDirMapping);
 685
 686     ++NumberOfUFSDirs;
 687
 688     assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured);
 689 }
 690
 691 void
 692 Fs::Ufs::UFSSwapDir::closeLog()
 693 {
 694     if (swaplog_fd < 0) /* not open */
 695         return;
 696
 697     file_close(swaplog_fd);
 698
 699     debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd);
 700
 701     swaplog_fd = -1;
 702
 703     --NumberOfUFSDirs;
 704
 705     assert(NumberOfUFSDirs >= 0);
 706
 707     if (0 == NumberOfUFSDirs)
 708         safe_free(UFSDirToGlobalDirMapping);
 709 }
 710
 711 bool
 712 Fs::Ufs::UFSSwapDir::validL1(int anInt) const
 713 {
 714     return anInt < l1;
 715 }
 716
 717 bool
 718 Fs::Ufs::UFSSwapDir::validL2(int anInt) const
 719 {
 720     return anInt < l2;
 721 }
 722
 723 StoreEntry *
 724 Fs::Ufs::UFSSwapDir::addDiskRestore(const cache_key * key,
 725                                     sfileno file_number,
 726                                     uint64_t swap_file_sz,
 727                                     time_t expires,
 728                                     time_t timestamp,
 729                                     time_t lastref,
 730                                     time_t lastmod,
 731                                     uint32_t refcount,
 732                                     uint16_t newFlags,
 733                                     int)
 734 {
 735     StoreEntry *e = NULL;
 736     debugs(47, 5, HERE << storeKeyText(key)  <<
 737            ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number);
 738     /* if you call this you'd better be sure file_number is not
 739      * already in use! */
 740     e = new StoreEntry();
 741     e->store_status = STORE_OK;
 742     e->setMemStatus(NOT_IN_MEMORY);
 743     e->swap_status = SWAPOUT_DONE;
 744     e->swap_filen = file_number;
 745     e->swap_dirn = index;
 746     e->swap_file_sz = swap_file_sz;
 747     e->lastref = lastref;
 748     e->timestamp = timestamp;
 749     e->expires = expires;
 750     e->lastmod = lastmod;
 751     e->refcount = refcount;
 752     e->flags = newFlags;
 753     EBIT_CLR(e->flags, RELEASE_REQUEST);
 754     EBIT_CLR(e->flags, KEY_PRIVATE);
 755     e->ping_status = PING_NONE;
 756     EBIT_CLR(e->flags, ENTRY_VALIDATED);
 757     mapBitSet(e->swap_filen);
 758     cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz);
 759     ++n_disk_objects;
 760     e->hashInsert(key); /* do it after we clear KEY_PRIVATE */
 761     replacementAdd (e);
 762     return e;
 763 }
 764
 765 void
 766 Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e)
 767 {
 768     debugs(47, 5, HERE << *e);
 769     replacementRemove(e); // checks swap_dirn so do it before we invalidate it
 770     // Do not unlink the file as it might be used by a subsequent entry.
 771     mapBitReset(e->swap_filen);
 772     e->swap_filen = -1;
 773     e->swap_dirn = -1;
 774     cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz);
 775     --n_disk_objects;
 776 }
 777
 778 void
 779 Fs::Ufs::UFSSwapDir::rebuild()
 780 {
 781     ++StoreController::store_dirs_rebuilding;
 782     eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1);
 783 }
 784
 785 void
 786 Fs::Ufs::UFSSwapDir::closeTmpSwapLog()
 787 {
 788     char *swaplog_path = xstrdup(logFile(NULL)); // where the swaplog should be
 789     char *tmp_path = xstrdup(logFile(".new")); // the temporary file we have generated
 790     int fd;
 791     file_close(swaplog_fd);
 792
 793     if (xrename(tmp_path, swaplog_path) < 0) {
 794         fatalf("Failed to rename log file %s to %s", tmp_path, swaplog_path);
 795     }
 796
 797     fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY);
 798
 799     if (fd < 0) {
 800         debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror());
 801         fatalf("Failed to open swap log %s", swaplog_path);
 802     }
 803
 804     xfree(swaplog_path);
 805     xfree(tmp_path);
 806     swaplog_fd = fd;
 807     debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd);
 808 }
 809
 810 FILE *
 811 Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag)
 812 {
 813     char *swaplog_path = xstrdup(logFile(NULL));
 814     char *clean_path = xstrdup(logFile(".last-clean"));
 815     char *new_path = xstrdup(logFile(".new"));
 816
 817     struct stat log_sb;
 818
 819     struct stat clean_sb;
 820     FILE *fp;
 821     int fd;
 822
 823     if (::stat(swaplog_path, &log_sb) < 0) {
 824         debugs(47, DBG_IMPORTANT, "Cache Dir #" << index << ": No log file");
 825         safe_free(swaplog_path);
 826         safe_free(clean_path);
 827         safe_free(new_path);
 828         return NULL;
 829     }
 830
 831     *zero_flag = log_sb.st_size == 0 ? 1 : 0;
 832     /* close the existing write-only FD */
 833
 834     if (swaplog_fd >= 0)
 835         file_close(swaplog_fd);
 836
 837     /* open a write-only FD for the new log */
 838     fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
 839
 840     if (fd < 0) {
 841         debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerror());
 842         fatalf("Failed to open swap log %s", new_path);
 843     }
 844
 845     swaplog_fd = fd;
 846
 847     {
 848         const StoreSwapLogHeader header;
 849         MemBuf buf;
 850         buf.init(header.record_size, header.record_size);
 851         buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
 852         // Pad to keep in sync with UFSSwapDir::writeCleanStart().
 853         memset(buf.space(), 0, header.gapSize());
 854         buf.appended(header.gapSize());
 855         file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
 856                    NULL, NULL, buf.freeFunc());
 857     }
 858
 859     /* open a read-only stream of the old log */
 860     fp = fopen(swaplog_path, "rb");
 861
 862     if (fp == NULL) {
 863         debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerror());
 864         fatalf("Failed to open swap log for reading %s", swaplog_path);
 865     }
 866
 867     memset(&clean_sb, '\0', sizeof(struct stat));
 868
 869     if (::stat(clean_path, &clean_sb) < 0)
 870         *clean_flag = 0;
 871     else if (clean_sb.st_mtime < log_sb.st_mtime)
 872         *clean_flag = 0;
 873     else
 874         *clean_flag = 1;
 875
 876     safeunlink(clean_path, 1);
 877
 878     safe_free(swaplog_path);
 879
 880     safe_free(clean_path);
 881
 882     safe_free(new_path);
 883
 884     return fp;
 885 }
 886
 887 /*
 888  * Begin the process to write clean cache state.  For AUFS this means
 889  * opening some log files and allocating write buffers.  Return 0 if
 890  * we succeed, and assign the 'func' and 'data' return pointers.
 891  */
 892 int
 893 Fs::Ufs::UFSSwapDir::writeCleanStart()
 894 {
 895     UFSCleanLog *state = new UFSCleanLog(this);
 896     StoreSwapLogHeader header;
 897 #if HAVE_FCHMOD
 898
 899     struct stat sb;
 900 #endif
 901
 902     cleanLog = NULL;
 903     state->newLog = xstrdup(logFile(".clean"));
 904     state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY);
 905
 906     if (state->fd < 0) {
 907         xfree(state->newLog);
 908         delete state;
 909         return -1;
 910     }
 911
 912     state->cur = xstrdup(logFile(NULL));
 913     state->cln = xstrdup(logFile(".last-clean"));
 914     state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1);
 915     state->outbuf_offset = 0;
 916     /*copy the header */
 917     memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
 918     // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
 919     memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
 920     state->outbuf_offset += header.record_size;
 921
 922     state->walker = repl->WalkInit(repl);
 923     ::unlink(state->cln);
 924     debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd);
 925 #if HAVE_FCHMOD
 926
 927     if (::stat(state->cur, &sb) == 0)
 928         fchmod(state->fd, sb.st_mode);
 929
 930 #endif
 931
 932     cleanLog = state;
 933     return 0;
 934 }
 935
 936 void
 937 Fs::Ufs::UFSSwapDir::writeCleanDone()
 938 {
 939     UFSCleanLog *state = (UFSCleanLog *)cleanLog;
 940     int fd;
 941
 942     if (NULL == state)
 943         return;
 944
 945     if (state->fd < 0)
 946         return;
 947
 948     state->walker->Done(state->walker);
 949
 950     if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) {
 951         debugs(50, DBG_CRITICAL, HERE << state->newLog << ": write: " << xstrerror());
 952         debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced.");
 953         file_close(state->fd);
 954         state->fd = -1;
 955         ::unlink(state->newLog);
 956     }
 957
 958     safe_free(state->outbuf);
 959     /*
 960      * You can't rename open files on Microsoft "operating systems"
 961      * so we have to close before renaming.
 962      */
 963     closeLog();
 964     /* save the fd value for a later test */
 965     fd = state->fd;
 966     /* rename */
 967
 968     if (state->fd >= 0) {
 969 #if _SQUID_OS2_ || _SQUID_WINDOWS_
 970         file_close(state->fd);
 971         state->fd = -1;
 972 #endif
 973
 974         xrename(state->newLog, state->cur);
 975     }
 976
 977     /* touch a timestamp file if we're not still validating */
 978     if (StoreController::store_dirs_rebuilding)
 979         (void) 0;
 980     else if (fd < 0)
 981         (void) 0;
 982     else
 983         file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY));
 984
 985     /* close */
 986     safe_free(state->cur);
 987
 988     safe_free(state->newLog);
 989
 990     safe_free(state->cln);
 991
 992     if (state->fd >= 0)
 993         file_close(state->fd);
 994
 995     state->fd = -1;
 996
 997     delete state;
 998
 999     cleanLog = NULL;
1000 }
1001
1002 void
1003 Fs::Ufs::UFSSwapDir::CleanEvent(void *)
1004 {
1005     static int swap_index = 0;
1006     int i;
1007     int j = 0;
1008     int n = 0;
1009     /*
1010      * Assert that there are UFS cache_dirs configured, otherwise
1011      * we should never be called.
1012      */
1013     assert(NumberOfUFSDirs);
1014
1015     if (NULL == UFSDirToGlobalDirMapping) {
1016         SwapDir *sd;
1017         /*
1018          * Initialize the little array that translates UFS cache_dir
1019          * number into the Config.cacheSwap.swapDirs array index.
1020          */
1021         UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping));
1022
1023         for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) {
1024             /* This is bogus, the controller should just clean each instance once */
1025             sd = dynamic_cast <SwapDir *>(INDEXSD(i));
1026
1027             if (!UFSSwapDir::IsUFSDir(sd))
1028                 continue;
1029
1030             UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd);
1031
1032             assert (usd);
1033
1034             UFSDirToGlobalDirMapping[n] = i;
1035             ++n;
1036
1037             j += (usd->l1 * usd->l2);
1038         }
1039
1040         assert(n == NumberOfUFSDirs);
1041         /*
1042          * Start the commonUfsDirClean() swap_index with a random
1043          * value.  j equals the total number of UFS level 2
1044          * swap directories
1045          */
1046         std::mt19937 mt(static_cast<uint32_t>(getCurrentTime() & 0xFFFFFFFF));
1047         std::uniform_int_distribution<> dist(0, j);
1048         swap_index = dist(mt);
1049     }
1050
1051     /* if the rebuild is finished, start cleaning directories. */
1052     if (0 == StoreController::store_dirs_rebuilding) {
1053         n = DirClean(swap_index);
1054         ++swap_index;
1055     }
1056
1057     eventAdd("storeDirClean", CleanEvent, NULL,
1058              15.0 * exp(-0.25 * n), 1);
1059 }
1060
1061 bool
1062 Fs::Ufs::UFSSwapDir::IsUFSDir(SwapDir * sd)
1063 {
1064     UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd);
1065     return (mySD != 0) ;
1066 }
1067
1068 /*
1069  * XXX: this is broken - it assumes all cache dirs use the same
1070  * l1 and l2 scheme. -RBC 20021215. Partial fix is in place -
1071  * if not UFSSwapDir return 0;
1072  */
1073 bool
1074 Fs::Ufs::UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2)
1075 {
1076     int D1, D2;
1077     int L1, L2;
1078     int filn = fn;
1079     assert(F0 < Config.cacheSwap.n_configured);
1080     assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0))));
1081     UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0));
1082
1083     if (!sd)
1084         return 0;
1085
1086     L1 = sd->l1;
1087
1088     L2 = sd->l2;
1089
1090     D1 = ((filn / L2) / L2) % L1;
1091
1092     if (F1 != D1)
1093         return 0;
1094
1095     D2 = (filn / L2) % L2;
1096
1097     if (F2 != D2)
1098         return 0;
1099
1100     return 1;
1101 }
1102
1103 int
1104 Fs::Ufs::UFSSwapDir::validFileno(sfileno filn, int flag) const
1105 {
1106     if (filn < 0)
1107         return 0;
1108
1109     /*
1110      * If flag is set it means out-of-range file number should
1111      * be considered invalid.
1112      */
1113     if (flag)
1114         if (filn > map->capacity())
1115             return 0;
1116
1117     return 1;
1118 }
1119
1120 void
1121 Fs::Ufs::UFSSwapDir::unlinkFile(sfileno f)
1122 {
1123     debugs(79, 3, HERE << "unlinking fileno " <<  std::setfill('0') <<
1124            std::hex << std::uppercase << std::setw(8) << f << " '" <<
1125            fullPath(f,NULL) << "'");
1126     /* commonUfsDirMapBitReset(this, f); */
1127     IO->unlinkFile(fullPath(f,NULL));
1128 }
1129
1130 bool
1131 Fs::Ufs::UFSSwapDir::unlinkdUseful() const
1132 {
1133     // unlinkd may be useful only in workers
1134     return IamWorkerProcess() && IO->io->unlinkdUseful();
1135 }
1136
1137 void
1138 Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e)
1139 {
1140     debugs(79, 3, HERE << "dirno " << index  << ", fileno "<<
1141            std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen);
1142     if (e.swap_status == SWAPOUT_DONE) {
1143         cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz);
1144         --n_disk_objects;
1145     }
1146     replacementRemove(&e);
1147     mapBitReset(e.swap_filen);
1148     UFSSwapDir::unlinkFile(e.swap_filen);
1149 }
1150
1151 void
1152 Fs::Ufs::UFSSwapDir::replacementAdd(StoreEntry * e)
1153 {
1154     debugs(47, 4, HERE << "added node " << e << " to dir " << index);
1155     repl->Add(repl, e, &e->repl);
1156 }
1157
1158 void
1159 Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e)
1160 {
1161     StorePointer SD;
1162
1163     if (e->swap_dirn < 0)
1164         return;
1165
1166     SD = INDEXSD(e->swap_dirn);
1167
1168     assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
1169
1170     debugs(47, 4, HERE << "remove node " << e << " from dir " << index);
1171
1172     repl->Remove(repl, e, &e->repl);
1173 }
1174
1175 void
1176 Fs::Ufs::UFSSwapDir::dump(StoreEntry & entry) const
1177 {
1178     storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2);
1179     dumpOptions(&entry);
1180 }
1181
1182 char *
1183 Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const
1184 {
1185     LOCAL_ARRAY(char, fullfilename, MAXPATHLEN);
1186     int L1 = l1;
1187     int L2 = l2;
1188
1189     if (!fullpath)
1190         fullpath = fullfilename;
1191
1192     fullpath[0] = '\0';
1193
1194     snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X",
1195              path,
1196              ((filn / L2) / L2) % L1,
1197              (filn / L2) % L2,
1198              filn);
1199
1200     return fullpath;
1201 }
1202
1203 int
1204 Fs::Ufs::UFSSwapDir::callback()
1205 {
1206     return IO->callback();
1207 }
1208
1209 void
1210 Fs::Ufs::UFSSwapDir::sync()
1211 {
1212     IO->sync();
1213 }
1214
1215 void
1216 Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e)
1217 {
1218     cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz);
1219     ++n_disk_objects;
1220 }
1221
1222 StoreSearch *
1223 Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *)
1224 {
1225     if (url.size())
1226         fatal ("Cannot search by url yet\n");
1227
1228     return new Fs::Ufs::StoreSearchUFS (this);
1229 }
1230
1231 void
1232 Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const
1233 {
1234     StoreSwapLogData *s = new StoreSwapLogData;
1235     s->op = (char) op;
1236     s->swap_filen = e.swap_filen;
1237     s->timestamp = e.timestamp;
1238     s->lastref = e.lastref;
1239     s->expires = e.expires;
1240     s->lastmod = e.lastmod;
1241     s->swap_file_sz = e.swap_file_sz;
1242     s->refcount = e.refcount;
1243     s->flags = e.flags;
1244     memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
1245     s->finalize();
1246     file_write(swaplog_fd,
1247                -1,
1248                s,
1249                sizeof(StoreSwapLogData),
1250                NULL,
1251                NULL,
1252                FreeObject);
1253 }
1254
1255 int
1256 Fs::Ufs::UFSSwapDir::DirClean(int swap_index)
1257 {
1258     DIR *dir_pointer = NULL;
1259
1260     LOCAL_ARRAY(char, p1, MAXPATHLEN + 1);
1261     LOCAL_ARRAY(char, p2, MAXPATHLEN + 1);
1262
1263     int files[20];
1264     int swapfileno;
1265     int fn;         /* same as swapfileno, but with dirn bits set */
1266     int n = 0;
1267     int k = 0;
1268     int N0, N1, N2;
1269     int D0, D1, D2;
1270     UFSSwapDir *SD;
1271     N0 = NumberOfUFSDirs;
1272     D0 = UFSDirToGlobalDirMapping[swap_index % N0];
1273     SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0));
1274     assert (SD);
1275     N1 = SD->l1;
1276     D1 = (swap_index / N0) % N1;
1277     N2 = SD->l2;
1278     D2 = ((swap_index / N0) / N1) % N2;
1279     snprintf(p1, MAXPATHLEN, "%s/%02X/%02X",
1280              SD->path, D1, D2);
1281     debugs(36, 3, HERE << "Cleaning directory " << p1);
1282     dir_pointer = opendir(p1);
1283
1284     if (dir_pointer == NULL) {
1285         if (errno == ENOENT) {
1286             debugs(36, DBG_CRITICAL, HERE << "WARNING: Creating " << p1);
1287             if (mkdir(p1, 0777) == 0)
1288                 return 0;
1289         }
1290
1291         debugs(50, DBG_CRITICAL, HERE << p1 << ": " << xstrerror());
1292         safeunlink(p1, 1);
1293         return 0;
1294     }
1295
1296     dirent_t *de;
1297     while ((de = readdir(dir_pointer)) != NULL && k < 20) {
1298         if (sscanf(de->d_name, "%X", &swapfileno) != 1)
1299             continue;
1300
1301         fn = swapfileno;    /* XXX should remove this cruft ! */
1302
1303         if (SD->validFileno(fn, 1))
1304             if (SD->mapBitTest(fn))
1305                 if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2))
1306                     continue;
1307
1308         files[k] = swapfileno;
1309         ++k;
1310     }
1311
1312     closedir(dir_pointer);
1313
1314     if (k == 0)
1315         return 0;
1316
1317     qsort(files, k, sizeof(int), rev_int_sort);
1318
1319     if (k > 10)
1320         k = 10;
1321
1322     for (n = 0; n < k; ++n) {
1323         debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]);
1324         snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]);
1325         safeunlink(p2, 0);
1326         ++statCounter.swap.files_cleaned;
1327     }
1328
1329     debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1);
1330     return k;
1331 }
1332