src/store_dir.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 47    Store Directory Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "Store.h"
  38 #include "MemObject.h"
  39 #include "SquidMath.h"
  40 #include "SquidTime.h"
  41 #include "SwapDir.h"
  42 #include "swap_log_op.h"
  43
  44 #if HAVE_STATVFS
  45 #if HAVE_SYS_STATVFS_H
  46 #include <sys/statvfs.h>
  47 #endif
  48 #endif /* HAVE_STATVFS */
  49 /* statfs() needs <sys/param.h> and <sys/mount.h> on BSD systems */
  50 #if HAVE_SYS_PARAM_H
  51 #include <sys/param.h>
  52 #endif
  53 #if HAVE_SYS_MOUNT_H
  54 #include <sys/mount.h>
  55 #endif
  56 /* Windows and Linux use sys/vfs.h */
  57 #if HAVE_SYS_VFS_H
  58 #include <sys/vfs.h>
  59 #endif
  60
  61 #include "StoreHashIndex.h"
  62
  63 static STDIRSELECT storeDirSelectSwapDirRoundRobin;
  64 static STDIRSELECT storeDirSelectSwapDirLeastLoad;
  65
  66 /*
  67  * store_dirs_rebuilding is initialized to _1_ as a hack so that
  68  * storeDirWriteCleanLogs() doesn't try to do anything unless _all_
  69  * cache_dirs have been read.  For example, without this hack, Squid
  70  * will try to write clean log files if -kparse fails (becasue it
  71  * calls fatal()).
  72  */
  73 int StoreController::store_dirs_rebuilding = 1;
  74
  75 StoreController::StoreController() : swapDir (new StoreHashIndex())
  76 {}
  77
  78 StoreController::~StoreController()
  79 {}
  80
  81 /*
  82  * This function pointer is set according to 'store_dir_select_algorithm'
  83  * in squid.conf.
  84  */
  85 STDIRSELECT *storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
  86
  87 void
  88 StoreController::init()
  89 {
  90     swapDir->init();
  91
  92     if (0 == strcasecmp(Config.store_dir_select_algorithm, "round-robin")) {
  93         storeDirSelectSwapDir = storeDirSelectSwapDirRoundRobin;
  94         debugs(47, 1, "Using Round Robin store dir selection");
  95     } else {
  96         storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
  97         debugs(47, 1, "Using Least Load store dir selection");
  98     }
  99 }
 100
 101 void
 102 StoreController::createOneStore(Store &aStore)
 103 {
 104     /*
 105      * On Windows, fork() is not available.
 106      * The following is a workaround for create store directories sequentially
 107      * when running on native Windows port.
 108      */
 109 #ifndef _SQUID_MSWIN_
 110
 111     if (fork())
 112         return;
 113
 114 #endif
 115
 116     aStore.create();
 117
 118 #ifndef _SQUID_MSWIN_
 119
 120     exit(0);
 121
 122 #endif
 123 }
 124
 125 void
 126 StoreController::create()
 127 {
 128     swapDir->create();
 129
 130 #ifndef _SQUID_MSWIN_
 131
 132     pid_t pid;
 133
 134     do {
 135         int status;
 136 #ifdef _SQUID_NEXT_
 137
 138         pid = wait3(&status, WNOHANG, NULL);
 139 #else
 140
 141         pid = waitpid(-1, &status, 0);
 142 #endif
 143
 144     } while (pid > 0 || (pid < 0 && errno == EINTR));
 145
 146 #endif
 147 }
 148
 149 /*
 150  * Determine whether the given directory can handle this object
 151  * size
 152  *
 153  * Note: if the object size is -1, then the only swapdirs that
 154  * will return true here are ones that have max_obj_size = -1,
 155  * ie any-sized-object swapdirs. This is a good thing.
 156  */
 157 bool
 158 SwapDir::objectSizeIsAcceptable(int64_t objsize) const
 159 {
 160     /*
 161      * If the swapdir's max_obj_size is -1, then it definitely can
 162      */
 163
 164     if (max_objsize == -1)
 165         return true;
 166
 167     /*
 168      * If the object size is -1, then if the storedir isn't -1 we
 169      * can't store it
 170      */
 171     if ((objsize == -1) && (max_objsize != -1))
 172         return false;
 173
 174     /*
 175      * Else, make sure that the max object size is larger than objsize
 176      */
 177     return max_objsize > objsize;
 178 }
 179
 180
 181 /*
 182  * This new selection scheme simply does round-robin on all SwapDirs.
 183  * A SwapDir is skipped if it is over the max_size (100%) limit, or
 184  * overloaded.
 185  */
 186 static int
 187 storeDirSelectSwapDirRoundRobin(const StoreEntry * e)
 188 {
 189     static int dirn = 0;
 190     int i;
 191     int load;
 192     RefCount<SwapDir> sd;
 193
 194     ssize_t objsize = e->objectLen();
 195     if (objsize != -1)
 196         objsize += e->mem_obj->swap_hdr_sz;
 197
 198     for (i = 0; i <= Config.cacheSwap.n_configured; i++) {
 199         if (++dirn >= Config.cacheSwap.n_configured)
 200             dirn = 0;
 201
 202         sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 203
 204         if (sd->flags.read_only)
 205             continue;
 206
 207         if (sd->cur_size > sd->max_size)
 208             continue;
 209
 210         if (!sd->objectSizeIsAcceptable(objsize))
 211             continue;
 212
 213         /* check for error or overload condition */
 214         load = sd->canStore(*e);
 215
 216         if (load < 0 || load > 1000) {
 217             continue;
 218         }
 219
 220         return dirn;
 221     }
 222
 223     return -1;
 224 }
 225
 226 /*
 227  * Spread load across all of the store directories
 228  *
 229  * Note: We should modify this later on to prefer sticking objects
 230  * in the *tightest fit* swapdir to conserve space, along with the
 231  * actual swapdir usage. But for now, this hack will do while
 232  * testing, so you should order your swapdirs in the config file
 233  * from smallest maxobjsize to unlimited (-1) maxobjsize.
 234  *
 235  * We also have to choose nleast == nconf since we need to consider
 236  * ALL swapdirs, regardless of state. Again, this is a hack while
 237  * we sort out the real usefulness of this algorithm.
 238  */
 239 static int
 240 storeDirSelectSwapDirLeastLoad(const StoreEntry * e)
 241 {
 242     ssize_t objsize;
 243     ssize_t most_free = 0, cur_free;
 244     ssize_t least_objsize = -1;
 245     int least_load = INT_MAX;
 246     int load;
 247     int dirn = -1;
 248     int i;
 249     RefCount<SwapDir> SD;
 250
 251     /* Calculate the object size */
 252     objsize = e->objectLen();
 253
 254     if (objsize != -1)
 255         objsize += e->mem_obj->swap_hdr_sz;
 256
 257     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 258         SD = dynamic_cast<SwapDir *>(INDEXSD(i));
 259         SD->flags.selected = 0;
 260         load = SD->canStore(*e);
 261
 262         if (load < 0 || load > 1000) {
 263             continue;
 264         }
 265
 266         if (!SD->objectSizeIsAcceptable(objsize))
 267             continue;
 268
 269         if (SD->flags.read_only)
 270             continue;
 271
 272         if (SD->cur_size > SD->max_size)
 273             continue;
 274
 275         if (load > least_load)
 276             continue;
 277
 278         cur_free = SD->max_size - SD->cur_size;
 279
 280         /* If the load is equal, then look in more details */
 281         if (load == least_load) {
 282             /* closest max_objsize fit */
 283
 284             if (least_objsize != -1)
 285                 if (SD->max_objsize > least_objsize || SD->max_objsize == -1)
 286                     continue;
 287
 288             /* most free */
 289             if (cur_free < most_free)
 290                 continue;
 291         }
 292
 293         least_load = load;
 294         least_objsize = SD->max_objsize;
 295         most_free = cur_free;
 296         dirn = i;
 297     }
 298
 299     if (dirn >= 0)
 300         dynamic_cast<SwapDir *>(INDEXSD(dirn))->flags.selected = 1;
 301
 302     return dirn;
 303 }
 304
 305 /*
 306  * An entry written to the swap log MUST have the following
 307  * properties.
 308  *   1.  It MUST be a public key.  It does no good to log
 309  *       a public ADD, change the key, then log a private
 310  *       DEL.  So we need to log a DEL before we change a
 311  *       key from public to private.
 312  *   2.  It MUST have a valid (> -1) swap_filen.
 313  */
 314 void
 315 storeDirSwapLog(const StoreEntry * e, int op)
 316 {
 317     assert (e);
 318     assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
 319     assert(e->swap_filen >= 0);
 320     /*
 321      * icons and such; don't write them to the swap log
 322      */
 323
 324     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 325         return;
 326
 327     assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
 328
 329     debugs(20, 3, "storeDirSwapLog: " <<
 330            swap_log_op_str[op] << " " <<
 331            e->getMD5Text() << " " <<
 332            e->swap_dirn << " " <<
 333            std::hex << std::uppercase << std::setfill('0') << std::setw(8) << e->swap_filen);
 334
 335     dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn))->logEntry(*e, op);
 336 }
 337
 338 void
 339 StoreController::updateSize(int64_t size, int sign)
 340 {
 341     fatal("StoreController has no independent size\n");
 342 }
 343
 344 void
 345 SwapDir::updateSize(int64_t size, int sign)
 346 {
 347     int64_t blks = (size + fs.blksize - 1) / fs.blksize;
 348     int64_t k = ((blks * fs.blksize) >> 10) * sign;
 349     cur_size += k;
 350     store_swap_size += k;
 351
 352     if (sign > 0)
 353         n_disk_objects++;
 354     else if (sign < 0)
 355         n_disk_objects--;
 356 }
 357
 358 void
 359 StoreController::stat(StoreEntry &output) const
 360 {
 361     storeAppendPrintf(&output, "Store Directory Statistics:\n");
 362     storeAppendPrintf(&output, "Store Entries          : %lu\n",
 363                       (unsigned long int)StoreEntry::inUseCount());
 364     storeAppendPrintf(&output, "Maximum Swap Size      : %"PRIu64" KB\n",
 365                       maxSize());
 366     storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n",
 367                       store_swap_size);
 368     storeAppendPrintf(&output, "Current Capacity       : %"PRId64"%% used, %"PRId64"%% free\n",
 369                       Math::int64Percent(store_swap_size, maxSize()),
 370                       Math::int64Percent((maxSize() - store_swap_size), maxSize()));
 371     /* FIXME Here we should output memory statistics */
 372
 373     /* now the swapDir */
 374     swapDir->stat(output);
 375 }
 376
 377 /* if needed, this could be taught to cache the result */
 378 uint64_t
 379 StoreController::maxSize() const
 380 {
 381     /* TODO: include memory cache ? */
 382     return swapDir->maxSize();
 383 }
 384
 385 uint64_t
 386 StoreController::minSize() const
 387 {
 388     /* TODO: include memory cache ? */
 389     return swapDir->minSize();
 390 }
 391
 392 void
 393 SwapDir::diskFull()
 394 {
 395     if (cur_size >= max_size)
 396         return;
 397
 398     max_size = cur_size;
 399
 400     debugs(20, 1, "WARNING: Shrinking cache_dir #" << index << " to " << cur_size << " KB");
 401 }
 402
 403 void
 404 storeDirOpenSwapLogs(void)
 405 {
 406     for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
 407         dynamic_cast<SwapDir *>(INDEXSD(dirn))->openLog();
 408 }
 409
 410 void
 411 storeDirCloseSwapLogs(void)
 412 {
 413     for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
 414         dynamic_cast<SwapDir *>(INDEXSD(dirn))->closeLog();
 415 }
 416
 417 /*
 418  *  storeDirWriteCleanLogs
 419  *
 420  *  Writes a "clean" swap log file from in-memory metadata.
 421  *  This is a rewrite of the original function to troll each
 422  *  StoreDir and write the logs, and flush at the end of
 423  *  the run. Thanks goes to Eric Stern, since this solution
 424  *  came out of his COSS code.
 425  */
 426 int
 427 storeDirWriteCleanLogs(int reopen)
 428 {
 429     const StoreEntry *e = NULL;
 430     int n = 0;
 431
 432     struct timeval start;
 433     double dt;
 434     RefCount<SwapDir> sd;
 435     int dirn;
 436     int notdone = 1;
 437
 438     if (StoreController::store_dirs_rebuilding) {
 439         debugs(20, 1, "Not currently OK to rewrite swap log.");
 440         debugs(20, 1, "storeDirWriteCleanLogs: Operation aborted.");
 441         return 0;
 442     }
 443
 444     debugs(20, 1, "storeDirWriteCleanLogs: Starting...");
 445     getCurrentTime();
 446     start = current_time;
 447
 448     for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
 449         sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 450
 451         if (sd->writeCleanStart() < 0) {
 452             debugs(20, 1, "log.clean.start() failed for dir #" << sd->index);
 453             continue;
 454         }
 455     }
 456
 457     /*
 458      * This may look inefficient as CPU wise it is more efficient to do this
 459      * sequentially, but I/O wise the parallellism helps as it allows more
 460      * hdd spindles to be active.
 461      */
 462     while (notdone) {
 463         notdone = 0;
 464
 465         for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
 466             sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 467
 468             if (NULL == sd->cleanLog)
 469                 continue;
 470
 471             e = sd->cleanLog->nextEntry();
 472
 473             if (!e)
 474                 continue;
 475
 476             notdone = 1;
 477
 478             if (!sd->canLog(*e))
 479                 continue;
 480
 481             sd->cleanLog->write(*e);
 482
 483             if ((++n & 0xFFFF) == 0) {
 484                 getCurrentTime();
 485                 debugs(20, 1, "  " << std::setw(7) << n  <<
 486                        " entries written so far.");
 487             }
 488         }
 489     }
 490
 491     /* Flush */
 492     for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++)
 493         dynamic_cast<SwapDir *>(INDEXSD(dirn))->writeCleanDone();
 494
 495     if (reopen)
 496         storeDirOpenSwapLogs();
 497
 498     getCurrentTime();
 499
 500     dt = tvSubDsec(start, current_time);
 501
 502     debugs(20, 1, "  Finished.  Wrote " << n << " entries.");
 503     debugs(20, 1, "  Took "<< std::setw(3)<< std::setprecision(2) << dt <<
 504            " seconds ("<< std::setw(6) << ((double) n / (dt > 0.0 ? dt : 1.0)) << " entries/sec).");
 505
 506
 507     return n;
 508 }
 509
 510 StoreSearch *
 511 StoreController::search(String const url, HttpRequest *request)
 512 {
 513     /* cheat, for now you can't search the memory hot cache */
 514     return swapDir->search(url, request);
 515 }
 516
 517 StorePointer
 518 StoreHashIndex::store(int const x) const
 519 {
 520     return INDEXSD(x);
 521 }
 522
 523 void
 524 StoreController::sync(void)
 525 {
 526     /* sync mem cache? */
 527     swapDir->sync();
 528 }
 529
 530 /*
 531  * handle callbacks all avaliable fs'es
 532  */
 533 int
 534 StoreController::callback()
 535 {
 536     /* This will likely double count. Thats ok. */
 537     PROF_start(storeDirCallback);
 538
 539     /* mem cache callbacks ? */
 540     int result = swapDir->callback();
 541
 542     PROF_stop(storeDirCallback);
 543
 544     return result;
 545 }
 546
 547 int
 548 storeDirGetBlkSize(const char *path, int *blksize)
 549 {
 550 #if HAVE_STATVFS
 551
 552     struct statvfs sfs;
 553
 554     if (statvfs(path, &sfs)) {
 555         debugs(50, 1, "" << path << ": " << xstrerror());
 556         *blksize = 2048;
 557         return 1;
 558     }
 559
 560     *blksize = (int) sfs.f_frsize;
 561 #else
 562
 563     struct statfs sfs;
 564
 565     if (statfs(path, &sfs)) {
 566         debugs(50, 1, "" << path << ": " << xstrerror());
 567         *blksize = 2048;
 568         return 1;
 569     }
 570
 571     *blksize = (int) sfs.f_bsize;
 572 #endif
 573     /*
 574      * Sanity check; make sure we have a meaningful value.
 575      */
 576
 577     if (*blksize < 512)
 578         *blksize = 2048;
 579
 580     return 0;
 581 }
 582
 583 #define fsbtoblk(num, fsbs, bs) \
 584     (((fsbs) != 0 && (fsbs) < (bs)) ? \
 585             (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
 586 int
 587 storeDirGetUFSStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
 588 {
 589 #if HAVE_STATVFS
 590
 591     struct statvfs sfs;
 592
 593     if (statvfs(path, &sfs)) {
 594         debugs(50, 1, "" << path << ": " << xstrerror());
 595         return 1;
 596     }
 597
 598     *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024);
 599     *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_frsize, 1024);
 600     *totl_in = (int) sfs.f_files;
 601     *free_in = (int) sfs.f_ffree;
 602 #else
 603
 604     struct statfs sfs;
 605
 606     if (statfs(path, &sfs)) {
 607         debugs(50, 1, "" << path << ": " << xstrerror());
 608         return 1;
 609     }
 610
 611     *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_bsize, 1024);
 612     *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_bsize, 1024);
 613     *totl_in = (int) sfs.f_files;
 614     *free_in = (int) sfs.f_ffree;
 615 #endif
 616
 617     return 0;
 618 }
 619
 620 void
 621 allocate_new_swapdir(SquidConfig::_cacheSwap * swap)
 622 {
 623     if (swap->swapDirs == NULL) {
 624         swap->n_allocated = 4;
 625         swap->swapDirs = static_cast<StorePointer *>(xcalloc(swap->n_allocated, sizeof(StorePointer)));
 626     }
 627
 628     if (swap->n_allocated == swap->n_configured) {
 629         StorePointer *tmp;
 630         swap->n_allocated <<= 1;
 631         tmp = static_cast<StorePointer *>(xcalloc(swap->n_allocated, sizeof(StorePointer)));
 632         xmemcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *));
 633         xfree(swap->swapDirs);
 634         swap->swapDirs = tmp;
 635     }
 636 }
 637
 638 void
 639 free_cachedir(SquidConfig::_cacheSwap * swap)
 640 {
 641     int i;
 642     /* DON'T FREE THESE FOR RECONFIGURE */
 643
 644     if (reconfiguring)
 645         return;
 646
 647     for (i = 0; i < swap->n_configured; i++) {
 648         /* TODO XXX this lets the swapdir free resources asynchronously
 649         * swap->swapDirs[i]->deactivate();
 650         * but there may be such a means already.
 651         * RBC 20041225
 652         */
 653         swap->swapDirs[i] = NULL;
 654     }
 655
 656     safe_free(swap->swapDirs);
 657     swap->swapDirs = NULL;
 658     swap->n_allocated = 0;
 659     swap->n_configured = 0;
 660 }
 661
 662 /* this should be a virtual method on StoreEntry,
 663  * i.e. e->referenced()
 664  * so that the entry can notify the creating Store
 665  */
 666 void
 667 StoreController::reference(StoreEntry &e)
 668 {
 669     /* Notify the fs that we're referencing this object again */
 670
 671     if (e.swap_dirn > -1)
 672         e.store()->reference(e);
 673
 674     /* Notify the memory cache that we're referencing this object again */
 675     if (e.mem_obj) {
 676         if (mem_policy->Referenced)
 677             mem_policy->Referenced(mem_policy, &e, &e.mem_obj->repl);
 678     }
 679 }
 680
 681 void
 682 StoreController::dereference(StoreEntry & e)
 683 {
 684     /* Notify the fs that we're not referencing this object any more */
 685
 686     if (e.swap_filen > -1)
 687         e.store()->dereference(e);
 688
 689     /* Notify the memory cache that we're not referencing this object any more */
 690     if (e.mem_obj) {
 691         if (mem_policy->Dereferenced)
 692             mem_policy->Dereferenced(mem_policy, &e, &e.mem_obj->repl);
 693     }
 694 }
 695
 696 StoreEntry *
 697
 698 StoreController::get
 699 (const cache_key *key)
 700 {
 701
 702     return swapDir->get
 703            (key);
 704 }
 705
 706 void
 707
 708 StoreController::get
 709 (String const key, STOREGETCLIENT aCallback, void *aCallbackData)
 710 {
 711     fatal("not implemented");
 712 }
 713
 714 StoreHashIndex::StoreHashIndex()
 715 {
 716     if (store_table)
 717         abort();
 718     assert (store_table == NULL);
 719 }
 720
 721 StoreHashIndex::~StoreHashIndex()
 722 {
 723     if (store_table) {
 724         hashFreeItems(store_table, destroyStoreEntry);
 725         hashFreeMemory(store_table);
 726         store_table = NULL;
 727     }
 728 }
 729
 730 int
 731 StoreHashIndex::callback()
 732 {
 733     int result = 0;
 734     int j;
 735     static int ndir = 0;
 736
 737     do {
 738         j = 0;
 739
 740         for (int i = 0; i < Config.cacheSwap.n_configured; i++) {
 741             if (ndir >= Config.cacheSwap.n_configured)
 742                 ndir = ndir % Config.cacheSwap.n_configured;
 743
 744             int temp_result = store(ndir)->callback();
 745
 746             ++ndir;
 747
 748             j += temp_result;
 749
 750             result += temp_result;
 751
 752             if (j > 100)
 753                 fatal ("too much io\n");
 754         }
 755     } while (j > 0);
 756
 757     ndir++;
 758
 759     return result;
 760 }
 761
 762 void
 763 StoreHashIndex::create()
 764 {
 765     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 766         store(i)->create();
 767 }
 768
 769 /* Lookup an object in the cache.
 770  * return just a reference to object, don't start swapping in yet. */
 771 StoreEntry *
 772
 773 StoreHashIndex::get
 774 (const cache_key *key)
 775 {
 776     PROF_start(storeGet);
 777     debugs(20, 3, "storeGet: looking up " << storeKeyText(key));
 778     StoreEntry *p = static_cast<StoreEntry *>(hash_lookup(store_table, key));
 779     PROF_stop(storeGet);
 780     return p;
 781 }
 782
 783 void
 784
 785 StoreHashIndex::get
 786 (String const key, STOREGETCLIENT aCallback, void *aCallbackData)
 787 {
 788     fatal("not implemented");
 789 }
 790
 791 void
 792 StoreHashIndex::init()
 793 {
 794     /* Calculate size of hash table (maximum currently 64k buckets).  */
 795     /* this is very bogus, its specific to the any Store maintaining an
 796      * in-core index, not global */
 797     size_t buckets = (Store::Root().maxSize() + ( Config.memMaxSize >> 10)) / Config.Store.avgObjectSize;
 798     debugs(20, 1, "Swap maxSize " << Store::Root().maxSize() <<
 799            " + " << ( Config.memMaxSize >> 10) << " KB, estimated " << buckets << " objects");
 800     buckets /= Config.Store.objectsPerBucket;
 801     debugs(20, 1, "Target number of buckets: " << buckets);
 802     /* ideally the full scan period should be configurable, for the
 803      * moment it remains at approximately 24 hours.  */
 804     store_hash_buckets = storeKeyHashBuckets(buckets);
 805     debugs(20, 1, "Using " << store_hash_buckets << " Store buckets");
 806     debugs(20, 1, "Max Mem  size: " << ( Config.memMaxSize >> 10) << " KB");
 807     debugs(20, 1, "Max Swap size: " << Store::Root().maxSize() << " KB");
 808
 809     store_table = hash_create(storeKeyHashCmp,
 810                               store_hash_buckets, storeKeyHashHash);
 811
 812     for (int i = 0; i < Config.cacheSwap.n_configured; i++) {
 813         /* this starts a search of the store dirs, loading their
 814          * index. under the new Store api this should be
 815          * driven by the StoreHashIndex, not by each store.
 816         *
 817         * That is, the HashIndex should perform a search of each dir it is
 818         * indexing to do the hash insertions. The search is then able to
 819         * decide 'from-memory', or 'from-clean-log' or 'from-dirty-log' or
 820         * 'from-no-log'.
 821         *
 822          * Step 1: make the store rebuilds use a search internally
 823         * Step 2: change the search logic to use the four modes described
 824         *         above
 825         * Step 3: have the hash index walk the searches itself.
 826          */
 827         store(i)->init();
 828     }
 829 }
 830
 831 uint64_t
 832 StoreHashIndex::maxSize() const
 833 {
 834     uint64_t result = 0;
 835
 836     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 837         result += store(i)->maxSize();
 838
 839     return result;
 840 }
 841
 842 uint64_t
 843 StoreHashIndex::minSize() const
 844 {
 845     uint64_t result = 0;
 846
 847     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 848         result += store(i)->minSize();
 849
 850     return result;
 851 }
 852
 853 void
 854 StoreHashIndex::stat(StoreEntry & output) const
 855 {
 856     int i;
 857
 858     /* Now go through each store, calling its stat routine */
 859
 860     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 861         storeAppendPrintf(&output, "\n");
 862         store(i)->stat(output);
 863     }
 864 }
 865
 866 void
 867 StoreHashIndex::reference(StoreEntry&)
 868 {}
 869
 870 void
 871 StoreHashIndex::dereference(StoreEntry&)
 872 {}
 873
 874 void
 875 StoreHashIndex::maintain()
 876 {
 877     int i;
 878     /* walk each fs */
 879
 880     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 881         /* XXX FixMe: This should be done "in parallell" on the different
 882          * cache_dirs, not one at a time.
 883          */
 884         /* call the maintain function .. */
 885         store(i)->maintain();
 886     }
 887 }
 888
 889 void
 890 StoreHashIndex::updateSize(int64_t, int)
 891 {}
 892
 893 void
 894 StoreHashIndex::sync()
 895 {
 896     for (int i = 0; i < Config.cacheSwap.n_configured; ++i)
 897         store(i)->sync();
 898 }
 899
 900 StoreSearch *
 901 StoreHashIndex::search(String const url, HttpRequest *)
 902 {
 903     if (url.size())
 904         fatal ("Cannot search by url yet\n");
 905
 906     return new StoreSearchHashIndex (this);
 907 }
 908
 909 CBDATA_CLASS_INIT(StoreSearchHashIndex);
 910
 911 StoreSearchHashIndex::StoreSearchHashIndex(RefCount<StoreHashIndex> aSwapDir) : sd(aSwapDir), _done (false), bucket (0)
 912 {}
 913
 914 /* do not link
 915 StoreSearchHashIndex::StoreSearchHashIndex(StoreSearchHashIndex const &);
 916 */
 917
 918 StoreSearchHashIndex::~StoreSearchHashIndex()
 919 {}
 920
 921 void
 922 StoreSearchHashIndex::next(void (aCallback)(void *), void *aCallbackData)
 923 {
 924     next();
 925     aCallback (aCallbackData);
 926 }
 927
 928 bool
 929 StoreSearchHashIndex::next()
 930 {
 931     if (entries.size())
 932         entries.pop_back();
 933
 934     while (!isDone() && !entries.size())
 935         copyBucket();
 936
 937     return currentItem() != NULL;
 938 }
 939
 940 bool
 941 StoreSearchHashIndex::error() const
 942 {
 943     return false;
 944 }
 945
 946 bool
 947 StoreSearchHashIndex::isDone() const
 948 {
 949     return bucket >= store_hash_buckets || _done;
 950 }
 951
 952 StoreEntry *
 953 StoreSearchHashIndex::currentItem()
 954 {
 955     if (!entries.size())
 956         return NULL;
 957
 958     return entries.back();
 959 }
 960
 961 void
 962 StoreSearchHashIndex::copyBucket()
 963 {
 964     /* probably need to lock the store entries...
 965      * we copy them all to prevent races on the links. */
 966     debugs(47, 3, "StoreSearchHashIndex::copyBucket #" << bucket);
 967     assert (!entries.size());
 968     hash_link *link_ptr = NULL;
 969     hash_link *link_next = NULL;
 970     link_next = hash_get_bucket(store_table, bucket);
 971
 972     while (NULL != (link_ptr = link_next)) {
 973         link_next = link_ptr->next;
 974         StoreEntry *e = (StoreEntry *) link_ptr;
 975
 976         entries.push_back(e);
 977     }
 978
 979     bucket++;
 980     debugs(47,3, "got entries: " << entries.size());
 981 }