src/store_dir.cc

   1
   2 /*
   3  * $Id: store_dir.cc,v 1.162 2007/08/13 17:20:51 hno Exp $
   4  *
   5  * DEBUG: section 47    Store Directory Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "Store.h"
  38 #include "MemObject.h"
  39 #include "SquidTime.h"
  40 #include "SwapDir.h"
  41
  42 #if HAVE_STATVFS
  43 #if HAVE_SYS_STATVFS_H
  44 #include <sys/statvfs.h>
  45 #endif
  46 #endif /* HAVE_STATVFS */
  47 /* statfs() needs <sys/param.h> and <sys/mount.h> on BSD systems */
  48 #if HAVE_SYS_PARAM_H
  49 #include <sys/param.h>
  50 #endif
  51 #if HAVE_SYS_MOUNT_H
  52 #include <sys/mount.h>
  53 #endif
  54 /* Windows and Linux use sys/vfs.h */
  55 #if HAVE_SYS_VFS_H
  56 #include <sys/vfs.h>
  57 #endif
  58
  59 #include "StoreHashIndex.h"
  60
  61 static STDIRSELECT storeDirSelectSwapDirRoundRobin;
  62 static STDIRSELECT storeDirSelectSwapDirLeastLoad;
  63
  64 /*
  65  * store_dirs_rebuilding is initialized to _1_ as a hack so that
  66  * storeDirWriteCleanLogs() doesn't try to do anything unless _all_
  67  * cache_dirs have been read.  For example, without this hack, Squid
  68  * will try to write clean log files if -kparse fails (becasue it
  69  * calls fatal()).
  70  */
  71 int StoreController::store_dirs_rebuilding = 1;
  72
  73 StoreController::StoreController() : swapDir (new StoreHashIndex())
  74 {}
  75
  76 StoreController::~StoreController()
  77 {}
  78
  79 /*
  80  * This function pointer is set according to 'store_dir_select_algorithm'
  81  * in squid.conf.
  82  */
  83 STDIRSELECT *storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
  84
  85 void
  86 StoreController::init()
  87 {
  88     swapDir->init();
  89
  90     if (0 == strcasecmp(Config.store_dir_select_algorithm, "round-robin")) {
  91         storeDirSelectSwapDir = storeDirSelectSwapDirRoundRobin;
  92         debugs(47, 1, "Using Round Robin store dir selection");
  93     } else {
  94         storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
  95         debugs(47, 1, "Using Least Load store dir selection");
  96     }
  97 }
  98
  99 void
 100 StoreController::createOneStore(Store &aStore)
 101 {
 102     /*
 103      * On Windows, fork() is not available.
 104      * The following is a workaround for create store directories sequentially
 105      * when running on native Windows port.
 106      */
 107 #ifndef _SQUID_MSWIN_
 108
 109     if (fork())
 110         return;
 111
 112 #endif
 113
 114     aStore.create();
 115
 116 #ifndef _SQUID_MSWIN_
 117
 118     exit(0);
 119
 120 #endif
 121 }
 122
 123 void
 124 StoreController::create()
 125 {
 126     swapDir->create();
 127
 128 #ifndef _SQUID_MSWIN_
 129
 130     pid_t pid;
 131
 132     do {
 133         int status;
 134 #ifdef _SQUID_NEXT_
 135
 136         pid = wait3(&status, WNOHANG, NULL);
 137 #else
 138
 139         pid = waitpid(-1, &status, 0);
 140 #endif
 141
 142     } while (pid > 0 || (pid < 0 && errno == EINTR));
 143
 144 #endif
 145 }
 146
 147 /*
 148  * Determine whether the given directory can handle this object
 149  * size
 150  *
 151  * Note: if the object size is -1, then the only swapdirs that
 152  * will return true here are ones that have max_obj_size = -1,
 153  * ie any-sized-object swapdirs. This is a good thing.
 154  */
 155 bool
 156 SwapDir::objectSizeIsAcceptable(ssize_t objsize) const
 157 {
 158     /*
 159      * If the swapdir's max_obj_size is -1, then it definitely can
 160      */
 161
 162     if (max_objsize == -1)
 163         return true;
 164
 165     /*
 166      * If the object size is -1, then if the storedir isn't -1 we
 167      * can't store it
 168      */
 169     if ((objsize == -1) && (max_objsize != -1))
 170         return false;
 171
 172     /*
 173      * Else, make sure that the max object size is larger than objsize
 174      */
 175     return max_objsize > objsize;
 176 }
 177
 178
 179 /*
 180  * This new selection scheme simply does round-robin on all SwapDirs.
 181  * A SwapDir is skipped if it is over the max_size (100%) limit, or
 182  * overloaded.
 183  */
 184 static int
 185 storeDirSelectSwapDirRoundRobin(const StoreEntry * e)
 186 {
 187     static int dirn = 0;
 188     int i;
 189     int load;
 190     RefCount<SwapDir> sd;
 191
 192     for (i = 0; i <= Config.cacheSwap.n_configured; i++) {
 193         if (++dirn >= Config.cacheSwap.n_configured)
 194             dirn = 0;
 195
 196         sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 197
 198         if (sd->flags.read_only)
 199             continue;
 200
 201         if (sd->cur_size > sd->max_size)
 202             continue;
 203
 204         if (!sd->objectSizeIsAcceptable(e->objectLen()))
 205             continue;
 206
 207         /* check for error or overload condition */
 208         load = sd->canStore(*e);
 209
 210         if (load < 0 || load > 1000) {
 211             continue;
 212         }
 213
 214         return dirn;
 215     }
 216
 217     return -1;
 218 }
 219
 220 /*
 221  * Spread load across all of the store directories
 222  *
 223  * Note: We should modify this later on to prefer sticking objects
 224  * in the *tightest fit* swapdir to conserve space, along with the
 225  * actual swapdir usage. But for now, this hack will do while
 226  * testing, so you should order your swapdirs in the config file
 227  * from smallest maxobjsize to unlimited (-1) maxobjsize.
 228  *
 229  * We also have to choose nleast == nconf since we need to consider
 230  * ALL swapdirs, regardless of state. Again, this is a hack while
 231  * we sort out the real usefulness of this algorithm.
 232  */
 233 static int
 234 storeDirSelectSwapDirLeastLoad(const StoreEntry * e)
 235 {
 236     ssize_t objsize;
 237     ssize_t most_free = 0, cur_free;
 238     ssize_t least_objsize = -1;
 239     int least_load = INT_MAX;
 240     int load;
 241     int dirn = -1;
 242     int i;
 243     RefCount<SwapDir> SD;
 244
 245     /* Calculate the object size */
 246     objsize = e->objectLen();
 247
 248     if (objsize != -1)
 249         objsize += e->mem_obj->swap_hdr_sz;
 250
 251     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 252         SD = dynamic_cast<SwapDir *>(INDEXSD(i));
 253         SD->flags.selected = 0;
 254         load = SD->canStore(*e);
 255
 256         if (load < 0 || load > 1000) {
 257             continue;
 258         }
 259
 260         if (!SD->objectSizeIsAcceptable(objsize))
 261             continue;
 262
 263         if (SD->flags.read_only)
 264             continue;
 265
 266         if (SD->cur_size > SD->max_size)
 267             continue;
 268
 269         if (load > least_load)
 270             continue;
 271
 272         cur_free = SD->max_size - SD->cur_size;
 273
 274         /* If the load is equal, then look in more details */
 275         if (load == least_load) {
 276             /* closest max_objsize fit */
 277
 278             if (least_objsize != -1)
 279                 if (SD->max_objsize > least_objsize || SD->max_objsize == -1)
 280                     continue;
 281
 282             /* most free */
 283             if (cur_free < most_free)
 284                 continue;
 285         }
 286
 287         least_load = load;
 288         least_objsize = SD->max_objsize;
 289         most_free = cur_free;
 290         dirn = i;
 291     }
 292
 293     if (dirn >= 0)
 294         dynamic_cast<SwapDir *>(INDEXSD(dirn))->flags.selected = 1;
 295
 296     return dirn;
 297 }
 298
 299 /*
 300  * An entry written to the swap log MUST have the following
 301  * properties.
 302  *   1.  It MUST be a public key.  It does no good to log
 303  *       a public ADD, change the key, then log a private
 304  *       DEL.  So we need to log a DEL before we change a
 305  *       key from public to private.
 306  *   2.  It MUST have a valid (> -1) swap_filen.
 307  */
 308 void
 309 storeDirSwapLog(const StoreEntry * e, int op)
 310 {
 311     assert (e);
 312     assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
 313     assert(e->swap_filen >= 0);
 314     /*
 315      * icons and such; don't write them to the swap log
 316      */
 317
 318     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 319         return;
 320
 321     assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
 322
 323     debugs(20, 3, "storeDirSwapLog: " <<
 324            swap_log_op_str[op] << " " <<
 325            e->getMD5Text() << " " <<
 326            e->swap_dirn << " " <<
 327            std::hex << std::uppercase << std::setfill('0') << std::setw(8) << e->swap_filen);
 328
 329     dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn))->logEntry(*e, op);
 330 }
 331
 332 void
 333 StoreController::updateSize(int64_t size, int sign)
 334 {
 335     fatal("StoreController has no independent size\n");
 336 }
 337
 338 void
 339 SwapDir::updateSize(int64_t size, int sign)
 340 {
 341     int blks = (size + fs.blksize - 1) / fs.blksize;
 342     int k = (blks * fs.blksize >> 10) * sign;
 343     cur_size += k;
 344     store_swap_size += k;
 345
 346     if (sign > 0)
 347         n_disk_objects++;
 348     else if (sign < 0)
 349         n_disk_objects--;
 350 }
 351
 352 void
 353 StoreController::stat(StoreEntry &output) const
 354 {
 355     storeAppendPrintf(&output, "Store Directory Statistics:\n");
 356     storeAppendPrintf(&output, "Store Entries          : %lu\n",
 357                       (unsigned long int)StoreEntry::inUseCount());
 358     storeAppendPrintf(&output, "Maximum Swap Size      : %8ld KB\n",
 359                       (long int) maxSize());
 360     storeAppendPrintf(&output, "Current Store Swap Size: %8lu KB\n",
 361                       store_swap_size);
 362     storeAppendPrintf(&output, "Current Capacity       : %d%% used, %d%% free\n",
 363                       percent((int) store_swap_size, (int) maxSize()),
 364                       percent((int) (maxSize() - store_swap_size), (int) maxSize()));
 365     /* FIXME Here we should output memory statistics */
 366
 367     /* now the swapDir */
 368     swapDir->stat(output);
 369 }
 370
 371 /* if needed, this could be taught to cache the result */
 372 size_t
 373 StoreController::maxSize() const
 374 {
 375     /* TODO: include memory cache ? */
 376     return swapDir->maxSize();
 377 }
 378
 379 size_t
 380 StoreController::minSize() const
 381 {
 382     /* TODO: include memory cache ? */
 383     return swapDir->minSize();
 384 }
 385
 386 void
 387 SwapDir::diskFull()
 388 {
 389     if (cur_size >= max_size)
 390         return;
 391
 392     max_size = cur_size;
 393
 394     debugs(20, 1, "WARNING: Shrinking cache_dir #" << index << " to " << cur_size << " KB");
 395 }
 396
 397 void
 398 storeDirOpenSwapLogs(void)
 399 {
 400     for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
 401         dynamic_cast<SwapDir *>(INDEXSD(dirn))->openLog();
 402 }
 403
 404 void
 405 storeDirCloseSwapLogs(void)
 406 {
 407     for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
 408         dynamic_cast<SwapDir *>(INDEXSD(dirn))->closeLog();
 409 }
 410
 411 /*
 412  *  storeDirWriteCleanLogs
 413  *
 414  *  Writes a "clean" swap log file from in-memory metadata.
 415  *  This is a rewrite of the original function to troll each
 416  *  StoreDir and write the logs, and flush at the end of
 417  *  the run. Thanks goes to Eric Stern, since this solution
 418  *  came out of his COSS code.
 419  */
 420 int
 421 storeDirWriteCleanLogs(int reopen)
 422 {
 423     const StoreEntry *e = NULL;
 424     int n = 0;
 425
 426     struct timeval start;
 427     double dt;
 428     RefCount<SwapDir> sd;
 429     int dirn;
 430     int notdone = 1;
 431
 432     if (StoreController::store_dirs_rebuilding) {
 433         debugs(20, 1, "Not currently OK to rewrite swap log.");
 434         debugs(20, 1, "storeDirWriteCleanLogs: Operation aborted.");
 435         return 0;
 436     }
 437
 438     debugs(20, 1, "storeDirWriteCleanLogs: Starting...");
 439     getCurrentTime();
 440     start = current_time;
 441
 442     for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
 443         sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 444
 445         if (sd->writeCleanStart() < 0) {
 446             debugs(20, 1, "log.clean.start() failed for dir #" << sd->index);
 447             continue;
 448         }
 449     }
 450
 451     /*
 452      * This may look inefficient as CPU wise it is more efficient to do this
 453      * sequentially, but I/O wise the parallellism helps as it allows more
 454      * hdd spindles to be active.
 455      */
 456     while (notdone) {
 457         notdone = 0;
 458
 459         for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) {
 460             sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
 461
 462             if (NULL == sd->cleanLog)
 463                 continue;
 464
 465             e = sd->cleanLog->nextEntry();
 466
 467             if (!e)
 468                 continue;
 469
 470             notdone = 1;
 471
 472             if (!sd->canLog(*e))
 473                 continue;
 474
 475             sd->cleanLog->write(*e);
 476
 477             if ((++n & 0xFFFF) == 0) {
 478                 getCurrentTime();
 479                 debugs(20, 1, "  " << std::setw(7) << n  <<
 480                        " entries written so far.");
 481             }
 482         }
 483     }
 484
 485     /* Flush */
 486     for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++)
 487         dynamic_cast<SwapDir *>(INDEXSD(dirn))->writeCleanDone();
 488
 489     if (reopen)
 490         storeDirOpenSwapLogs();
 491
 492     getCurrentTime();
 493
 494     dt = tvSubDsec(start, current_time);
 495
 496     debugs(20, 1, "  Finished.  Wrote " << n << " entries.");
 497     debugs(20, 1, "  Took "<< std::setw(3)<< std::setprecision(2) << dt <<
 498            " seconds ("<< std::setw(6) << ((double) n / (dt > 0.0 ? dt : 1.0)) << " entries/sec).");
 499
 500
 501     return n;
 502 }
 503
 504 StoreSearch *
 505 StoreController::search(String const url, HttpRequest *request)
 506 {
 507     /* cheat, for now you can't search the memory hot cache */
 508     return swapDir->search(url, request);
 509 }
 510
 511 StorePointer
 512 StoreHashIndex::store(int const x) const
 513 {
 514     return INDEXSD(x);
 515 }
 516
 517 void
 518 StoreController::sync(void)
 519 {
 520     /* sync mem cache? */
 521     swapDir->sync();
 522 }
 523
 524 /*
 525  * handle callbacks all avaliable fs'es
 526  */
 527 int
 528 StoreController::callback()
 529 {
 530     /* This will likely double count. Thats ok. */
 531     PROF_start(storeDirCallback);
 532
 533     /* mem cache callbacks ? */
 534     int result = swapDir->callback();
 535
 536     PROF_stop(storeDirCallback);
 537
 538     return result;
 539 }
 540
 541 int
 542 storeDirGetBlkSize(const char *path, int *blksize)
 543 {
 544 #if HAVE_STATVFS
 545
 546     struct statvfs sfs;
 547
 548     if (statvfs(path, &sfs)) {
 549         debugs(50, 1, "" << path << ": " << xstrerror());
 550         *blksize = 2048;
 551         return 1;
 552     }
 553
 554     *blksize = (int) sfs.f_frsize;
 555 #else
 556
 557     struct statfs sfs;
 558
 559     if (statfs(path, &sfs)) {
 560         debugs(50, 1, "" << path << ": " << xstrerror());
 561         *blksize = 2048;
 562         return 1;
 563     }
 564
 565     *blksize = (int) sfs.f_bsize;
 566 #endif
 567     /*
 568      * Sanity check; make sure we have a meaningful value.
 569      */
 570
 571     if (*blksize < 512)
 572         *blksize = 2048;
 573
 574     return 0;
 575 }
 576
 577 #define fsbtoblk(num, fsbs, bs) \
 578     (((fsbs) != 0 && (fsbs) < (bs)) ? \
 579             (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
 580 int
 581 storeDirGetUFSStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
 582 {
 583 #if HAVE_STATVFS
 584
 585     struct statvfs sfs;
 586
 587     if (statvfs(path, &sfs)) {
 588         debugs(50, 1, "" << path << ": " << xstrerror());
 589         return 1;
 590     }
 591
 592     *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024);
 593     *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_frsize, 1024);
 594     *totl_in = (int) sfs.f_files;
 595     *free_in = (int) sfs.f_ffree;
 596 #else
 597
 598     struct statfs sfs;
 599
 600     if (statfs(path, &sfs)) {
 601         debugs(50, 1, "" << path << ": " << xstrerror());
 602         return 1;
 603     }
 604
 605     *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_bsize, 1024);
 606     *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_bsize, 1024);
 607     *totl_in = (int) sfs.f_files;
 608     *free_in = (int) sfs.f_ffree;
 609 #endif
 610
 611     return 0;
 612 }
 613
 614 void
 615 allocate_new_swapdir(_SquidConfig::_cacheSwap * swap)
 616 {
 617     if (swap->swapDirs == NULL) {
 618         swap->n_allocated = 4;
 619         swap->swapDirs = static_cast<StorePointer *>(xcalloc(swap->n_allocated, sizeof(StorePointer)));
 620     }
 621
 622     if (swap->n_allocated == swap->n_configured) {
 623         StorePointer *tmp;
 624         swap->n_allocated <<= 1;
 625         tmp = static_cast<StorePointer *>(xcalloc(swap->n_allocated, sizeof(StorePointer)));
 626         xmemcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *));
 627         xfree(swap->swapDirs);
 628         swap->swapDirs = tmp;
 629     }
 630 }
 631
 632 void
 633 free_cachedir(_SquidConfig::_cacheSwap * swap)
 634 {
 635     int i;
 636     /* DON'T FREE THESE FOR RECONFIGURE */
 637
 638     if (reconfiguring)
 639         return;
 640
 641     for (i = 0; i < swap->n_configured; i++) {
 642         /* TODO XXX this lets the swapdir free resources asynchronously
 643         * swap->swapDirs[i]->deactivate();
 644         * but there may be such a means already.
 645         * RBC 20041225
 646         */
 647         swap->swapDirs[i] = NULL;
 648     }
 649
 650     safe_free(swap->swapDirs);
 651     swap->swapDirs = NULL;
 652     swap->n_allocated = 0;
 653     swap->n_configured = 0;
 654 }
 655
 656 /* this should be a virtual method on StoreEntry,
 657  * i.e. e->referenced()
 658  * so that the entry can notify the creating Store
 659  */
 660 void
 661 StoreController::reference(StoreEntry &e)
 662 {
 663     /* Notify the fs that we're referencing this object again */
 664
 665     if (e.swap_dirn > -1)
 666         e.store()->reference(e);
 667
 668     /* Notify the memory cache that we're referencing this object again */
 669     if (e.mem_obj) {
 670         if (mem_policy->Referenced)
 671             mem_policy->Referenced(mem_policy, &e, &e.mem_obj->repl);
 672     }
 673 }
 674
 675 void
 676 StoreController::dereference(StoreEntry & e)
 677 {
 678     /* Notify the fs that we're not referencing this object any more */
 679
 680     if (e.swap_filen > -1)
 681         e.store()->dereference(e);
 682
 683     /* Notify the memory cache that we're not referencing this object any more */
 684     if (e.mem_obj) {
 685         if (mem_policy->Dereferenced)
 686             mem_policy->Dereferenced(mem_policy, &e, &e.mem_obj->repl);
 687     }
 688 }
 689
 690 StoreEntry *
 691
 692 StoreController::get
 693     (const cache_key *key)
 694 {
 695
 696     return swapDir->get
 697            (key);
 698 }
 699
 700 void
 701
 702 StoreController::get
 703     (String const key, STOREGETCLIENT callback, void *cbdata)
 704 {
 705     fatal("not implemented");
 706 }
 707
 708 StoreHashIndex::StoreHashIndex()
 709 {
 710     if (store_table)
 711         abort();
 712     assert (store_table == NULL);
 713 }
 714
 715 StoreHashIndex::~StoreHashIndex()
 716 {
 717     if (store_table) {
 718         hashFreeItems(store_table, destroyStoreEntry);
 719         hashFreeMemory(store_table);
 720         store_table = NULL;
 721     }
 722 }
 723
 724 int
 725 StoreHashIndex::callback()
 726 {
 727     int result = 0;
 728     int j;
 729     static int ndir = 0;
 730
 731     do {
 732         j = 0;
 733
 734         for (int i = 0; i < Config.cacheSwap.n_configured; i++) {
 735             if (ndir >= Config.cacheSwap.n_configured)
 736                 ndir = ndir % Config.cacheSwap.n_configured;
 737
 738             int temp_result = store(ndir)->callback();
 739
 740             ++ndir;
 741
 742             j += temp_result;
 743
 744             result += temp_result;
 745
 746             if (j > 100)
 747                 fatal ("too much io\n");
 748         }
 749     } while (j > 0);
 750
 751     ndir++;
 752
 753     return result;
 754 }
 755
 756 void
 757 StoreHashIndex::create()
 758 {
 759     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 760         store(i)->create();
 761 }
 762
 763 /* Lookup an object in the cache.
 764  * return just a reference to object, don't start swapping in yet. */
 765 StoreEntry *
 766
 767 StoreHashIndex::get
 768     (const cache_key *key)
 769 {
 770     PROF_start(storeGet);
 771     debugs(20, 3, "storeGet: looking up " << storeKeyText(key));
 772     StoreEntry *p = static_cast<StoreEntry *>(hash_lookup(store_table, key));
 773     PROF_stop(storeGet);
 774     return p;
 775 }
 776
 777 void
 778
 779 StoreHashIndex::get
 780     (String const key, STOREGETCLIENT callback, void *cbdata)
 781 {
 782     fatal("not implemented");
 783 }
 784
 785 void
 786 StoreHashIndex::init()
 787 {
 788     /* Calculate size of hash table (maximum currently 64k buckets).  */
 789     /* this is very bogus, its specific to the any Store maintaining an
 790      * in-core index, not global */
 791     size_t buckets = Store::Root().maxSize() / Config.Store.avgObjectSize;
 792     debugs(20, 1, "Swap maxSize " << Store::Root().maxSize() <<
 793            " KB, estimated " << buckets << " objects");
 794     buckets /= Config.Store.objectsPerBucket;
 795     debugs(20, 1, "Target number of buckets: " << buckets);
 796     /* ideally the full scan period should be configurable, for the
 797      * moment it remains at approximately 24 hours.  */
 798     store_hash_buckets = storeKeyHashBuckets(buckets);
 799     debugs(20, 1, "Using " << store_hash_buckets << " Store buckets");
 800     debugs(20, 1, "Max Mem  size: " << ( Config.memMaxSize >> 10) << " KB");
 801     debugs(20, 1, "Max Swap size: " << Store::Root().maxSize() << " KB");
 802
 803     store_table = hash_create(storeKeyHashCmp,
 804                               store_hash_buckets, storeKeyHashHash);
 805
 806     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 807         /* this starts a search of the store dirs, loading their
 808          * index. under the new Store api this should be
 809          * driven by the StoreHashIndex, not by each store.
 810         *
 811         * That is, the HashIndex should perform a search of each dir it is
 812         * indexing to do the hash insertions. The search is then able to
 813         * decide 'from-memory', or 'from-clean-log' or 'from-dirty-log' or
 814         * 'from-no-log'.
 815         *
 816          * Step 1: make the store rebuilds use a search internally
 817         * Step 2: change the search logic to use the four modes described
 818         *         above
 819         * Step 3: have the hash index walk the searches itself.
 820          */
 821         store(i)->init();
 822
 823 }
 824
 825 size_t
 826 StoreHashIndex::maxSize() const
 827 {
 828     int i;
 829     size_t result = 0;
 830
 831     for (i = 0; i < Config.cacheSwap.n_configured; i++)
 832         result += store(i)->maxSize();
 833
 834     return result;
 835 }
 836
 837 size_t
 838 StoreHashIndex::minSize() const
 839 {
 840     size_t result = 0;
 841
 842     for (int i = 0; i < Config.cacheSwap.n_configured; i++)
 843         result += store(i)->minSize();
 844
 845     return result;
 846 }
 847
 848 void
 849 StoreHashIndex::stat(StoreEntry & output) const
 850 {
 851     int i;
 852
 853     /* Now go through each store, calling its stat routine */
 854
 855     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 856         storeAppendPrintf(&output, "\n");
 857         store(i)->stat(output);
 858     }
 859 }
 860
 861 void
 862 StoreHashIndex::reference(StoreEntry&)
 863 {}
 864
 865 void
 866 StoreHashIndex::dereference(StoreEntry&)
 867 {}
 868
 869 void
 870 StoreHashIndex::maintain()
 871 {
 872     int i;
 873     /* walk each fs */
 874
 875     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 876         /* XXX FixMe: This should be done "in parallell" on the different
 877          * cache_dirs, not one at a time.
 878          */
 879         /* call the maintain function .. */
 880         store(i)->maintain();
 881     }
 882 }
 883
 884 void
 885 StoreHashIndex::updateSize(int64_t, int)
 886 {}
 887
 888 void
 889 StoreHashIndex::sync()
 890 {
 891     for (int i = 0; i < Config.cacheSwap.n_configured; ++i)
 892         store(i)->sync();
 893 }
 894
 895 StoreSearch *
 896 StoreHashIndex::search(String const url, HttpRequest *)
 897 {
 898     if (url.size())
 899         fatal ("Cannot search by url yet\n");
 900
 901     return new StoreSearchHashIndex (this);
 902 }
 903
 904 CBDATA_CLASS_INIT(StoreSearchHashIndex);
 905
 906 StoreSearchHashIndex::StoreSearchHashIndex(RefCount<StoreHashIndex> aSwapDir) : sd(aSwapDir), _done (false), bucket (0)
 907 {}
 908
 909 /* do not link
 910 StoreSearchHashIndex::StoreSearchHashIndex(StoreSearchHashIndex const &);
 911 */
 912
 913 StoreSearchHashIndex::~StoreSearchHashIndex()
 914 {}
 915
 916 void
 917 StoreSearchHashIndex::next(void (callback)(void *cbdata), void *cbdata)
 918 {
 919     next();
 920     callback (cbdata);
 921 }
 922
 923 bool
 924 StoreSearchHashIndex::next()
 925 {
 926     if (entries.size())
 927         entries.pop_back();
 928
 929     while (!isDone() && !entries.size())
 930         copyBucket();
 931
 932     return currentItem() != NULL;
 933 }
 934
 935 bool
 936 StoreSearchHashIndex::error() const
 937 {
 938     return false;
 939 }
 940
 941 bool
 942 StoreSearchHashIndex::isDone() const
 943 {
 944     return bucket >= store_hash_buckets || _done;
 945 }
 946
 947 StoreEntry *
 948 StoreSearchHashIndex::currentItem()
 949 {
 950     if (!entries.size())
 951         return NULL;
 952
 953     return entries.back();
 954 }
 955
 956 void
 957 StoreSearchHashIndex::copyBucket()
 958 {
 959     /* probably need to lock the store entries...
 960      * we copy them all to prevent races on the links. */
 961     debugs(47, 3, "StoreSearchHashIndex::copyBucket #" << bucket);
 962     assert (!entries.size());
 963     hash_link *link_ptr = NULL;
 964     hash_link *link_next = NULL;
 965     link_next = hash_get_bucket(store_table, bucket);
 966
 967     while (NULL != (link_ptr = link_next)) {
 968         link_next = link_ptr->next;
 969         StoreEntry *e = (StoreEntry *) link_ptr;
 970
 971         entries.push_back(e);
 972     }
 973
 974     bucket++;
 975     debugs(47,3, "got entries: " << entries.size());
 976 }