src/store_rebuild.cc

   1
   2 /*
   3  * $Id: store_rebuild.cc,v 1.89 2007/04/28 22:26:38 hno Exp $
   4  *
   5  * DEBUG: section 20    Store Rebuild Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "event.h"
  38 #include "Store.h"
  39 #include "SwapDir.h"
  40 #include "StoreSearch.h"
  41 #include "SquidTime.h"
  42
  43 static struct _store_rebuild_data counts;
  44
  45 static struct timeval rebuild_start;
  46 static void storeCleanup(void *);
  47
  48 typedef struct
  49 {
  50     /* total number of "swap.state" entries that will be read */
  51     int total;
  52     /* number of entries read so far */
  53     int scanned;
  54 } store_rebuild_progress;
  55
  56 static store_rebuild_progress *RebuildProgress = NULL;
  57
  58 static int
  59 storeCleanupDoubleCheck(StoreEntry * e)
  60 {
  61     SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
  62     return (SD->doubleCheck(*e));
  63 }
  64
  65 static void
  66 storeCleanup(void *datanotused)
  67 {
  68     static int store_errors = 0;
  69     static StoreSearchPointer currentSearch;
  70     static int validated = 0;
  71
  72     if (currentSearch == NULL || currentSearch->isDone())
  73         currentSearch = Store::Root().search(NULL, NULL);
  74
  75     size_t statCount = 500;
  76
  77     while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
  78         ++validated;
  79         StoreEntry *e;
  80
  81         e = currentSearch->currentItem();
  82
  83         if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
  84             continue;
  85
  86         /*
  87          * Calling StoreEntry->release() has no effect because we're
  88          * still in 'store_rebuilding' state
  89          */
  90         if (e->swap_filen < 0)
  91             continue;
  92
  93         if (opt_store_doublecheck)
  94             if (storeCleanupDoubleCheck(e))
  95                 store_errors++;
  96
  97         EBIT_SET(e->flags, ENTRY_VALIDATED);
  98
  99         /*
 100          * Only set the file bit if we know its a valid entry
 101          * otherwise, set it in the validation procedure
 102          */
 103         e->store()->updateSize(e->swap_file_sz, 1);
 104
 105         if ((++validated & 0x3FFFF) == 0)
 106             /* TODO format the int with with a stream operator */
 107             debugs(20, 1, "  " << validated << " Entries Validated so far.");
 108     }
 109
 110     if (currentSearch->isDone()) {
 111         debugs(20, 1, "  Completed Validation Procedure");
 112         debugs(20, 1, "  Validated " << validated << " Entries");
 113         debugs(20, 1, "  store_swap_size = " << store_swap_size);
 114         StoreController::store_dirs_rebuilding--;
 115         assert(0 == StoreController::store_dirs_rebuilding);
 116
 117         if (opt_store_doublecheck)
 118             assert(store_errors == 0);
 119
 120         if (store_digest)
 121             storeDigestNoteStoreReady();
 122
 123         currentSearch = NULL;
 124     } else
 125         eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 126 }
 127
 128 /* meta data recreated from disk image in swap directory */
 129 void
 130
 131 storeRebuildComplete(struct _store_rebuild_data *dc)
 132 {
 133     double dt;
 134     counts.objcount += dc->objcount;
 135     counts.expcount += dc->expcount;
 136     counts.scancount += dc->scancount;
 137     counts.clashcount += dc->clashcount;
 138     counts.dupcount += dc->dupcount;
 139     counts.cancelcount += dc->cancelcount;
 140     counts.invalid += dc->invalid;
 141     counts.badflags += dc->badflags;
 142     counts.bad_log_op += dc->bad_log_op;
 143     counts.zero_object_sz += dc->zero_object_sz;
 144     /*
 145      * When store_dirs_rebuilding == 1, it means we are done reading
 146      * or scanning all cache_dirs.  Now report the stats and start
 147      * the validation (storeCleanup()) thread.
 148      */
 149
 150     if (StoreController::store_dirs_rebuilding > 1)
 151         return;
 152
 153     dt = tvSubDsec(rebuild_start, current_time);
 154
 155     debugs(20, 1, "Finished rebuilding storage from disk.");
 156     debugs(20, 1, "  " << std::setw(7) << counts.scancount  << " Entries scanned");
 157     debugs(20, 1, "  " << std::setw(7) << counts.invalid  << " Invalid entries.");
 158     debugs(20, 1, "  " << std::setw(7) << counts.badflags  << " With invalid flags.");
 159     debugs(20, 1, "  " << std::setw(7) << counts.objcount  << " Objects loaded.");
 160     debugs(20, 1, "  " << std::setw(7) << counts.expcount  << " Objects expired.");
 161     debugs(20, 1, "  " << std::setw(7) << counts.cancelcount  << " Objects cancelled.");
 162     debugs(20, 1, "  " << std::setw(7) << counts.dupcount  << " Duplicate URLs purged.");
 163     debugs(20, 1, "  " << std::setw(7) << counts.clashcount  << " Swapfile clashes avoided.");
 164     debugs(20, 1, "  Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
 165            ((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
 166     debugs(20, 1, "Beginning Validation Procedure");
 167
 168     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 169
 170     xfree(RebuildProgress);
 171
 172     RebuildProgress = NULL;
 173 }
 174
 175 /*
 176  * this is ugly.  We don't actually start any rebuild threads here,
 177  * but only initialize counters, etc.  The rebuild threads are
 178  * actually started by the filesystem "fooDirInit" function.
 179  */
 180 void
 181 storeRebuildStart(void)
 182 {
 183     memset(&counts, '\0', sizeof(counts));
 184     rebuild_start = current_time;
 185     /*
 186      * Note: store_dirs_rebuilding is initialized to 1.
 187      *
 188      * When we parse the configuration and construct each swap dir,
 189      * the construction of that raises the rebuild count.
 190      *
 191      * This prevents us from trying to write clean logs until we
 192      * finished rebuilding - including after a reconfiguration that opens an
 193      * existing swapdir.  The corresponding decrement * occurs in
 194      * storeCleanup(), when it is finished.
 195      */
 196     RebuildProgress = (store_rebuild_progress *)xcalloc(Config.cacheSwap.n_configured,
 197                       sizeof(store_rebuild_progress));
 198 }
 199
 200 /*
 201  * A fs-specific rebuild procedure periodically reports its
 202  * progress.
 203  */
 204 void
 205 storeRebuildProgress(int sd_index, int total, int sofar)
 206 {
 207     static time_t last_report = 0;
 208     double n = 0.0;
 209     double d = 0.0;
 210
 211     if (sd_index < 0)
 212         return;
 213
 214     if (sd_index >= Config.cacheSwap.n_configured)
 215         return;
 216
 217     if (NULL == RebuildProgress)
 218         return;
 219
 220     RebuildProgress[sd_index].total = total;
 221
 222     RebuildProgress[sd_index].scanned = sofar;
 223
 224     if (squid_curtime - last_report < 15)
 225         return;
 226
 227     for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
 228         n += (double) RebuildProgress[sd_index].scanned;
 229         d += (double) RebuildProgress[sd_index].total;
 230     }
 231
 232     debugs(20, 1, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
 233     last_report = squid_curtime;
 234 }