src/store_rebuild.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 20    Store Rebuild Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "event.h"
  38 #include "Store.h"
  39 #include "SwapDir.h"
  40 #include "StoreSearch.h"
  41 #include "SquidTime.h"
  42
  43 static struct _store_rebuild_data counts;
  44
  45 static struct timeval rebuild_start;
  46 static void storeCleanup(void *);
  47
  48 typedef struct {
  49     /* total number of "swap.state" entries that will be read */
  50     int total;
  51     /* number of entries read so far */
  52     int scanned;
  53 } store_rebuild_progress;
  54
  55 static store_rebuild_progress *RebuildProgress = NULL;
  56
  57 static int
  58 storeCleanupDoubleCheck(StoreEntry * e)
  59 {
  60     SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
  61     return (SD->doubleCheck(*e));
  62 }
  63
  64 static void
  65 storeCleanup(void *datanotused)
  66 {
  67     static int store_errors = 0;
  68     static StoreSearchPointer currentSearch;
  69     static int validated = 0;
  70
  71     if (currentSearch == NULL || currentSearch->isDone())
  72         currentSearch = Store::Root().search(NULL, NULL);
  73
  74     size_t statCount = 500;
  75
  76     while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
  77         ++validated;
  78         StoreEntry *e;
  79
  80         e = currentSearch->currentItem();
  81
  82         if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
  83             continue;
  84
  85         /*
  86          * Calling StoreEntry->release() has no effect because we're
  87          * still in 'store_rebuilding' state
  88          */
  89         if (e->swap_filen < 0)
  90             continue;
  91
  92         if (opt_store_doublecheck)
  93             if (storeCleanupDoubleCheck(e))
  94                 store_errors++;
  95
  96         EBIT_SET(e->flags, ENTRY_VALIDATED);
  97
  98         /*
  99          * Only set the file bit if we know its a valid entry
 100          * otherwise, set it in the validation procedure
 101          */
 102         e->store()->updateSize(e->swap_file_sz, 1);
 103
 104         if ((++validated & 0x3FFFF) == 0)
 105             /* TODO format the int with with a stream operator */
 106             debugs(20, 1, "  " << validated << " Entries Validated so far.");
 107     }
 108
 109     if (currentSearch->isDone()) {
 110         debugs(20, 1, "  Completed Validation Procedure");
 111         debugs(20, 1, "  Validated " << validated << " Entries");
 112         debugs(20, 1, "  store_swap_size = " << store_swap_size);
 113         StoreController::store_dirs_rebuilding--;
 114         assert(0 == StoreController::store_dirs_rebuilding);
 115
 116         if (opt_store_doublecheck)
 117             assert(store_errors == 0);
 118
 119         if (store_digest)
 120             storeDigestNoteStoreReady();
 121
 122         currentSearch = NULL;
 123     } else
 124         eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 125 }
 126
 127 /* meta data recreated from disk image in swap directory */
 128 void
 129
 130 storeRebuildComplete(struct _store_rebuild_data *dc)
 131 {
 132     double dt;
 133     counts.objcount += dc->objcount;
 134     counts.expcount += dc->expcount;
 135     counts.scancount += dc->scancount;
 136     counts.clashcount += dc->clashcount;
 137     counts.dupcount += dc->dupcount;
 138     counts.cancelcount += dc->cancelcount;
 139     counts.invalid += dc->invalid;
 140     counts.badflags += dc->badflags;
 141     counts.bad_log_op += dc->bad_log_op;
 142     counts.zero_object_sz += dc->zero_object_sz;
 143     /*
 144      * When store_dirs_rebuilding == 1, it means we are done reading
 145      * or scanning all cache_dirs.  Now report the stats and start
 146      * the validation (storeCleanup()) thread.
 147      */
 148
 149     if (StoreController::store_dirs_rebuilding > 1)
 150         return;
 151
 152     dt = tvSubDsec(rebuild_start, current_time);
 153
 154     debugs(20, 1, "Finished rebuilding storage from disk.");
 155     debugs(20, 1, "  " << std::setw(7) << counts.scancount  << " Entries scanned");
 156     debugs(20, 1, "  " << std::setw(7) << counts.invalid  << " Invalid entries.");
 157     debugs(20, 1, "  " << std::setw(7) << counts.badflags  << " With invalid flags.");
 158     debugs(20, 1, "  " << std::setw(7) << counts.objcount  << " Objects loaded.");
 159     debugs(20, 1, "  " << std::setw(7) << counts.expcount  << " Objects expired.");
 160     debugs(20, 1, "  " << std::setw(7) << counts.cancelcount  << " Objects cancelled.");
 161     debugs(20, 1, "  " << std::setw(7) << counts.dupcount  << " Duplicate URLs purged.");
 162     debugs(20, 1, "  " << std::setw(7) << counts.clashcount  << " Swapfile clashes avoided.");
 163     debugs(20, 1, "  Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
 164            ((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
 165     debugs(20, 1, "Beginning Validation Procedure");
 166
 167     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 168
 169     xfree(RebuildProgress);
 170
 171     RebuildProgress = NULL;
 172 }
 173
 174 /*
 175  * this is ugly.  We don't actually start any rebuild threads here,
 176  * but only initialize counters, etc.  The rebuild threads are
 177  * actually started by the filesystem "fooDirInit" function.
 178  */
 179 void
 180 storeRebuildStart(void)
 181 {
 182     memset(&counts, '\0', sizeof(counts));
 183     rebuild_start = current_time;
 184     /*
 185      * Note: store_dirs_rebuilding is initialized to 1.
 186      *
 187      * When we parse the configuration and construct each swap dir,
 188      * the construction of that raises the rebuild count.
 189      *
 190      * This prevents us from trying to write clean logs until we
 191      * finished rebuilding - including after a reconfiguration that opens an
 192      * existing swapdir.  The corresponding decrement * occurs in
 193      * storeCleanup(), when it is finished.
 194      */
 195     RebuildProgress = (store_rebuild_progress *)xcalloc(Config.cacheSwap.n_configured,
 196                       sizeof(store_rebuild_progress));
 197 }
 198
 199 /*
 200  * A fs-specific rebuild procedure periodically reports its
 201  * progress.
 202  */
 203 void
 204 storeRebuildProgress(int sd_index, int total, int sofar)
 205 {
 206     static time_t last_report = 0;
 207     double n = 0.0;
 208     double d = 0.0;
 209
 210     if (sd_index < 0)
 211         return;
 212
 213     if (sd_index >= Config.cacheSwap.n_configured)
 214         return;
 215
 216     if (NULL == RebuildProgress)
 217         return;
 218
 219     RebuildProgress[sd_index].total = total;
 220
 221     RebuildProgress[sd_index].scanned = sofar;
 222
 223     if (squid_curtime - last_report < 15)
 224         return;
 225
 226     for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
 227         n += (double) RebuildProgress[sd_index].scanned;
 228         d += (double) RebuildProgress[sd_index].total;
 229     }
 230
 231     debugs(20, 1, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
 232     last_report = squid_curtime;
 233 }