src/store_rebuild.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 20    Store Rebuild Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "event.h"
  38 #include "Store.h"
  39 #include "SwapDir.h"
  40 #include "StoreSearch.h"
  41 #include "SquidTime.h"
  42
  43 static struct _store_rebuild_data counts;
  44
  45 static struct timeval rebuild_start;
  46 static void storeCleanup(void *);
  47
  48 typedef struct {
  49     /* total number of "swap.state" entries that will be read */
  50     int total;
  51     /* number of entries read so far */
  52     int scanned;
  53 } store_rebuild_progress;
  54
  55 static store_rebuild_progress *RebuildProgress = NULL;
  56
  57 static int
  58 storeCleanupDoubleCheck(StoreEntry * e)
  59 {
  60     SwapDir *SD = dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn));
  61     return (SD->doubleCheck(*e));
  62 }
  63
  64 static void
  65 storeCleanup(void *datanotused)
  66 {
  67     static int store_errors = 0;
  68     static StoreSearchPointer currentSearch;
  69     static int validated = 0;
  70
  71     if (currentSearch == NULL || currentSearch->isDone())
  72         currentSearch = Store::Root().search(NULL, NULL);
  73
  74     size_t statCount = 500;
  75
  76     while (statCount-- && !currentSearch->isDone() && currentSearch->next()) {
  77         StoreEntry *e;
  78
  79         e = currentSearch->currentItem();
  80
  81         if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
  82             continue;
  83
  84         /*
  85          * Calling StoreEntry->release() has no effect because we're
  86          * still in 'store_rebuilding' state
  87          */
  88         if (e->swap_filen < 0)
  89             continue;
  90
  91         if (opt_store_doublecheck)
  92             if (storeCleanupDoubleCheck(e))
  93                 store_errors++;
  94
  95         EBIT_SET(e->flags, ENTRY_VALIDATED);
  96
  97         /*
  98          * Only set the file bit if we know its a valid entry
  99          * otherwise, set it in the validation procedure
 100          */
 101
 102
 103         if (e->swap_status == SWAPOUT_DONE)
 104             e->store()->updateSize(e->swap_file_sz, 1);
 105
 106         if ((++validated & 0x3FFFF) == 0)
 107             /* TODO format the int with with a stream operator */
 108             debugs(20, 1, "  " << validated << " Entries Validated so far.");
 109     }
 110
 111     if (currentSearch->isDone()) {
 112         debugs(20, 1, "  Completed Validation Procedure");
 113         debugs(20, 1, "  Validated " << validated << " Entries");
 114         debugs(20, 1, "  store_swap_size = " << store_swap_size);
 115         StoreController::store_dirs_rebuilding--;
 116         assert(0 == StoreController::store_dirs_rebuilding);
 117
 118         if (opt_store_doublecheck)
 119             assert(store_errors == 0);
 120
 121         if (store_digest)
 122             storeDigestNoteStoreReady();
 123
 124         currentSearch = NULL;
 125     } else
 126         eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 127 }
 128
 129 /* meta data recreated from disk image in swap directory */
 130 void
 131
 132 storeRebuildComplete(struct _store_rebuild_data *dc)
 133 {
 134     double dt;
 135     counts.objcount += dc->objcount;
 136     counts.expcount += dc->expcount;
 137     counts.scancount += dc->scancount;
 138     counts.clashcount += dc->clashcount;
 139     counts.dupcount += dc->dupcount;
 140     counts.cancelcount += dc->cancelcount;
 141     counts.invalid += dc->invalid;
 142     counts.badflags += dc->badflags;
 143     counts.bad_log_op += dc->bad_log_op;
 144     counts.zero_object_sz += dc->zero_object_sz;
 145     /*
 146      * When store_dirs_rebuilding == 1, it means we are done reading
 147      * or scanning all cache_dirs.  Now report the stats and start
 148      * the validation (storeCleanup()) thread.
 149      */
 150
 151     if (StoreController::store_dirs_rebuilding > 1)
 152         return;
 153
 154     dt = tvSubDsec(rebuild_start, current_time);
 155
 156     debugs(20, 1, "Finished rebuilding storage from disk.");
 157     debugs(20, 1, "  " << std::setw(7) << counts.scancount  << " Entries scanned");
 158     debugs(20, 1, "  " << std::setw(7) << counts.invalid  << " Invalid entries.");
 159     debugs(20, 1, "  " << std::setw(7) << counts.badflags  << " With invalid flags.");
 160     debugs(20, 1, "  " << std::setw(7) << counts.objcount  << " Objects loaded.");
 161     debugs(20, 1, "  " << std::setw(7) << counts.expcount  << " Objects expired.");
 162     debugs(20, 1, "  " << std::setw(7) << counts.cancelcount  << " Objects cancelled.");
 163     debugs(20, 1, "  " << std::setw(7) << counts.dupcount  << " Duplicate URLs purged.");
 164     debugs(20, 1, "  " << std::setw(7) << counts.clashcount  << " Swapfile clashes avoided.");
 165     debugs(20, 1, "  Took "<< std::setw(3)<< std::setprecision(2) << dt << " seconds ("<< std::setw(6) <<
 166            ((double) counts.objcount / (dt > 0.0 ? dt : 1.0)) << " objects/sec).");
 167     debugs(20, 1, "Beginning Validation Procedure");
 168
 169     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 170
 171     xfree(RebuildProgress);
 172
 173     RebuildProgress = NULL;
 174 }
 175
 176 /*
 177  * this is ugly.  We don't actually start any rebuild threads here,
 178  * but only initialize counters, etc.  The rebuild threads are
 179  * actually started by the filesystem "fooDirInit" function.
 180  */
 181 void
 182 storeRebuildStart(void)
 183 {
 184     memset(&counts, '\0', sizeof(counts));
 185     rebuild_start = current_time;
 186     /*
 187      * Note: store_dirs_rebuilding is initialized to 1.
 188      *
 189      * When we parse the configuration and construct each swap dir,
 190      * the construction of that raises the rebuild count.
 191      *
 192      * This prevents us from trying to write clean logs until we
 193      * finished rebuilding - including after a reconfiguration that opens an
 194      * existing swapdir.  The corresponding decrement * occurs in
 195      * storeCleanup(), when it is finished.
 196      */
 197     RebuildProgress = (store_rebuild_progress *)xcalloc(Config.cacheSwap.n_configured,
 198                       sizeof(store_rebuild_progress));
 199 }
 200
 201 /*
 202  * A fs-specific rebuild procedure periodically reports its
 203  * progress.
 204  */
 205 void
 206 storeRebuildProgress(int sd_index, int total, int sofar)
 207 {
 208     static time_t last_report = 0;
 209     double n = 0.0;
 210     double d = 0.0;
 211
 212     if (sd_index < 0)
 213         return;
 214
 215     if (sd_index >= Config.cacheSwap.n_configured)
 216         return;
 217
 218     if (NULL == RebuildProgress)
 219         return;
 220
 221     RebuildProgress[sd_index].total = total;
 222
 223     RebuildProgress[sd_index].scanned = sofar;
 224
 225     if (squid_curtime - last_report < 15)
 226         return;
 227
 228     for (sd_index = 0; sd_index < Config.cacheSwap.n_configured; sd_index++) {
 229         n += (double) RebuildProgress[sd_index].scanned;
 230         d += (double) RebuildProgress[sd_index].total;
 231     }
 232
 233     debugs(20, 1, "Store rebuilding is "<< std::setw(4)<< std::setprecision(2) << 100.0 * n / d << "% complete");
 234     last_report = squid_curtime;
 235 }
 236
 237 #include "fde.h"
 238 #include "StoreMetaUnpacker.h"
 239 #include "StoreMeta.h"
 240 #include "Generic.h"
 241
 242 struct InitStoreEntry : public unary_function<StoreMeta, void> {
 243     InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
 244
 245     void operator()(StoreMeta const &x) {
 246         switch (x.getType()) {
 247
 248         case STORE_META_KEY:
 249             assert(x.length == SQUID_MD5_DIGEST_LENGTH);
 250             memcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH);
 251             break;
 252
 253         case STORE_META_STD:
 254             struct old_metahdr {
 255                 time_t timestamp;
 256                 time_t lastref;
 257                 time_t expires;
 258                 time_t lastmod;
 259                 size_t swap_file_sz;
 260                 u_short refcount;
 261                 u_short flags;
 262             } *tmp;
 263             tmp = (struct old_metahdr *)x.value;
 264             assert(x.length == STORE_HDR_METASIZE_OLD);
 265             what->timestamp = tmp->timestamp;
 266             what->lastref = tmp->lastref;
 267             what->expires = tmp->expires;
 268             what->lastmod = tmp->lastmod;
 269             what->swap_file_sz = tmp->swap_file_sz;
 270             what->refcount = tmp->refcount;
 271             what->flags = tmp->flags;
 272             break;
 273
 274         case STORE_META_STD_LFS:
 275             assert(x.length == STORE_HDR_METASIZE);
 276             memcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
 277             break;
 278
 279         default:
 280             break;
 281         }
 282     }
 283
 284     StoreEntry *what;
 285     cache_key *index;
 286 };
 287
 288 bool
 289 storeRebuildLoadEntry(int fd, StoreEntry &tmpe, cache_key *key,
 290                       struct _store_rebuild_data &counts, uint64_t expectedSize)
 291 {
 292     if (fd < 0)
 293         return false;
 294
 295     char hdr_buf[SM_PAGE_SIZE];
 296
 297     ++counts.scancount;
 298     statCounter.syscalls.disk.reads++;
 299     int len;
 300     if ((len = FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE)) < 0) {
 301         debugs(47, 1, HERE << "failed to read swap entry meta data: " << xstrerror());
 302         return false;
 303     }
 304
 305     int swap_hdr_len = 0;
 306     StoreMetaUnpacker aBuilder(hdr_buf, len, &swap_hdr_len);
 307     if (aBuilder.isBufferZero()) {
 308         debugs(47,5, HERE << "skipping empty record.");
 309         return false;
 310     }
 311
 312     if (!aBuilder.isBufferSane()) {
 313         debugs(47,1, "Warning: Ignoring malformed cache entry.");
 314         return false;
 315     }
 316
 317     StoreMeta *tlv_list = aBuilder.createStoreMeta();
 318     if (!tlv_list) {
 319         debugs(47, 1, HERE << "failed to get swap entry meta data list");
 320         return false;
 321     }
 322
 323     debugs(47,7, HERE << "successful swap meta unpacking");
 324     memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
 325
 326     InitStoreEntry visitor(&tmpe, key);
 327     for_each(*tlv_list, visitor);
 328     storeSwapTLVFree(tlv_list);
 329     tlv_list = NULL;
 330
 331     if (storeKeyNull(key)) {
 332         debugs(47,1, HERE << "NULL swap entry key");
 333         return false;
 334     }
 335
 336     tmpe.key = key;
 337     /* check sizes */
 338
 339     if (expectedSize > 0) {
 340         if (tmpe.swap_file_sz == 0) {
 341             tmpe.swap_file_sz = expectedSize;
 342         } else if (tmpe.swap_file_sz == (uint64_t)(expectedSize - swap_hdr_len)) {
 343             tmpe.swap_file_sz = expectedSize;
 344         } else if (tmpe.swap_file_sz != expectedSize) {
 345             debugs(47, 1, HERE << "swap entry SIZE MISMATCH " <<
 346                    tmpe.swap_file_sz << "!=" << expectedSize);
 347             return false;
 348         }
 349     }
 350
 351     if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
 352         counts.badflags++;
 353         return false;
 354     }
 355
 356     return true;
 357 }
 358
 359 bool
 360 storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key,
 361                       struct _store_rebuild_data &counts)
 362 {
 363     /* this needs to become
 364      * 1) unpack url
 365      * 2) make synthetic request with headers ?? or otherwise search
 366      * for a matching object in the store
 367      * TODO FIXME change to new async api
 368      * TODO FIXME I think there is a race condition here with the
 369      * async api :
 370      * store A reads in object foo, searchs for it, and finds nothing.
 371      * store B reads in object foo, searchs for it, finds nothing.
 372      * store A gets called back with nothing, so registers the object
 373      * store B gets called back with nothing, so registers the object,
 374      * which will conflict when the in core index gets around to scanning
 375      * store B.
 376      *
 377      * this suggests that rather than searching for duplicates, the
 378      * index rebuild should just assume its the most recent accurate
 379      * store entry and whoever indexes the stores handles duplicates.
 380      */
 381     if (StoreEntry *e = Store::Root().get(key)) {
 382
 383         if (e->lastref >= tmpe.lastref) {
 384             /* key already exists, old entry is newer */
 385             /* keep old, ignore new */
 386             counts.dupcount++;
 387             return false;
 388         } else {
 389             /* URL already exists, this swapfile not being used */
 390             /* junk old, load new */
 391             e->release();       /* release old entry */
 392             counts.dupcount++;
 393         }
 394     }
 395
 396     return true;
 397 }