src/store_rebuild.cc

   1
   2 /*
   3  * $Id: store_rebuild.cc,v 1.63 1999/05/25 07:02:00 wessels Exp $
   4  *
   5  * DEBUG: section 20    Store Rebuild Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  Duane Wessels and the University of California San Diego.  Please
  16  *  see the COPYRIGHT file for full details.  Squid incorporates
  17  *  software developed and/or copyrighted by other sources.  Please see
  18  *  the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37
  38 static struct _store_rebuild_data counts;
  39 static struct timeval rebuild_start;
  40 static void storeCleanup(void *);
  41
  42 static void
  43 storeCleanup(void *datanotused)
  44 {
  45     static int bucketnum = -1;
  46     static int validnum = 0;
  47     static int store_errors = 0;
  48     StoreEntry *e;
  49     hash_link *link_ptr = NULL;
  50     hash_link *link_next = NULL;
  51     if (++bucketnum >= store_hash_buckets) {
  52         debug(20, 1) ("  Completed Validation Procedure\n");
  53         debug(20, 1) ("  Validated %d Entries\n", validnum);
  54         debug(20, 1) ("  store_swap_size = %dk\n", store_swap_size);
  55         store_dirs_rebuilding--;
  56         assert(0 == store_dirs_rebuilding);
  57         if (opt_store_doublecheck)
  58             assert(store_errors == 0);
  59         if (store_digest)
  60             storeDigestNoteStoreReady();
  61         return;
  62     }
  63     link_next = hash_get_bucket(store_table, bucketnum);
  64     while (NULL != (link_ptr = link_next)) {
  65         link_next = link_ptr->next;
  66         e = (StoreEntry *) link_ptr;
  67         if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
  68             continue;
  69         /*
  70          * Calling storeRelease() has no effect because we're
  71          * still in 'store_rebuilding' state
  72          */
  73         if (e->swap_file_number < 0)
  74             continue;
  75         if (opt_store_doublecheck) {
  76             /* XXX too UFS specific */
  77             struct stat sb;
  78             int dirn = e->swap_file_number >> SWAP_DIR_SHIFT;
  79             if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_UFS)
  80                 (void) 0;
  81             else if (Config.cacheSwap.swapDirs[dirn].type == SWAPDIR_UFS_ASYNC)
  82                 (void) 0;
  83             else
  84                 continue;
  85             if (stat(storeUfsFullPath(e->swap_file_number, NULL), &sb) < 0) {
  86                 store_errors++;
  87                 debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n");
  88                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
  89                 debug(20, 0) ("storeCleanup: PATH %s\n",
  90                     storeUfsFullPath(e->swap_file_number, NULL));
  91                 storeEntryDump(e, 0);
  92                 continue;
  93             }
  94             if (e->swap_file_sz != sb.st_size) {
  95                 store_errors++;
  96                 debug(20, 0) ("storeCleanup: SIZE MISMATCH\n");
  97                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
  98                 debug(20, 0) ("storeCleanup: PATH %s\n",
  99                     storeUfsFullPath(e->swap_file_number, NULL));
 100                 debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n",
 101                     e->swap_file_sz, (int) sb.st_size);
 102                 storeEntryDump(e, 0);
 103                 continue;
 104             }
 105         }
 106         EBIT_SET(e->flags, ENTRY_VALIDATED);
 107         /* Only set the file bit if we know its a valid entry */
 108         /* otherwise, set it in the validation procedure */
 109         storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
 110         if ((++validnum & 0xFFFF) == 0)
 111             debug(20, 1) ("  %7d Entries Validated so far.\n", validnum);
 112     }
 113     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 114 }
 115
 116 /* meta data recreated from disk image in swap directory */
 117 void
 118 storeRebuildComplete(struct _store_rebuild_data *dc)
 119 {
 120     double dt;
 121     counts.objcount += dc->objcount;
 122     counts.expcount += dc->expcount;
 123     counts.scancount += dc->scancount;
 124     counts.clashcount += dc->clashcount;
 125     counts.dupcount += dc->dupcount;
 126     counts.cancelcount += dc->cancelcount;
 127     counts.invalid += dc->invalid;
 128     counts.badflags += dc->badflags;
 129     counts.bad_log_op += dc->bad_log_op;
 130     counts.zero_object_sz += dc->zero_object_sz;
 131     /*
 132      * When store_dirs_rebuilding == 1, it means we are done reading
 133      * or scanning all cache_dirs.  Now report the stats and start
 134      * the validation (storeCleanup()) thread.
 135      */
 136     if (store_dirs_rebuilding > 1)
 137         return;
 138     dt = tvSubDsec(rebuild_start, current_time);
 139     debug(20, 1) ("Finished rebuilding storage from disk.\n");
 140     debug(20, 1) ("  %7d Entries scanned\n", counts.scancount);
 141     debug(20, 1) ("  %7d Invalid entries.\n", counts.invalid);
 142     debug(20, 1) ("  %7d With invalid flags.\n", counts.badflags);
 143     debug(20, 1) ("  %7d Objects loaded.\n", counts.objcount);
 144     debug(20, 1) ("  %7d Objects expired.\n", counts.expcount);
 145     debug(20, 1) ("  %7d Objects cancelled.\n", counts.cancelcount);
 146     debug(20, 1) ("  %7d Duplicate URLs purged.\n", counts.dupcount);
 147     debug(20, 1) ("  %7d Swapfile clashes avoided.\n", counts.clashcount);
 148     debug(20, 1) ("  Took %3.1f seconds (%6.1f objects/sec).\n", dt,
 149         (double) counts.objcount / (dt > 0.0 ? dt : 1.0));
 150     debug(20, 1) ("Beginning Validation Procedure\n");
 151     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 152 }
 153
 154 /*
 155  * this is ugly.  We don't actually start any rebuild threads here,
 156  * but only initialize counters, etc.  The rebuild threads are
 157  * actually started by the filesystem "fooDirInit" function.
 158  */
 159 void
 160 storeRebuildStart(void)
 161 {
 162     memset(&counts, '\0', sizeof(counts));
 163     rebuild_start = current_time;
 164     /*
 165      * Normally store_dirs_rebuilding is incremented once for each
 166      * cache_dir.  We increment it here as well for the disk storage
 167      * system as a whole.  The corresponding decrement occurs in
 168      * storeCleanup(), when it is finished.
 169      */
 170     store_dirs_rebuilding++;
 171 }