src/store_rebuild.cc

   1 /*
   2  * $Id: store_rebuild.cc,v 1.46 1998/08/24 22:06:48 wessels Exp $
   3  *
   4  * DEBUG: section 20    Store Rebuild Routines
   5  * AUTHOR: Duane Wessels
   6  *
   7  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from the
  11  *  Internet community.  Development is led by Duane Wessels of the
  12  *  National Laboratory for Applied Network Research and funded by the
  13  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  14  *  Duane Wessels and the University of California San Diego.  Please
  15  *  see the COPYRIGHT file for full details.  Squid incorporates
  16  *  software developed and/or copyrighted by other sources.  Please see
  17  *  the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 #include "squid.h"
  36
  37 #define STORE_META_BUFSZ 4096
  38
  39
  40 typedef struct _rebuild_dir rebuild_dir;
  41 typedef int RBHD(rebuild_dir * d);
  42
  43 struct _rebuild_dir {
  44     int dirn;
  45     int n_read;
  46     FILE *log;
  47     int speed;
  48     int clean;
  49     int curlvl1;
  50     int curlvl2;
  51     int flag;
  52     int done;
  53     int in_dir;
  54     int fn;
  55     struct dirent *entry;
  56     DIR *td;
  57     RBHD *rebuild_func;
  58     rebuild_dir *next;
  59     char fullpath[SQUID_MAXPATHLEN];
  60     char fullfilename[SQUID_MAXPATHLEN];
  61 };
  62
  63 struct storeRebuildState {
  64     rebuild_dir *rebuild_dir;
  65     int objcount;               /* # objects successfully reloaded */
  66     int expcount;               /* # objects expired */
  67     int linecount;              /* # lines parsed from cache logfile */
  68     int statcount;              /* # entries from directory walking */
  69     int clashcount;             /* # swapfile clashes avoided */
  70     int dupcount;               /* # duplicates purged */
  71     int cancelcount;            /* # SWAP_LOG_DEL objects purged */
  72     int invalid;                /* # bad lines */
  73     int badflags;               /* # bad e->flags */
  74     int need_to_validate;
  75     int bad_log_op;
  76     int zero_object_sz;
  77     time_t start;
  78     time_t stop;
  79 } RebuildState;
  80
  81 typedef struct valid_ctrl_t {
  82     struct stat *sb;
  83     StoreEntry *e;
  84     STVLDCB *callback;
  85     void *callback_data;
  86 } valid_ctrl_t;
  87
  88 static RBHD storeRebuildFromDirectory;
  89 static RBHD storeRebuildFromSwapLog;
  90 static void storeRebuildComplete(void);
  91 static EVH storeRebuildADirectory;
  92 static int storeGetNextFile(rebuild_dir *, int *sfileno, int *size);
  93 static StoreEntry *storeAddDiskRestore(const cache_key * key,
  94     int file_number,
  95     size_t swap_file_sz,
  96     time_t expires,
  97     time_t timestamp,
  98     time_t lastref,
  99     time_t lastmod,
 100     u_num32 refcount,
 101     u_num32 flags,
 102     int clean);
 103 static AIOCB storeValidateComplete;
 104
 105 static int
 106 storeRebuildFromDirectory(rebuild_dir * d)
 107 {
 108     LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE);
 109     StoreEntry *e = NULL;
 110     StoreEntry tmpe;
 111     cache_key key[MD5_DIGEST_CHARS];
 112     int sfileno = 0;
 113     int count;
 114     int size;
 115     struct stat sb;
 116     int swap_hdr_len;
 117     int fd = -1;
 118     tlv *tlv_list;
 119     tlv *t;
 120     assert(d != NULL);
 121     debug(20, 3) ("storeRebuildFromDirectory: DIR #%d\n", d->dirn);
 122     for (count = 0; count < d->speed; count++) {
 123         assert(fd == -1);
 124         fd = storeGetNextFile(d, &sfileno, &size);
 125         if (fd == -2) {
 126             debug(20, 1) ("storeRebuildFromDirectory: DIR #%d done!\n", d->dirn);
 127             storeDirCloseTmpSwapLog(d->dirn);
 128             store_rebuilding = 0;
 129             return -1;
 130         } else if (fd < 0) {
 131             continue;
 132         }
 133         assert(fd > -1);
 134         /* lets get file stats here */
 135         if (fstat(fd, &sb) < 0) {
 136             debug(20, 1) ("storeRebuildFromDirectory: fstat(FD %d): %s\n",
 137                 fd, xstrerror());
 138             file_close(fd);
 139             fd = -1;
 140             continue;
 141         }
 142         if ((++RebuildState.statcount & 0xFFFF) == 0)
 143             debug(20, 1) ("  %7d files opened so far.\n",
 144                 RebuildState.statcount);
 145         debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
 146         if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) {
 147             debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n",
 148                 fd, xstrerror());
 149             file_close(fd);
 150             fd = -1;
 151             continue;
 152         }
 153         file_close(fd);
 154         fd = -1;
 155         swap_hdr_len = 0;
 156         tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
 157         if (tlv_list == NULL) {
 158             debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n");
 159             storeUnlinkFileno(sfileno);
 160             continue;
 161         }
 162         debug(20, 3) ("storeRebuildFromDirectory: successful swap meta unpacking\n");
 163         memset(key, '\0', MD5_DIGEST_CHARS);
 164         memset(&tmpe, '\0', sizeof(StoreEntry));
 165         for (t = tlv_list; t; t = t->next) {
 166             switch (t->type) {
 167             case STORE_META_KEY:
 168                 assert(t->length == MD5_DIGEST_CHARS);
 169                 xmemcpy(key, t->value, MD5_DIGEST_CHARS);
 170                 break;
 171             case STORE_META_STD:
 172                 assert(t->length == STORE_HDR_METASIZE);
 173                 xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
 174                 break;
 175             default:
 176                 break;
 177             }
 178         }
 179         storeSwapTLVFree(tlv_list);
 180         tlv_list = NULL;
 181         if (storeKeyNull(key)) {
 182             debug(20, 1) ("storeRebuildFromDirectory: NULL key\n");
 183             storeUnlinkFileno(sfileno);
 184             continue;
 185         }
 186         tmpe.key = key;
 187         /* check sizes */
 188         if (tmpe.swap_file_sz == 0) {
 189             tmpe.swap_file_sz = sb.st_size;
 190         } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
 191             tmpe.swap_file_sz = sb.st_size;
 192         } else if (tmpe.swap_file_sz != sb.st_size) {
 193             debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
 194                 tmpe.swap_file_sz, (int) sb.st_size);
 195             storeUnlinkFileno(sfileno);
 196             continue;
 197         }
 198         if (EBIT_TEST(tmpe.flag, KEY_PRIVATE)) {
 199             storeUnlinkFileno(sfileno);
 200             RebuildState.badflags++;
 201             continue;
 202         }
 203         e = storeGet(key);
 204         if (e && e->lastref >= tmpe.lastref) {
 205             /* key already exists, current entry is newer */
 206             /* keep old, ignore new */
 207             RebuildState.dupcount++;
 208             continue;
 209         } else if (NULL != e) {
 210             /* URL already exists, this swapfile not being used */
 211             /* junk old, load new */
 212             storeRelease(e);    /* release old entry */
 213             RebuildState.dupcount++;
 214         }
 215         RebuildState.objcount++;
 216         storeEntryDump(&tmpe, 5);
 217         e = storeAddDiskRestore(key,
 218             sfileno,
 219             tmpe.swap_file_sz,
 220             tmpe.expires,
 221             tmpe.timestamp,
 222             tmpe.lastref,
 223             tmpe.lastmod,
 224             tmpe.refcount,      /* refcount */
 225             tmpe.flag,          /* flags */
 226             d->clean);
 227     }
 228     return count;
 229 }
 230
 231 static int
 232 storeRebuildFromSwapLog(rebuild_dir * d)
 233 {
 234     StoreEntry *e = NULL;
 235     storeSwapLogData s;
 236     size_t ss = sizeof(storeSwapLogData);
 237     int count;
 238     int used;                   /* is swapfile already in use? */
 239     int disk_entry_newer;       /* is the log entry newer than current entry? */
 240     double x;
 241     assert(d != NULL);
 242     /* load a number of objects per invocation */
 243     for (count = 0; count < d->speed; count++) {
 244         if (fread(&s, ss, 1, d->log) != 1) {
 245             debug(20, 1) ("Done reading Cache Dir #%d swaplog (%d entries)\n",
 246                 d->dirn, d->n_read);
 247             fclose(d->log);
 248             d->log = NULL;
 249             storeDirCloseTmpSwapLog(d->dirn);
 250             return -1;
 251         }
 252         d->n_read++;
 253         if (s.op <= SWAP_LOG_NOP)
 254             continue;
 255         if (s.op >= SWAP_LOG_MAX)
 256             continue;
 257         s.swap_file_number = storeDirProperFileno(d->dirn, s.swap_file_number);
 258         debug(20, 3) ("storeRebuildFromSwapLog: %s %s %08X\n",
 259             swap_log_op_str[(int) s.op],
 260             storeKeyText(s.key),
 261             s.swap_file_number);
 262         if (s.op == SWAP_LOG_ADD) {
 263             (void) 0;
 264         } else if (s.op == SWAP_LOG_DEL) {
 265             if ((e = storeGet(s.key)) != NULL) {
 266                 /*
 267                  * Make sure we don't unlink the file, it might be
 268                  * in use by a subsequent entry.  Also note that
 269                  * we don't have to subtract from store_swap_size
 270                  * because adding to store_swap_size happens in
 271                  * the cleanup procedure.
 272                  */
 273                 storeExpireNow(e);
 274                 storeSetPrivateKey(e);
 275                 EBIT_SET(e->flag, RELEASE_REQUEST);
 276                 if (e->swap_file_number > -1) {
 277                     storeDirMapBitReset(e->swap_file_number);
 278                     e->swap_file_number = -1;
 279                 }
 280                 RebuildState.objcount--;
 281                 RebuildState.cancelcount++;
 282             }
 283             continue;
 284         } else {
 285             x = log(++RebuildState.bad_log_op) / log(10.0);
 286             if (0.0 == x - (double) (int) x)
 287                 debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
 288                     RebuildState.bad_log_op);
 289             RebuildState.invalid++;
 290             continue;
 291         }
 292         if ((++RebuildState.linecount & 0xFFFF) == 0)
 293             debug(20, 1) ("  %7d Entries read so far.\n",
 294                 RebuildState.linecount);
 295         if (!storeDirValidFileno(s.swap_file_number)) {
 296             RebuildState.invalid++;
 297             continue;
 298         }
 299         if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
 300             RebuildState.badflags++;
 301             continue;
 302         }
 303         e = storeGet(s.key);
 304         used = storeDirMapBitTest(s.swap_file_number);
 305         /* If this URL already exists in the cache, does the swap log
 306          * appear to have a newer entry?  Compare 'lastref' from the
 307          * swap log to e->lastref. */
 308         disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
 309         if (used && !disk_entry_newer) {
 310             /* log entry is old, ignore it */
 311             RebuildState.clashcount++;
 312             continue;
 313         } else if (used && e && e->swap_file_number == s.swap_file_number) {
 314             /* swapfile taken, same URL, newer, update meta */
 315             /* XXX what if e->store_status != STORE_OK? */
 316             e->lastref = s.timestamp;
 317             e->timestamp = s.timestamp;
 318             e->expires = s.expires;
 319             e->lastmod = s.lastmod;
 320             e->flag |= s.flags;
 321             e->refcount += s.refcount;
 322             continue;
 323         } else if (used) {
 324             /* swapfile in use, not by this URL, log entry is newer */
 325             /* This is sorta bad: the log entry should NOT be newer at this
 326              * point.  If the log is dirty, the filesize check should have
 327              * caught this.  If the log is clean, there should never be a
 328              * newer entry. */
 329             debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n",
 330                 s.swap_file_number);
 331             /* I'm tempted to remove the swapfile here just to be safe,
 332              * but there is a bad race condition in the NOVM version if
 333              * the swapfile has recently been opened for writing, but
 334              * not yet opened for reading.  Because we can't map
 335              * swapfiles back to StoreEntrys, we don't know the state
 336              * of the entry using that file.  */
 337             /* We'll assume the existing entry is valid, probably because
 338              * were in a slow rebuild and the the swap file number got taken
 339              * and the validation procedure hasn't run. */
 340             assert(RebuildState.need_to_validate);
 341             RebuildState.clashcount++;
 342             continue;
 343         } else if (e && !disk_entry_newer) {
 344             /* key already exists, current entry is newer */
 345             /* keep old, ignore new */
 346             RebuildState.dupcount++;
 347             continue;
 348         } else if (e) {
 349             /* key already exists, this swapfile not being used */
 350             /* junk old, load new */
 351             storeExpireNow(e);
 352             storeSetPrivateKey(e);
 353             EBIT_SET(e->flag, RELEASE_REQUEST);
 354             if (e->swap_file_number > -1) {
 355                 storeDirMapBitReset(e->swap_file_number);
 356                 e->swap_file_number = -1;
 357             }
 358             RebuildState.dupcount++;
 359         } else {
 360             /* URL doesnt exist, swapfile not in use */
 361             /* load new */
 362             (void) 0;
 363         }
 364         /* update store_swap_size */
 365         RebuildState.objcount++;
 366         e = storeAddDiskRestore(s.key,
 367             s.swap_file_number,
 368             s.swap_file_sz,
 369             s.expires,
 370             s.timestamp,
 371             s.lastref,
 372             s.lastmod,
 373             s.refcount,
 374             s.flags,
 375             d->clean);
 376         storeDirSwapLog(e, SWAP_LOG_ADD);
 377     }
 378     return count;
 379 }
 380
 381 static void
 382 storeRebuildADirectory(void *unused)
 383 {
 384     int count;
 385     rebuild_dir *d;
 386     rebuild_dir **D;
 387     if ((d = RebuildState.rebuild_dir) == NULL) {
 388         storeRebuildComplete();
 389         return;
 390     }
 391     count = d->rebuild_func(d);
 392     RebuildState.rebuild_dir = d->next;
 393     if (count < 0) {
 394         xfree(d);
 395     } else {
 396         for (D = &RebuildState.rebuild_dir; *D; D = &(*D)->next);
 397         *D = d;
 398         d->next = NULL;
 399     }
 400     if (opt_foreground_rebuild)
 401         storeRebuildADirectory(NULL);
 402     else
 403         eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
 404 }
 405
 406 #if TEMP_UNUSED_CODE
 407 static void
 408 storeConvertFile(const cache_key * key,
 409     int file_number,
 410     size_t swap_file_sz,
 411     time_t expires,
 412     time_t timestamp,
 413     time_t lastref,
 414     time_t lastmod,
 415     u_short refcount,
 416     u_short flags,
 417     int clean)
 418 {
 419     int fd_r, fd_w;
 420     int hdr_len, x, y;
 421     LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN);
 422     LOCAL_ARRAY(char, copybuf, DISK_PAGE_SIZE);
 423     char *buf;
 424     tlv *tlv_list;
 425     StoreEntry e;
 426     e.key = key;
 427     e.swap_file_sz = swap_file_sz;
 428     e.expires = expires;
 429     e.lastref = lastref;
 430     e.refcount = refcount;
 431     e.flag = flags;
 432     storeSwapFullPath(file_number, swapfilename);
 433     fd_r = file_open(swapfilename, O_RDONLY, NULL, NULL, NULL);
 434     if (fd_r < 0)
 435         return;
 436     safeunlink(swapfilename, 1);
 437     fd_w = file_open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC, NULL, NULL, NULL);
 438     tlv_list = storeSwapMetaBuild(&e);
 439     buf = storeSwapMetaPack(tlv_list, &hdr_len);
 440     x = write(fd_w, buf, hdr_len);
 441     while (x > 0) {
 442         y = read(fd_r, copybuf, DISK_PAGE_SIZE);
 443         x = write(fd_w, copybuf, y);
 444     }
 445     file_close(fd_r);
 446     file_close(fd_w);
 447     xfree(buf);
 448     storeSwapTLVFree(tlv_list);
 449 }
 450 #endif
 451
 452 static int
 453 storeGetNextFile(rebuild_dir * d, int *sfileno, int *size)
 454 {
 455     int fd = -1;
 456     int used = 0;
 457     debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n",
 458         d->flag,
 459         d->dirn,
 460         d->curlvl1,
 461         d->curlvl2);
 462     if (d->done)
 463         return -2;
 464     while (fd < 0 && d->done == 0) {
 465         fd = -1;
 466         if (0 == d->flag) {     /* initialize, open first file */
 467             d->done = 0;
 468             d->curlvl1 = 0;
 469             d->curlvl2 = 0;
 470             d->in_dir = 0;
 471             d->flag = 1;
 472             assert(Config.cacheSwap.n_configured > 0);
 473         }
 474         if (0 == d->in_dir) {   /* we need to read in a new directory */
 475             snprintf(d->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
 476                 Config.cacheSwap.swapDirs[d->dirn].path,
 477                 d->curlvl1, d->curlvl2);
 478             if (d->flag && d->td != NULL)
 479                 closedir(d->td);
 480             d->td = opendir(d->fullpath);
 481             if (d->td == NULL) {
 482                 debug(50, 1) ("storeGetNextFile: opendir: %s: %s\n",
 483                     d->fullpath, xstrerror());
 484                 break;
 485             }
 486             d->entry = readdir(d->td);  /* skip . and .. */
 487             d->entry = readdir(d->td);
 488             if (d->entry == NULL && errno == ENOENT)
 489                 debug(20, 1) ("storeGetNextFile: directory does not exist!.\n");
 490             debug(20, 3) ("storeGetNextFile: Directory %s\n", d->fullpath);
 491         }
 492         if (d->td != NULL && (d->entry = readdir(d->td)) != NULL) {
 493             d->in_dir++;
 494             if (sscanf(d->entry->d_name, "%x", &d->fn) != 1) {
 495                 debug(20, 3) ("storeGetNextFile: invalid %s\n",
 496                     d->entry->d_name);
 497                 continue;
 498             }
 499             if (!storeFilenoBelongsHere(d->fn, d->dirn, d->curlvl1, d->curlvl2)) {
 500                 debug(20, 3) ("storeGetNextFile: %08X does not belong in %d/%d/%d\n",
 501                     d->fn, d->dirn, d->curlvl1, d->curlvl2);
 502                 continue;
 503             }
 504             d->fn = storeDirProperFileno(d->dirn, d->fn);
 505             used = storeDirMapBitTest(d->fn);
 506             if (used) {
 507                 debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n");
 508                 continue;
 509             }
 510             snprintf(d->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
 511                 d->fullpath, d->entry->d_name);
 512             debug(20, 3) ("storeGetNextFile: Opening %s\n", d->fullfilename);
 513             fd = file_open(d->fullfilename, O_RDONLY, NULL, NULL, NULL);
 514             if (fd < 0)
 515                 debug(50, 1) ("storeGetNextFile: %s: %s\n", d->fullfilename, xstrerror());
 516             continue;
 517         }
 518         d->in_dir = 0;
 519         if (++d->curlvl2 < Config.cacheSwap.swapDirs[d->dirn].l2)
 520             continue;
 521         d->curlvl2 = 0;
 522         if (++d->curlvl1 < Config.cacheSwap.swapDirs[d->dirn].l1)
 523             continue;
 524         d->curlvl1 = 0;
 525         d->done = 1;
 526     }
 527     *sfileno = d->fn;
 528     return fd;
 529 }
 530
 531 /* Add a new object to the cache with empty memory copy and pointer to disk
 532  * use to rebuild store from disk. */
 533 static StoreEntry *
 534 storeAddDiskRestore(const cache_key * key,
 535     int file_number,
 536     size_t swap_file_sz,
 537     time_t expires,
 538     time_t timestamp,
 539     time_t lastref,
 540     time_t lastmod,
 541     u_num32 refcount,
 542     u_num32 flags,
 543     int clean)
 544 {
 545     StoreEntry *e = NULL;
 546     debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
 547     /* if you call this you'd better be sure file_number is not
 548      * already in use! */
 549     e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
 550     e->store_status = STORE_OK;
 551     storeSetMemStatus(e, NOT_IN_MEMORY);
 552     e->swap_status = SWAPOUT_DONE;
 553     e->swap_file_number = file_number;
 554     e->swap_file_sz = swap_file_sz;
 555     e->lock_count = 0;
 556     e->refcount = 0;
 557     e->lastref = lastref;
 558     e->timestamp = timestamp;
 559     e->expires = expires;
 560     e->lastmod = lastmod;
 561     e->refcount = refcount;
 562     e->flag = flags;
 563     EBIT_SET(e->flag, ENTRY_CACHABLE);
 564     EBIT_CLR(e->flag, RELEASE_REQUEST);
 565     EBIT_CLR(e->flag, KEY_PRIVATE);
 566     e->ping_status = PING_NONE;
 567     EBIT_CLR(e->flag, ENTRY_VALIDATED);
 568     storeDirMapBitSet(e->swap_file_number);
 569     storeHashInsert(e, key);    /* do it after we clear KEY_PRIVATE */
 570     return e;
 571 }
 572
 573 static void
 574 storeCleanup(void *datanotused)
 575 {
 576     static int bucketnum = -1;
 577     static int validnum = 0;
 578     static int store_errors = 0;
 579     StoreEntry *e;
 580     hash_link *link_ptr = NULL;
 581     hash_link *link_next = NULL;
 582     if (++bucketnum >= store_hash_buckets) {
 583         debug(20, 1) ("  Completed Validation Procedure\n");
 584         debug(20, 1) ("  Validated %d Entries\n", validnum);
 585         debug(20, 1) ("  store_swap_size = %dk\n", store_swap_size);
 586         store_rebuilding = 0;
 587         if (opt_store_doublecheck)
 588             assert(store_errors == 0);
 589         if (store_digest)
 590             storeDigestNoteStoreReady();
 591         return;
 592     }
 593     link_next = hash_get_bucket(store_table, bucketnum);
 594     while (NULL != (link_ptr = link_next)) {
 595         link_next = link_ptr->next;
 596         e = (StoreEntry *) link_ptr;
 597         if (EBIT_TEST(e->flag, ENTRY_VALIDATED))
 598             continue;
 599         /*
 600          * Calling storeRelease() has no effect because we're
 601          * still in 'store_rebuilding' state
 602          */
 603         if (e->swap_file_number < 0)
 604             continue;
 605         if (opt_store_doublecheck) {
 606             struct stat sb;
 607             if (stat(storeSwapFullPath(e->swap_file_number, NULL), &sb) < 0) {
 608                 store_errors++;
 609                 debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n");
 610                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
 611                 debug(20, 0) ("storeCleanup: PATH %s\n",
 612                     storeSwapFullPath(e->swap_file_number, NULL));
 613                 storeEntryDump(e, 0);
 614                 continue;
 615             }
 616             if (e->swap_file_sz != sb.st_size) {
 617                 store_errors++;
 618                 debug(20, 0) ("storeCleanup: SIZE MISMATCH\n");
 619                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
 620                 debug(20, 0) ("storeCleanup: PATH %s\n",
 621                     storeSwapFullPath(e->swap_file_number, NULL));
 622                 debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n",
 623                     e->swap_file_sz, (int) sb.st_size);
 624                 storeEntryDump(e, 0);
 625                 continue;
 626             }
 627         }
 628         EBIT_SET(e->flag, ENTRY_VALIDATED);
 629         /* Only set the file bit if we know its a valid entry */
 630         /* otherwise, set it in the validation procedure */
 631         storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
 632         if ((++validnum & 0xFFFF) == 0)
 633             debug(20, 1) ("  %7d Entries Validated so far.\n", validnum);
 634     }
 635     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 636 }
 637
 638 void
 639 storeValidate(StoreEntry * e, STVLDCB * callback, void *callback_data, void *tag)
 640 {
 641     valid_ctrl_t *ctrlp;
 642     char *path;
 643     struct stat *sb;
 644 #if !USE_ASYNC_IO
 645     int x;
 646 #endif
 647     assert(!EBIT_TEST(e->flag, ENTRY_VALIDATED));
 648     if (e->swap_file_number < 0) {
 649         EBIT_CLR(e->flag, ENTRY_VALIDATED);
 650         callback(callback_data, 0, 0);
 651         return;
 652     }
 653     path = storeSwapFullPath(e->swap_file_number, NULL);
 654     sb = xmalloc(sizeof(struct stat));
 655     ctrlp = xmalloc(sizeof(valid_ctrl_t));
 656     ctrlp->sb = sb;
 657     ctrlp->e = e;
 658     ctrlp->callback = callback;
 659     ctrlp->callback_data = callback_data;
 660 #if USE_ASYNC_IO
 661     aioStat(path, sb, storeValidateComplete, ctrlp, tag);
 662 #else
 663     /*
 664      * When evaluating the actual arguments in a function call, the order
 665      * in which the arguments and the function expression are evaluated is
 666      * not specified;
 667      */
 668     x = stat(path, sb);
 669     storeValidateComplete(-1, ctrlp, x, errno);
 670 #endif
 671     return;
 672 }
 673
 674 void
 675 storeValidateComplete(int fd, void *data, int retcode, int errcode)
 676 {
 677     valid_ctrl_t *ctrlp = data;
 678     struct stat *sb = ctrlp->sb;
 679     StoreEntry *e = ctrlp->e;
 680     char *path;
 681
 682     if (retcode == -2 && errcode == -2) {
 683         xfree(sb);
 684         xfree(ctrlp);
 685         ctrlp->callback(ctrlp->callback_data, retcode, errcode);
 686         return;
 687     }
 688     if (retcode < 0 && errcode == EWOULDBLOCK) {
 689         path = storeSwapFullPath(e->swap_file_number, NULL);
 690         retcode = stat(path, sb);
 691     }
 692     if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->swap_file_sz) {
 693         EBIT_CLR(e->flag, ENTRY_VALIDATED);
 694     } else {
 695         EBIT_SET(e->flag, ENTRY_VALIDATED);
 696         storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
 697     }
 698     errno = errcode;
 699     ctrlp->callback(ctrlp->callback_data, retcode, errcode);
 700     xfree(sb);
 701     xfree(ctrlp);
 702 }
 703
 704 /* meta data recreated from disk image in swap directory */
 705 static void
 706 storeRebuildComplete(void)
 707 {
 708     time_t r;
 709     time_t stop;
 710     stop = squid_curtime;
 711     r = stop - RebuildState.start;
 712     debug(20, 1) ("Finished rebuilding storage disk.\n");
 713     debug(20, 1) ("  %7d Entries read from previous logfile.\n",
 714         RebuildState.linecount);
 715     debug(20, 1) ("  %7d Entries scanned from swap files.\n",
 716         RebuildState.statcount);
 717     debug(20, 1) ("  %7d Invalid entries.\n", RebuildState.invalid);
 718     debug(20, 1) ("  %7d With invalid flags.\n", RebuildState.badflags);
 719     debug(20, 1) ("  %7d Objects loaded.\n", RebuildState.objcount);
 720     debug(20, 1) ("  %7d Objects expired.\n", RebuildState.expcount);
 721     debug(20, 1) ("  %7d Objects cancelled.\n", RebuildState.cancelcount);
 722     debug(20, 1) ("  %7d Duplicate URLs purged.\n", RebuildState.dupcount);
 723     debug(20, 1) ("  %7d Swapfile clashes avoided.\n", RebuildState.clashcount);
 724     debug(20, 1) ("  Took %d seconds (%6.1f objects/sec).\n",
 725         r > 0 ? (int) r : 0,
 726         (double) RebuildState.objcount / (r > 0 ? r : 1));
 727     debug(20, 1) ("Beginning Validation Procedure\n");
 728     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 729 }
 730
 731 void
 732 storeRebuildStart(void)
 733 {
 734     rebuild_dir *d;
 735     int clean = 0;
 736     int zero = 0;
 737     FILE *fp;
 738     int i;
 739     memset(&RebuildState, '\0', sizeof(RebuildState));
 740     RebuildState.start = squid_curtime;
 741     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 742         d = xcalloc(1, sizeof(rebuild_dir));
 743         d->dirn = i;
 744         d->speed = opt_foreground_rebuild ? 1 << 30 : 50;
 745         /*
 746          * If the swap.state file exists in the cache_dir, then
 747          * we'll use storeRebuildFromSwapLog(), otherwise we'll
 748          * use storeRebuildFromDirectory() to open up each file
 749          * and suck in the meta data.
 750          */
 751         fp = storeDirOpenTmpSwapLog(i, &clean, &zero);
 752         if (fp == NULL || zero) {
 753             if (fp != NULL)
 754                 fclose(fp);
 755             d->rebuild_func = storeRebuildFromDirectory;
 756         } else {
 757             d->rebuild_func = storeRebuildFromSwapLog;
 758             d->log = fp;
 759             d->clean = clean;
 760         }
 761         d->next = RebuildState.rebuild_dir;
 762         RebuildState.rebuild_dir = d;
 763         if (!clean)
 764             RebuildState.need_to_validate = 1;
 765         debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n",
 766             i, clean ? "CLEAN" : "DIRTY");
 767     }
 768     eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
 769 }