src/store_rebuild.cc

   1
   2 /*
   3  * $Id: store_rebuild.cc,v 1.52 1998/10/09 17:46:37 wessels Exp $
   4  *
   5  * DEBUG: section 20    Store Rebuild Routines
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Internet Object Cache  http://squid.nlanr.net/Squid/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from the
  12  *  Internet community.  Development is led by Duane Wessels of the
  13  *  National Laboratory for Applied Network Research and funded by the
  14  *  National Science Foundation.  Squid is Copyrighted (C) 1998 by
  15  *  Duane Wessels and the University of California San Diego.  Please
  16  *  see the COPYRIGHT file for full details.  Squid incorporates
  17  *  software developed and/or copyrighted by other sources.  Please see
  18  *  the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37
  38 #define STORE_META_BUFSZ 4096
  39
  40
  41 typedef struct _rebuild_dir rebuild_dir;
  42 typedef int RBHD(rebuild_dir * d);
  43
  44 struct _rebuild_dir {
  45     int dirn;
  46     int n_read;
  47     FILE *log;
  48     int speed;
  49     int clean;
  50     int curlvl1;
  51     int curlvl2;
  52     int flag;
  53     int done;
  54     int in_dir;
  55     int fn;
  56     struct dirent *entry;
  57     DIR *td;
  58     RBHD *rebuild_func;
  59     rebuild_dir *next;
  60     char fullpath[SQUID_MAXPATHLEN];
  61     char fullfilename[SQUID_MAXPATHLEN];
  62 };
  63
  64 struct storeRebuildState {
  65     rebuild_dir *rebuild_dir;
  66     int objcount;               /* # objects successfully reloaded */
  67     int expcount;               /* # objects expired */
  68     int linecount;              /* # lines parsed from cache logfile */
  69     int statcount;              /* # entries from directory walking */
  70     int clashcount;             /* # swapfile clashes avoided */
  71     int dupcount;               /* # duplicates purged */
  72     int cancelcount;            /* # SWAP_LOG_DEL objects purged */
  73     int invalid;                /* # bad lines */
  74     int badflags;               /* # bad e->flags */
  75     int need_to_validate;
  76     int bad_log_op;
  77     int zero_object_sz;
  78     time_t start;
  79     time_t stop;
  80 } RebuildState;
  81
  82 typedef struct valid_ctrl_t {
  83     struct stat *sb;
  84     StoreEntry *e;
  85     STVLDCB *callback;
  86     void *callback_data;
  87 } valid_ctrl_t;
  88
  89 static RBHD storeRebuildFromDirectory;
  90 static RBHD storeRebuildFromSwapLog;
  91 static void storeRebuildComplete(void);
  92 static EVH storeRebuildADirectory;
  93 static int storeGetNextFile(rebuild_dir *, int *sfileno, int *size);
  94 static StoreEntry *storeAddDiskRestore(const cache_key * key,
  95     int file_number,
  96     size_t swap_file_sz,
  97     time_t expires,
  98     time_t timestamp,
  99     time_t lastref,
 100     time_t lastmod,
 101     u_num32 refcount,
 102     u_short flags,
 103     int clean);
 104 static AIOCB storeValidateComplete;
 105
 106 static int
 107 storeRebuildFromDirectory(rebuild_dir * d)
 108 {
 109     LOCAL_ARRAY(char, hdr_buf, DISK_PAGE_SIZE);
 110     StoreEntry *e = NULL;
 111     StoreEntry tmpe;
 112     cache_key key[MD5_DIGEST_CHARS];
 113     int sfileno = 0;
 114     int count;
 115     int size;
 116     struct stat sb;
 117     int swap_hdr_len;
 118     int fd = -1;
 119     tlv *tlv_list;
 120     tlv *t;
 121     assert(d != NULL);
 122     debug(20, 3) ("storeRebuildFromDirectory: DIR #%d\n", d->dirn);
 123     for (count = 0; count < d->speed; count++) {
 124         assert(fd == -1);
 125         fd = storeGetNextFile(d, &sfileno, &size);
 126         if (fd == -2) {
 127             debug(20, 1) ("storeRebuildFromDirectory: DIR #%d done!\n", d->dirn);
 128             storeDirCloseTmpSwapLog(d->dirn);
 129             store_rebuilding = 0;
 130             return -1;
 131         } else if (fd < 0) {
 132             continue;
 133         }
 134         assert(fd > -1);
 135         /* lets get file stats here */
 136         if (fstat(fd, &sb) < 0) {
 137             debug(20, 1) ("storeRebuildFromDirectory: fstat(FD %d): %s\n",
 138                 fd, xstrerror());
 139             file_close(fd);
 140             fd = -1;
 141             continue;
 142         }
 143         if ((++RebuildState.statcount & 0xFFFF) == 0)
 144             debug(20, 1) ("  %7d files opened so far.\n",
 145                 RebuildState.statcount);
 146         debug(20, 9) ("file_in: fd=%d %08X\n", fd, sfileno);
 147         Counter.syscalls.disk.reads++;
 148         if (read(fd, hdr_buf, DISK_PAGE_SIZE) < 0) {
 149             debug(20, 1) ("storeRebuildFromDirectory: read(FD %d): %s\n",
 150                 fd, xstrerror());
 151             file_close(fd);
 152             fd = -1;
 153             continue;
 154         }
 155         file_close(fd);
 156         fd = -1;
 157         swap_hdr_len = 0;
 158         tlv_list = storeSwapMetaUnpack(hdr_buf, &swap_hdr_len);
 159         if (tlv_list == NULL) {
 160             debug(20, 1) ("storeRebuildFromDirectory: failed to get meta data\n");
 161             storeUnlinkFileno(sfileno);
 162             continue;
 163         }
 164         debug(20, 3) ("storeRebuildFromDirectory: successful swap meta unpacking\n");
 165         memset(key, '\0', MD5_DIGEST_CHARS);
 166         memset(&tmpe, '\0', sizeof(StoreEntry));
 167         for (t = tlv_list; t; t = t->next) {
 168             switch (t->type) {
 169             case STORE_META_KEY:
 170                 assert(t->length == MD5_DIGEST_CHARS);
 171                 xmemcpy(key, t->value, MD5_DIGEST_CHARS);
 172                 break;
 173             case STORE_META_STD:
 174                 assert(t->length == STORE_HDR_METASIZE);
 175                 xmemcpy(&tmpe.timestamp, t->value, STORE_HDR_METASIZE);
 176                 break;
 177             default:
 178                 break;
 179             }
 180         }
 181         storeSwapTLVFree(tlv_list);
 182         tlv_list = NULL;
 183         if (storeKeyNull(key)) {
 184             debug(20, 1) ("storeRebuildFromDirectory: NULL key\n");
 185             storeUnlinkFileno(sfileno);
 186             continue;
 187         }
 188         tmpe.key = key;
 189         /* check sizes */
 190         if (tmpe.swap_file_sz == 0) {
 191             tmpe.swap_file_sz = sb.st_size;
 192         } else if (tmpe.swap_file_sz == sb.st_size - swap_hdr_len) {
 193             tmpe.swap_file_sz = sb.st_size;
 194         } else if (tmpe.swap_file_sz != sb.st_size) {
 195             debug(20, 1) ("storeRebuildFromDirectory: SIZE MISMATCH %d!=%d\n",
 196                 tmpe.swap_file_sz, (int) sb.st_size);
 197             storeUnlinkFileno(sfileno);
 198             continue;
 199         }
 200         if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
 201             storeUnlinkFileno(sfileno);
 202             RebuildState.badflags++;
 203             continue;
 204         }
 205         e = storeGet(key);
 206         if (e && e->lastref >= tmpe.lastref) {
 207             /* key already exists, current entry is newer */
 208             /* keep old, ignore new */
 209             RebuildState.dupcount++;
 210             continue;
 211         } else if (NULL != e) {
 212             /* URL already exists, this swapfile not being used */
 213             /* junk old, load new */
 214             storeRelease(e);    /* release old entry */
 215             RebuildState.dupcount++;
 216         }
 217         RebuildState.objcount++;
 218         storeEntryDump(&tmpe, 5);
 219         e = storeAddDiskRestore(key,
 220             sfileno,
 221             tmpe.swap_file_sz,
 222             tmpe.expires,
 223             tmpe.timestamp,
 224             tmpe.lastref,
 225             tmpe.lastmod,
 226             tmpe.refcount,      /* refcount */
 227             tmpe.flags,         /* flags */
 228             d->clean);
 229     }
 230     return count;
 231 }
 232
 233 static int
 234 storeRebuildFromSwapLog(rebuild_dir * d)
 235 {
 236     StoreEntry *e = NULL;
 237     storeSwapLogData s;
 238     size_t ss = sizeof(storeSwapLogData);
 239     int count;
 240     int used;                   /* is swapfile already in use? */
 241     int disk_entry_newer;       /* is the log entry newer than current entry? */
 242     double x;
 243     assert(d != NULL);
 244     /* load a number of objects per invocation */
 245     for (count = 0; count < d->speed; count++) {
 246         if (fread(&s, ss, 1, d->log) != 1) {
 247             debug(20, 1) ("Done reading Cache Dir #%d swaplog (%d entries)\n",
 248                 d->dirn, d->n_read);
 249             fclose(d->log);
 250             d->log = NULL;
 251             storeDirCloseTmpSwapLog(d->dirn);
 252             return -1;
 253         }
 254         d->n_read++;
 255         if (s.op <= SWAP_LOG_NOP)
 256             continue;
 257         if (s.op >= SWAP_LOG_MAX)
 258             continue;
 259         s.swap_file_number = storeDirProperFileno(d->dirn, s.swap_file_number);
 260         debug(20, 3) ("storeRebuildFromSwapLog: %s %s %08X\n",
 261             swap_log_op_str[(int) s.op],
 262             storeKeyText(s.key),
 263             s.swap_file_number);
 264         if (s.op == SWAP_LOG_ADD) {
 265             (void) 0;
 266         } else if (s.op == SWAP_LOG_DEL) {
 267             if ((e = storeGet(s.key)) != NULL) {
 268                 /*
 269                  * Make sure we don't unlink the file, it might be
 270                  * in use by a subsequent entry.  Also note that
 271                  * we don't have to subtract from store_swap_size
 272                  * because adding to store_swap_size happens in
 273                  * the cleanup procedure.
 274                  */
 275                 storeExpireNow(e);
 276                 storeReleaseRequest(e);
 277                 if (e->swap_file_number > -1) {
 278                     storeDirMapBitReset(e->swap_file_number);
 279                     e->swap_file_number = -1;
 280                 }
 281                 RebuildState.objcount--;
 282                 RebuildState.cancelcount++;
 283             }
 284             continue;
 285         } else {
 286             x = log(++RebuildState.bad_log_op) / log(10.0);
 287             if (0.0 == x - (double) (int) x)
 288                 debug(20, 1) ("WARNING: %d invalid swap log entries found\n",
 289                     RebuildState.bad_log_op);
 290             RebuildState.invalid++;
 291             continue;
 292         }
 293         if ((++RebuildState.linecount & 0xFFFF) == 0)
 294             debug(20, 1) ("  %7d Entries read so far.\n",
 295                 RebuildState.linecount);
 296         if (!storeDirValidFileno(s.swap_file_number)) {
 297             RebuildState.invalid++;
 298             continue;
 299         }
 300         if (EBIT_TEST(s.flags, KEY_PRIVATE)) {
 301             RebuildState.badflags++;
 302             continue;
 303         }
 304         e = storeGet(s.key);
 305         used = storeDirMapBitTest(s.swap_file_number);
 306         /* If this URL already exists in the cache, does the swap log
 307          * appear to have a newer entry?  Compare 'lastref' from the
 308          * swap log to e->lastref. */
 309         disk_entry_newer = e ? (s.lastref > e->lastref ? 1 : 0) : 0;
 310         if (used && !disk_entry_newer) {
 311             /* log entry is old, ignore it */
 312             RebuildState.clashcount++;
 313             continue;
 314         } else if (used && e && e->swap_file_number == s.swap_file_number) {
 315             /* swapfile taken, same URL, newer, update meta */
 316             if (e->store_status == STORE_OK) {
 317                 e->lastref = s.timestamp;
 318                 e->timestamp = s.timestamp;
 319                 e->expires = s.expires;
 320                 e->lastmod = s.lastmod;
 321                 e->flags = s.flags;
 322                 e->refcount += s.refcount;
 323             } else {
 324                 debug_trap("storeRebuildFromSwapLog: bad condition");
 325                 debug(20, 1) ("\tSee %s:%d\n", __FILE__, __LINE__);
 326             }
 327             continue;
 328         } else if (used) {
 329             /* swapfile in use, not by this URL, log entry is newer */
 330             /* This is sorta bad: the log entry should NOT be newer at this
 331              * point.  If the log is dirty, the filesize check should have
 332              * caught this.  If the log is clean, there should never be a
 333              * newer entry. */
 334             debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n",
 335                 s.swap_file_number);
 336             /* I'm tempted to remove the swapfile here just to be safe,
 337              * but there is a bad race condition in the NOVM version if
 338              * the swapfile has recently been opened for writing, but
 339              * not yet opened for reading.  Because we can't map
 340              * swapfiles back to StoreEntrys, we don't know the state
 341              * of the entry using that file.  */
 342             /* We'll assume the existing entry is valid, probably because
 343              * were in a slow rebuild and the the swap file number got taken
 344              * and the validation procedure hasn't run. */
 345             assert(RebuildState.need_to_validate);
 346             RebuildState.clashcount++;
 347             continue;
 348         } else if (e && !disk_entry_newer) {
 349             /* key already exists, current entry is newer */
 350             /* keep old, ignore new */
 351             RebuildState.dupcount++;
 352             continue;
 353         } else if (e) {
 354             /* key already exists, this swapfile not being used */
 355             /* junk old, load new */
 356             storeExpireNow(e);
 357             storeReleaseRequest(e);
 358             if (e->swap_file_number > -1) {
 359                 storeDirMapBitReset(e->swap_file_number);
 360                 e->swap_file_number = -1;
 361             }
 362             RebuildState.dupcount++;
 363         } else {
 364             /* URL doesnt exist, swapfile not in use */
 365             /* load new */
 366             (void) 0;
 367         }
 368         /* update store_swap_size */
 369         RebuildState.objcount++;
 370         e = storeAddDiskRestore(s.key,
 371             s.swap_file_number,
 372             s.swap_file_sz,
 373             s.expires,
 374             s.timestamp,
 375             s.lastref,
 376             s.lastmod,
 377             s.refcount,
 378             s.flags,
 379             d->clean);
 380         storeDirSwapLog(e, SWAP_LOG_ADD);
 381     }
 382     return count;
 383 }
 384
 385 static void
 386 storeRebuildADirectory(void *unused)
 387 {
 388     int count;
 389     rebuild_dir *d;
 390     rebuild_dir **D;
 391     if ((d = RebuildState.rebuild_dir) == NULL) {
 392         storeRebuildComplete();
 393         return;
 394     }
 395     count = d->rebuild_func(d);
 396     RebuildState.rebuild_dir = d->next;
 397     if (count < 0) {
 398         xfree(d);
 399     } else {
 400         for (D = &RebuildState.rebuild_dir; *D; D = &(*D)->next);
 401         *D = d;
 402         d->next = NULL;
 403     }
 404     if (opt_foreground_rebuild)
 405         storeRebuildADirectory(NULL);
 406     else
 407         eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
 408 }
 409
 410 #if TEMP_UNUSED_CODE
 411 static void
 412 storeConvertFile(const cache_key * key,
 413     int file_number,
 414     size_t swap_file_sz,
 415     time_t expires,
 416     time_t timestamp,
 417     time_t lastref,
 418     time_t lastmod,
 419     u_short refcount,
 420     u_short flags,
 421     int clean)
 422 {
 423     int fd_r, fd_w;
 424     int hdr_len, x, y;
 425     LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN);
 426     LOCAL_ARRAY(char, copybuf, DISK_PAGE_SIZE);
 427     char *buf;
 428     tlv *tlv_list;
 429     StoreEntry e;
 430     e.key = key;
 431     e.swap_file_sz = swap_file_sz;
 432     e.expires = expires;
 433     e.lastref = lastref;
 434     e.refcount = refcount;
 435     e.flag = flags;
 436     storeSwapFullPath(file_number, swapfilename);
 437     fd_r = file_open(swapfilename, O_RDONLY, NULL, NULL, NULL);
 438     if (fd_r < 0)
 439         return;
 440     safeunlink(swapfilename, 1);
 441     fd_w = file_open(swapfilename, O_CREAT | O_WRONLY | O_TRUNC, NULL, NULL, NULL);
 442     tlv_list = storeSwapMetaBuild(&e);
 443     buf = storeSwapMetaPack(tlv_list, &hdr_len);
 444     x = write(fd_w, buf, hdr_len);
 445     while (x > 0) {
 446         y = read(fd_r, copybuf, DISK_PAGE_SIZE);
 447         x = write(fd_w, copybuf, y);
 448     }
 449     file_close(fd_r);
 450     file_close(fd_w);
 451     xfree(buf);
 452     storeSwapTLVFree(tlv_list);
 453 }
 454 #endif
 455
 456 static int
 457 storeGetNextFile(rebuild_dir * d, int *sfileno, int *size)
 458 {
 459     int fd = -1;
 460     int used = 0;
 461     debug(20, 3) ("storeGetNextFile: flag=%d, %d: /%02X/%02X\n",
 462         d->flag,
 463         d->dirn,
 464         d->curlvl1,
 465         d->curlvl2);
 466     if (d->done)
 467         return -2;
 468     while (fd < 0 && d->done == 0) {
 469         fd = -1;
 470         if (0 == d->flag) {     /* initialize, open first file */
 471             d->done = 0;
 472             d->curlvl1 = 0;
 473             d->curlvl2 = 0;
 474             d->in_dir = 0;
 475             d->flag = 1;
 476             assert(Config.cacheSwap.n_configured > 0);
 477         }
 478         if (0 == d->in_dir) {   /* we need to read in a new directory */
 479             snprintf(d->fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
 480                 Config.cacheSwap.swapDirs[d->dirn].path,
 481                 d->curlvl1, d->curlvl2);
 482             if (d->flag && d->td != NULL)
 483                 closedir(d->td);
 484             d->td = opendir(d->fullpath);
 485             if (d->td == NULL) {
 486                 debug(50, 1) ("storeGetNextFile: opendir: %s: %s\n",
 487                     d->fullpath, xstrerror());
 488                 break;
 489             }
 490             d->entry = readdir(d->td);  /* skip . and .. */
 491             d->entry = readdir(d->td);
 492             if (d->entry == NULL && errno == ENOENT)
 493                 debug(20, 1) ("storeGetNextFile: directory does not exist!.\n");
 494             debug(20, 3) ("storeGetNextFile: Directory %s\n", d->fullpath);
 495         }
 496         if (d->td != NULL && (d->entry = readdir(d->td)) != NULL) {
 497             d->in_dir++;
 498             if (sscanf(d->entry->d_name, "%x", &d->fn) != 1) {
 499                 debug(20, 3) ("storeGetNextFile: invalid %s\n",
 500                     d->entry->d_name);
 501                 continue;
 502             }
 503             if (!storeFilenoBelongsHere(d->fn, d->dirn, d->curlvl1, d->curlvl2)) {
 504                 debug(20, 3) ("storeGetNextFile: %08X does not belong in %d/%d/%d\n",
 505                     d->fn, d->dirn, d->curlvl1, d->curlvl2);
 506                 continue;
 507             }
 508             d->fn = storeDirProperFileno(d->dirn, d->fn);
 509             used = storeDirMapBitTest(d->fn);
 510             if (used) {
 511                 debug(20, 3) ("storeGetNextFile: Locked, continuing with next.\n");
 512                 continue;
 513             }
 514             snprintf(d->fullfilename, SQUID_MAXPATHLEN, "%s/%s",
 515                 d->fullpath, d->entry->d_name);
 516             debug(20, 3) ("storeGetNextFile: Opening %s\n", d->fullfilename);
 517             fd = file_open(d->fullfilename, O_RDONLY, NULL, NULL, NULL);
 518             if (fd < 0)
 519                 debug(50, 1) ("storeGetNextFile: %s: %s\n", d->fullfilename, xstrerror());
 520             continue;
 521         }
 522         d->in_dir = 0;
 523         if (++d->curlvl2 < Config.cacheSwap.swapDirs[d->dirn].l2)
 524             continue;
 525         d->curlvl2 = 0;
 526         if (++d->curlvl1 < Config.cacheSwap.swapDirs[d->dirn].l1)
 527             continue;
 528         d->curlvl1 = 0;
 529         d->done = 1;
 530     }
 531     *sfileno = d->fn;
 532     return fd;
 533 }
 534
 535 /* Add a new object to the cache with empty memory copy and pointer to disk
 536  * use to rebuild store from disk. */
 537 static StoreEntry *
 538 storeAddDiskRestore(const cache_key * key,
 539     int file_number,
 540     size_t swap_file_sz,
 541     time_t expires,
 542     time_t timestamp,
 543     time_t lastref,
 544     time_t lastmod,
 545     u_num32 refcount,
 546     u_short flags,
 547     int clean)
 548 {
 549     StoreEntry *e = NULL;
 550     debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number);
 551     /* if you call this you'd better be sure file_number is not
 552      * already in use! */
 553     e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL);
 554     e->store_status = STORE_OK;
 555     storeSetMemStatus(e, NOT_IN_MEMORY);
 556     e->swap_status = SWAPOUT_DONE;
 557     e->swap_file_number = file_number;
 558     e->swap_file_sz = swap_file_sz;
 559     e->lock_count = 0;
 560     e->refcount = 0;
 561     e->lastref = lastref;
 562     e->timestamp = timestamp;
 563     e->expires = expires;
 564     e->lastmod = lastmod;
 565     e->refcount = refcount;
 566     e->flags = flags;
 567     EBIT_SET(e->flags, ENTRY_CACHABLE);
 568     EBIT_CLR(e->flags, RELEASE_REQUEST);
 569     EBIT_CLR(e->flags, KEY_PRIVATE);
 570     e->ping_status = PING_NONE;
 571     EBIT_CLR(e->flags, ENTRY_VALIDATED);
 572     storeDirMapBitSet(e->swap_file_number);
 573     storeHashInsert(e, key);    /* do it after we clear KEY_PRIVATE */
 574     return e;
 575 }
 576
 577 static void
 578 storeCleanup(void *datanotused)
 579 {
 580     static int bucketnum = -1;
 581     static int validnum = 0;
 582     static int store_errors = 0;
 583     StoreEntry *e;
 584     hash_link *link_ptr = NULL;
 585     hash_link *link_next = NULL;
 586     if (++bucketnum >= store_hash_buckets) {
 587         debug(20, 1) ("  Completed Validation Procedure\n");
 588         debug(20, 1) ("  Validated %d Entries\n", validnum);
 589         debug(20, 1) ("  store_swap_size = %dk\n", store_swap_size);
 590         store_rebuilding = 0;
 591         if (opt_store_doublecheck)
 592             assert(store_errors == 0);
 593         if (store_digest)
 594             storeDigestNoteStoreReady();
 595         return;
 596     }
 597     link_next = hash_get_bucket(store_table, bucketnum);
 598     while (NULL != (link_ptr = link_next)) {
 599         link_next = link_ptr->next;
 600         e = (StoreEntry *) link_ptr;
 601         if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
 602             continue;
 603         /*
 604          * Calling storeRelease() has no effect because we're
 605          * still in 'store_rebuilding' state
 606          */
 607         if (e->swap_file_number < 0)
 608             continue;
 609         if (opt_store_doublecheck) {
 610             struct stat sb;
 611             if (stat(storeSwapFullPath(e->swap_file_number, NULL), &sb) < 0) {
 612                 store_errors++;
 613                 debug(20, 0) ("storeCleanup: MISSING SWAP FILE\n");
 614                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
 615                 debug(20, 0) ("storeCleanup: PATH %s\n",
 616                     storeSwapFullPath(e->swap_file_number, NULL));
 617                 storeEntryDump(e, 0);
 618                 continue;
 619             }
 620             if (e->swap_file_sz != sb.st_size) {
 621                 store_errors++;
 622                 debug(20, 0) ("storeCleanup: SIZE MISMATCH\n");
 623                 debug(20, 0) ("storeCleanup: FILENO %08X\n", e->swap_file_number);
 624                 debug(20, 0) ("storeCleanup: PATH %s\n",
 625                     storeSwapFullPath(e->swap_file_number, NULL));
 626                 debug(20, 0) ("storeCleanup: ENTRY SIZE: %d, FILE SIZE: %d\n",
 627                     e->swap_file_sz, (int) sb.st_size);
 628                 storeEntryDump(e, 0);
 629                 continue;
 630             }
 631         }
 632         EBIT_SET(e->flags, ENTRY_VALIDATED);
 633         /* Only set the file bit if we know its a valid entry */
 634         /* otherwise, set it in the validation procedure */
 635         storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
 636         if ((++validnum & 0xFFFF) == 0)
 637             debug(20, 1) ("  %7d Entries Validated so far.\n", validnum);
 638     }
 639     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 640 }
 641
 642 void
 643 storeValidate(StoreEntry * e, STVLDCB * callback, void *callback_data, void *tag)
 644 {
 645     valid_ctrl_t *ctrlp;
 646     char *path;
 647     struct stat *sb;
 648 #if !USE_ASYNC_IO
 649     int x;
 650 #endif
 651     assert(!EBIT_TEST(e->flags, ENTRY_VALIDATED));
 652     if (e->swap_file_number < 0) {
 653         EBIT_CLR(e->flags, ENTRY_VALIDATED);
 654         callback(callback_data, 0, 0);
 655         return;
 656     }
 657     path = storeSwapFullPath(e->swap_file_number, NULL);
 658     sb = xmalloc(sizeof(struct stat));
 659     ctrlp = xmalloc(sizeof(valid_ctrl_t));
 660     ctrlp->sb = sb;
 661     ctrlp->e = e;
 662     ctrlp->callback = callback;
 663     ctrlp->callback_data = callback_data;
 664 #if USE_ASYNC_IO
 665     aioStat(path, sb, storeValidateComplete, ctrlp, tag);
 666 #else
 667     /*
 668      * When evaluating the actual arguments in a function call, the order
 669      * in which the arguments and the function expression are evaluated is
 670      * not specified;
 671      */
 672     x = stat(path, sb);
 673     storeValidateComplete(-1, ctrlp, x, errno);
 674 #endif
 675     return;
 676 }
 677
 678 static void
 679 storeValidateComplete(int fd, void *data, int retcode, int errcode)
 680 {
 681     valid_ctrl_t *ctrlp = data;
 682     struct stat *sb = ctrlp->sb;
 683     StoreEntry *e = ctrlp->e;
 684     char *path;
 685
 686     if (retcode == -2 && errcode == -2) {
 687         xfree(sb);
 688         xfree(ctrlp);
 689         ctrlp->callback(ctrlp->callback_data, retcode, errcode);
 690         return;
 691     }
 692     if (retcode < 0 && errcode == EWOULDBLOCK) {
 693         path = storeSwapFullPath(e->swap_file_number, NULL);
 694         retcode = stat(path, sb);
 695     }
 696     if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->swap_file_sz) {
 697         EBIT_CLR(e->flags, ENTRY_VALIDATED);
 698     } else {
 699         EBIT_SET(e->flags, ENTRY_VALIDATED);
 700         storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, 1);
 701     }
 702     errno = errcode;
 703     ctrlp->callback(ctrlp->callback_data, retcode, errcode);
 704     xfree(sb);
 705     xfree(ctrlp);
 706 }
 707
 708 /* meta data recreated from disk image in swap directory */
 709 static void
 710 storeRebuildComplete(void)
 711 {
 712     time_t r;
 713     time_t stop;
 714     stop = squid_curtime;
 715     r = stop - RebuildState.start;
 716     debug(20, 1) ("Finished rebuilding storage disk.\n");
 717     debug(20, 1) ("  %7d Entries read from previous logfile.\n",
 718         RebuildState.linecount);
 719     debug(20, 1) ("  %7d Entries scanned from swap files.\n",
 720         RebuildState.statcount);
 721     debug(20, 1) ("  %7d Invalid entries.\n", RebuildState.invalid);
 722     debug(20, 1) ("  %7d With invalid flags.\n", RebuildState.badflags);
 723     debug(20, 1) ("  %7d Objects loaded.\n", RebuildState.objcount);
 724     debug(20, 1) ("  %7d Objects expired.\n", RebuildState.expcount);
 725     debug(20, 1) ("  %7d Objects cancelled.\n", RebuildState.cancelcount);
 726     debug(20, 1) ("  %7d Duplicate URLs purged.\n", RebuildState.dupcount);
 727     debug(20, 1) ("  %7d Swapfile clashes avoided.\n", RebuildState.clashcount);
 728     debug(20, 1) ("  Took %d seconds (%6.1f objects/sec).\n",
 729         r > 0 ? (int) r : 0,
 730         (double) RebuildState.objcount / (r > 0 ? r : 1));
 731     debug(20, 1) ("Beginning Validation Procedure\n");
 732     eventAdd("storeCleanup", storeCleanup, NULL, 0.0, 1);
 733 }
 734
 735 void
 736 storeRebuildStart(void)
 737 {
 738     rebuild_dir *d;
 739     int clean = 0;
 740     int zero = 0;
 741     FILE *fp;
 742     int i;
 743     memset(&RebuildState, '\0', sizeof(RebuildState));
 744     RebuildState.start = squid_curtime;
 745     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 746         d = xcalloc(1, sizeof(rebuild_dir));
 747         d->dirn = i;
 748         d->speed = opt_foreground_rebuild ? 1 << 30 : 50;
 749         /*
 750          * If the swap.state file exists in the cache_dir, then
 751          * we'll use storeRebuildFromSwapLog(), otherwise we'll
 752          * use storeRebuildFromDirectory() to open up each file
 753          * and suck in the meta data.
 754          */
 755         fp = storeDirOpenTmpSwapLog(i, &clean, &zero);
 756         if (fp == NULL || zero) {
 757             if (fp != NULL)
 758                 fclose(fp);
 759             d->rebuild_func = storeRebuildFromDirectory;
 760         } else {
 761             d->rebuild_func = storeRebuildFromSwapLog;
 762             d->log = fp;
 763             d->clean = clean;
 764         }
 765         d->next = RebuildState.rebuild_dir;
 766         RebuildState.rebuild_dir = d;
 767         if (!clean)
 768             RebuildState.need_to_validate = 1;
 769         debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n",
 770             i, clean ? "CLEAN" : "DIRTY");
 771     }
 772     eventAdd("storeRebuild", storeRebuildADirectory, NULL, 0.0, 1);
 773 }