src/fs/ufs/ufscommon.cc

   1 /*
   2  * $Id: ufscommon.cc,v 1.13 2007/08/13 17:20:57 hno Exp $
   3  * vim: set et :
   4  *
   5  * DEBUG: section 47    Store Directory Routines
   6  * AUTHOR: Robert Collins
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org>
  35  */
  36
  37 #include "ufscommon.h"
  38 #include "Store.h"
  39 #include "fde.h"
  40 #include "StoreMeta.h"
  41 #include "Generic.h"
  42 #include "StoreMetaUnpacker.h"
  43 #include "RefCount.h"
  44 #include "StoreSwapLogData.h"
  45
  46 CBDATA_CLASS_INIT(RebuildState);
  47
  48
  49 class UFSSwapLogParser_old:public UFSSwapLogParser{
  50 public:
  51     struct StoreSwapLogDataOld{
  52         char op;
  53         sfileno swap_filen;
  54         time_t timestamp;
  55         time_t lastref;
  56         time_t expires;
  57         time_t lastmod;
  58         size_t swap_file_sz;
  59         u_short refcount;
  60         u_short flags;
  61         unsigned char key[MD5_DIGEST_CHARS];
  62     };
  63     UFSSwapLogParser_old(FILE *fp):UFSSwapLogParser(fp)
  64     {
  65         record_size = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld);
  66     }
  67     bool ReadRecord(StoreSwapLogData &swapData);
  68 };
  69
  70
  71 bool UFSSwapLogParser_old::ReadRecord(StoreSwapLogData &swapData){
  72     UFSSwapLogParser_old::StoreSwapLogDataOld readData;
  73     int bytes = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld);
  74
  75     assert(log);
  76
  77     if (fread(&readData, bytes, 1, log) != 1){
  78         return false;
  79     }
  80     swapData.op = readData.op;
  81     swapData.swap_filen = readData.swap_filen;
  82     swapData.timestamp = readData.timestamp;
  83     swapData.lastref = readData.lastref;
  84     swapData.expires = readData.expires;
  85     swapData.lastmod = readData.lastmod;
  86     swapData.swap_file_sz = readData.swap_file_sz;
  87     swapData.refcount = readData.refcount;
  88     swapData.flags = readData.flags;
  89     xmemcpy(swapData.key, readData.key, MD5_DIGEST_CHARS);
  90     return true;
  91 }
  92
  93
  94 class UFSSwapLogParser_v1:public UFSSwapLogParser{
  95 public:
  96     UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp)
  97     {
  98         record_size = sizeof(StoreSwapLogData);
  99     }
 100     bool ReadRecord(StoreSwapLogData &swapData);
 101 };
 102
 103
 104 bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData)
 105 {
 106     int bytes = sizeof(StoreSwapLogData);
 107
 108     assert(log);
 109
 110     if (fread(&swapData, bytes, 1, log) != 1){
 111         return false;
 112     }
 113     return true;
 114 }
 115
 116
 117 UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp)
 118 {
 119     StoreSwapLogHeader header;
 120
 121     assert(fp);
 122
 123     if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1)
 124          return NULL;
 125
 126     if (header.op != SWAP_LOG_VERSION){
 127         debugs(47, 1, "Old swap file detected... ");
 128         fseek(fp, 0, SEEK_SET);
 129         return new UFSSwapLogParser_old(fp);
 130     }
 131
 132     if (header.version == 1){
 133         if (fseek(fp, header.record_size, SEEK_SET) != 0)
 134             return NULL;
 135
 136         if (header.record_size == sizeof(struct UFSSwapLogParser_old::StoreSwapLogDataOld)){
 137             debugs(47, 1, "Version 1 of swap file without LFS support detected... ");
 138             return new UFSSwapLogParser_old(fp);
 139         }
 140
 141         if (header.record_size == sizeof(StoreSwapLogData)){
 142             debugs(47, 1, "Version 1 of swap file with LFS support detected... ");
 143             return new UFSSwapLogParser_v1(fp);
 144         }
 145
 146         debugs(47, 1, "The swap file has wrong format!... ");
 147         return NULL;
 148     }
 149
 150     return NULL;
 151 }
 152
 153 int UFSSwapLogParser::SwapLogEntries(){
 154     struct stat sb;
 155
 156     if (log_entries >= 0)
 157         return log_entries;
 158
 159     if (log && record_size && 0 == fstat(fileno(log), &sb)){
 160         log_entries = sb.st_size/record_size;
 161         return log_entries;
 162     }
 163
 164     return 0;
 165 }
 166
 167
 168
 169
 170 RebuildState::RebuildState (RefCount<UFSSwapDir> aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false)
 171 {
 172     speed = opt_foreground_rebuild ? 1 << 30 : 50;
 173     /*
 174      * If the swap.state file exists in the cache_dir, then
 175      * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll
 176      * use commonUfsDirRebuildFromDirectory() to open up each file
 177      * and suck in the meta data.
 178      */
 179     int clean = 0;
 180     int zeroLengthLog = 0;
 181     FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog);
 182
 183     if (fp && !zeroLengthLog)
 184         LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp);
 185
 186     if (LogParser == NULL ) {
 187         fromLog = false;
 188
 189         if (fp != NULL)
 190             fclose(fp);
 191
 192     } else {
 193         fromLog = true;
 194         flags.clean = (unsigned int) clean;
 195     }
 196
 197     if (!clean)
 198         flags.need_to_validate = 1;
 199
 200     debugs(47, 1, "Rebuilding storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")");
 201 }
 202
 203 RebuildState::~RebuildState()
 204 {
 205     sd->closeTmpSwapLog();
 206
 207     if (LogParser)
 208         delete LogParser;
 209 }
 210
 211 void
 212 RebuildState::RebuildStep(void *data)
 213 {
 214     RebuildState *rb = (RebuildState *)data;
 215     rb->rebuildStep();
 216
 217     if (!rb->isDone())
 218         eventAdd("storeRebuild", RebuildStep, rb, 0.0, 1);
 219     else {
 220         StoreController::store_dirs_rebuilding--;
 221         storeRebuildComplete(&rb->counts);
 222         delete rb;
 223     }
 224 }
 225
 226 void
 227 RebuildState::rebuildStep()
 228 {
 229     if (fromLog)
 230         rebuildFromSwapLog();
 231     else
 232         rebuildFromDirectory();
 233 }
 234
 235 struct InitStoreEntry : public unary_function<StoreMeta, void>
 236 {
 237     InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey){}
 238
 239     void operator()(StoreMeta const &x)
 240     {
 241         switch (x.getType()) {
 242
 243         case STORE_META_KEY:
 244             assert(x.length == MD5_DIGEST_CHARS);
 245             xmemcpy(index, x.value, MD5_DIGEST_CHARS);
 246             break;
 247
 248         case STORE_META_STD:
 249         struct old_metahdr{
 250                 time_t timestamp;
 251                 time_t lastref;
 252                 time_t expires;
 253                 time_t lastmod;
 254                 size_t swap_file_sz;
 255                 u_short refcount;
 256                 u_short flags;
 257             } *tmp;
 258             tmp = (struct old_metahdr *)x.value;
 259             assert(x.length == STORE_HDR_METASIZE_OLD);
 260             what->timestamp = tmp->timestamp;
 261             what->lastref = tmp->lastref;
 262             what->expires = tmp->expires;
 263             what->lastmod = tmp->lastmod;
 264             what->swap_file_sz = tmp->swap_file_sz;
 265             what->refcount = tmp->refcount;
 266             what->flags = tmp->flags;
 267             break;
 268
 269         case STORE_META_STD_LFS:
 270             assert(x.length == STORE_HDR_METASIZE);
 271             xmemcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
 272             break;
 273
 274         default:
 275             break;
 276         }
 277     }
 278
 279     StoreEntry *what;
 280     cache_key *index;
 281 };
 282
 283 void
 284 RebuildState::rebuildFromDirectory()
 285 {
 286     LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
 287     currentEntry(NULL);
 288     StoreEntry tmpe;
 289     cache_key key[MD5_DIGEST_CHARS];
 290
 291     struct stat sb;
 292     int swap_hdr_len;
 293     int fd = -1;
 294     StoreMeta *tlv_list;
 295     assert(this != NULL);
 296     debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index);
 297
 298     for (int count = 0; count < speed; count++) {
 299         assert(fd == -1);
 300         sfileno filn = 0;
 301         int size;
 302         fd = getNextFile(&filn, &size);
 303
 304         if (fd == -2) {
 305             debugs(47, 1, "Done scanning " << sd->path << " swaplog (" << n_read << " entries)");
 306             _done = true;
 307             return;
 308         } else if (fd < 0) {
 309             continue;
 310         }
 311
 312         assert(fd > -1);
 313         /* lets get file stats here */
 314
 315         if (fstat(fd, &sb) < 0) {
 316             debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror());
 317             file_close(fd);
 318             store_open_disk_fd--;
 319             fd = -1;
 320             continue;
 321         }
 322
 323         if ((++counts.scancount & 0xFFFF) == 0)
 324             debugs(47, 3, "  " << sd->path  << " " << std::setw(7) << counts.scancount  << " files opened so far.");
 325             debugs(47, 9, "file_in: fd=" << fd  << " "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn);
 326
 327
 328         statCounter.syscalls.disk.reads++;
 329
 330         int len;
 331
 332         if ((len = FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE)) < 0) {
 333             debugs(47, 1, "commonUfsDirRebuildFromDirectory: read(FD " << fd << "): " << xstrerror());
 334             file_close(fd);
 335             store_open_disk_fd--;
 336             fd = -1;
 337             continue;
 338         }
 339
 340         file_close(fd);
 341         store_open_disk_fd--;
 342         fd = -1;
 343         swap_hdr_len = 0;
 344
 345         StoreMetaUnpacker aBuilder(hdr_buf, len, &swap_hdr_len);
 346
 347         if (!aBuilder.isBufferSane()) {
 348             debugs(47, 1, "commonUfsDirRebuildFromDirectory: Swap data buffer length is not sane.");
 349             /* XXX shouldn't this be a call to commonUfsUnlink ? */
 350             sd->unlinkFile ( filn);
 351             continue;
 352         }
 353
 354         tlv_list = aBuilder.createStoreMeta ();
 355
 356         if (tlv_list == NULL) {
 357             debugs(47, 1, "commonUfsDirRebuildFromDirectory: failed to get meta data");
 358             /* XXX shouldn't this be a call to commonUfsUnlink ? */
 359             sd->unlinkFile (filn);
 360             continue;
 361         }
 362
 363         debugs(47, 3, "commonUfsDirRebuildFromDirectory: successful swap meta unpacking");
 364         memset(key, '\0', MD5_DIGEST_CHARS);
 365         memset(&tmpe, '\0', sizeof(StoreEntry));
 366         InitStoreEntry visitor(&tmpe, key);
 367         for_each(*tlv_list, visitor);
 368         storeSwapTLVFree(tlv_list);
 369         tlv_list = NULL;
 370
 371         if (storeKeyNull(key)) {
 372             debugs(47, 1, "commonUfsDirRebuildFromDirectory: NULL key");
 373             sd->unlinkFile(filn);
 374             continue;
 375         }
 376
 377         tmpe.key = key;
 378         /* check sizes */
 379
 380         if (tmpe.swap_file_sz == 0) {
 381             tmpe.swap_file_sz = (uint64_t) sb.st_size;
 382         } else if (tmpe.swap_file_sz == (uint64_t)(sb.st_size - swap_hdr_len)) {
 383             tmpe.swap_file_sz = (uint64_t) sb.st_size;
 384         } else if (tmpe.swap_file_sz != (uint64_t)sb.st_size) {
 385             debugs(47, 1, "commonUfsDirRebuildFromDirectory: SIZE MISMATCH " <<
 386                    tmpe.swap_file_sz << "!=" <<
 387                    sb.st_size);
 388
 389             sd->unlinkFile(filn);
 390             continue;
 391         }
 392
 393         if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
 394             sd->unlinkFile(filn);
 395             counts.badflags++;
 396             continue;
 397         }
 398
 399         /* this needs to become
 400          * 1) unpack url
 401          * 2) make synthetic request with headers ?? or otherwise search
 402          * for a matching object in the store
 403          * TODO FIXME change to new async api
 404          * TODO FIXME I think there is a race condition here with the
 405          * async api :
 406          * store A reads in object foo, searchs for it, and finds nothing.
 407          * store B reads in object foo, searchs for it, finds nothing.
 408          * store A gets called back with nothing, so registers the object
 409          * store B gets called back with nothing, so registers the object,
 410          * which will conflict when the in core index gets around to scanning
 411          * store B.
 412          *
 413          * this suggests that rather than searching for duplicates, the
 414          * index rebuild should just assume its the most recent accurate
 415          * store entry and whoever indexes the stores handles duplicates.
 416          */
 417         e = Store::Root().get(key);
 418
 419         if (e && e->lastref >= tmpe.lastref) {
 420             /* key already exists, current entry is newer */
 421             /* keep old, ignore new */
 422             counts.dupcount++;
 423             continue;
 424         } else if (NULL != e) {
 425             /* URL already exists, this swapfile not being used */
 426             /* junk old, load new */
 427             e->release();       /* release old entry */
 428             counts.dupcount++;
 429         }
 430
 431         counts.objcount++;
 432         // tmpe.dump(5);
 433         currentEntry(sd->addDiskRestore(key,
 434                                         filn,
 435                                         tmpe.swap_file_sz,
 436                                         tmpe.expires,
 437                                         tmpe.timestamp,
 438                                         tmpe.lastref,
 439                                         tmpe.lastmod,
 440                                         tmpe.refcount,  /* refcount */
 441                                         tmpe.flags,             /* flags */
 442                                         (int) flags.clean));
 443         storeDirSwapLog(currentEntry(), SWAP_LOG_ADD);
 444     }
 445
 446 }
 447
 448 StoreEntry *
 449 RebuildState::currentEntry() const
 450 {
 451     return e;
 452 }
 453
 454 void
 455 RebuildState::currentEntry(StoreEntry *newValue)
 456 {
 457     e = newValue;
 458 }
 459
 460 void
 461 RebuildState::rebuildFromSwapLog()
 462 {
 463     currentEntry (NULL);
 464     double x;
 465     /* load a number of objects per invocation */
 466
 467     for (int count = 0; count < speed; count++) {
 468         StoreSwapLogData swapData;
 469
 470         if (LogParser->ReadRecord(swapData) != 1) {
 471             debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)");
 472             LogParser->Close();
 473             delete LogParser;
 474             LogParser = NULL;
 475             _done = true;
 476             return;
 477         }
 478
 479         n_read++;
 480
 481         if (swapData.op <= SWAP_LOG_NOP)
 482             continue;
 483
 484         if (swapData.op >= SWAP_LOG_MAX)
 485             continue;
 486
 487         /*
 488          * BC: during 2.4 development, we changed the way swap file
 489          * numbers are assigned and stored.  The high 16 bits used
 490          * to encode the SD index number.  There used to be a call
 491          * to storeDirProperFileno here that re-assigned the index
 492          * bits.  Now, for backwards compatibility, we just need
 493          * to mask it off.
 494          */
 495         swapData.swap_filen &= 0x00FFFFFF;
 496
 497         debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " <<
 498                swap_log_op_str[(int) swapData.op]  << " " <<
 499                storeKeyText(swapData.key)  << " "<< std::setfill('0') <<
 500                std::hex << std::uppercase << std::setw(8) <<
 501                swapData.swap_filen);
 502
 503         if (swapData.op == SWAP_LOG_ADD) {
 504             (void) 0;
 505         } else if (swapData.op == SWAP_LOG_DEL) {
 506             /* Delete unless we already have a newer copy anywhere in any store */
 507             /* this needs to become
 508              * 1) unpack url
 509              * 2) make synthetic request with headers ?? or otherwise search
 510              * for a matching object in the store
 511              * TODO FIXME change to new async api
 512              */
 513             currentEntry (Store::Root().get(swapData.key));
 514
 515             if (currentEntry() != NULL && swapData.lastref >= e->lastref) {
 516                 /*
 517                  * Make sure we don't unlink the file, it might be
 518                  * in use by a subsequent entry.  Also note that
 519                  * we don't have to subtract from store_swap_size
 520                  * because adding to store_swap_size happens in
 521                  * the cleanup procedure.
 522                  */
 523                 currentEntry()->expireNow();
 524                 currentEntry()->releaseRequest();
 525
 526                 if (currentEntry()->swap_filen > -1) {
 527                     UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn));
 528                     assert (sdForThisEntry);
 529                     sdForThisEntry->replacementRemove(currentEntry());
 530                     sdForThisEntry->mapBitReset(currentEntry()->swap_filen);
 531                     currentEntry()->swap_filen = -1;
 532                     currentEntry()->swap_dirn = -1;
 533                 }
 534
 535                 currentEntry()->release();
 536                 counts.objcount--;
 537                 counts.cancelcount++;
 538             }
 539
 540             continue;
 541         } else {
 542             x = ::log(static_cast<double>(++counts.bad_log_op)) / ::log(10.0);
 543
 544             if (0.0 == x - (double) (int) x)
 545                 debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found");
 546
 547             counts.invalid++;
 548
 549             continue;
 550         }
 551
 552         if ((++counts.scancount & 0xFFF) == 0) {
 553
 554             int swap_entries = LogParser->SwapLogEntries();
 555
 556             if (0 != swap_entries )
 557                 storeRebuildProgress(sd->index,
 558                                      swap_entries, n_read);
 559         }
 560
 561         if (!sd->validFileno(swapData.swap_filen, 0)) {
 562             counts.invalid++;
 563             continue;
 564         }
 565
 566         if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) {
 567             counts.badflags++;
 568             continue;
 569         }
 570
 571         /* this needs to become
 572          * 1) unpack url
 573          * 2) make synthetic request with headers ?? or otherwise search
 574          * for a matching object in the store
 575          * TODO FIXME change to new async api
 576          */
 577         currentEntry (Store::Root().get(swapData.key));
 578
 579         int used;                       /* is swapfile already in use? */
 580
 581         used = sd->mapBitTest(swapData.swap_filen);
 582
 583         /* If this URL already exists in the cache, does the swap log
 584          * appear to have a newer entry?  Compare 'lastref' from the
 585          * swap log to e->lastref. */
 586         /* is the log entry newer than current entry? */
 587         int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0;
 588
 589         if (used && !disk_entry_newer) {
 590             /* log entry is old, ignore it */
 591             counts.clashcount++;
 592             continue;
 593         } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) {
 594             /* swapfile taken, same URL, newer, update meta */
 595
 596             if (currentEntry()->store_status == STORE_OK) {
 597                 currentEntry()->lastref = swapData.timestamp;
 598                 currentEntry()->timestamp = swapData.timestamp;
 599                 currentEntry()->expires = swapData.expires;
 600                 currentEntry()->lastmod = swapData.lastmod;
 601                 currentEntry()->flags = swapData.flags;
 602                 currentEntry()->refcount += swapData.refcount;
 603                 sd->dereference(*currentEntry());
 604             } else {
 605                 debug_trap("commonUfsDirRebuildFromSwapLog: bad condition");
 606                 debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__);
 607             }
 608
 609             continue;
 610         } else if (used) {
 611             /* swapfile in use, not by this URL, log entry is newer */
 612             /* This is sorta bad: the log entry should NOT be newer at this
 613              * point.  If the log is dirty, the filesize check should have
 614              * caught this.  If the log is clean, there should never be a
 615              * newer entry. */
 616              debugs(47, 1, "WARNING: newer swaplog entry for dirno " <<
 617                     sd->index  << ", fileno "<< std::setfill('0') << std::hex <<
 618                     std::uppercase << std::setw(8) << swapData.swap_filen);
 619
 620             /* I'm tempted to remove the swapfile here just to be safe,
 621              * but there is a bad race condition in the NOVM version if
 622              * the swapfile has recently been opened for writing, but
 623              * not yet opened for reading.  Because we can't map
 624              * swapfiles back to StoreEntrys, we don't know the state
 625              * of the entry using that file.  */
 626             /* We'll assume the existing entry is valid, probably because
 627              * were in a slow rebuild and the the swap file number got taken
 628              * and the validation procedure hasn't run. */
 629             assert(flags.need_to_validate);
 630             counts.clashcount++;
 631             continue;
 632         } else if (currentEntry() && !disk_entry_newer) {
 633             /* key already exists, current entry is newer */
 634             /* keep old, ignore new */
 635             counts.dupcount++;
 636             continue;
 637         } else if (currentEntry()) {
 638             /* key already exists, this swapfile not being used */
 639             /* junk old, load new */
 640             currentEntry()->expireNow();
 641             currentEntry()->releaseRequest();
 642
 643             if (currentEntry()->swap_filen > -1) {
 644                 UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn));
 645                 sdForThisEntry->replacementRemove(currentEntry());
 646                 /* Make sure we don't actually unlink the file */
 647                 sdForThisEntry->mapBitReset(currentEntry()->swap_filen);
 648                 currentEntry()->swap_filen = -1;
 649                 currentEntry()->swap_dirn = -1;
 650             }
 651
 652             currentEntry()->release();
 653             counts.dupcount++;
 654         } else {
 655             /* URL doesnt exist, swapfile not in use */
 656             /* load new */
 657             (void) 0;
 658         }
 659
 660         /* update store_swap_size */
 661         counts.objcount++;
 662
 663         currentEntry(sd->addDiskRestore(swapData.key,
 664                                         swapData.swap_filen,
 665                                         swapData.swap_file_sz,
 666                                         swapData.expires,
 667                                         swapData.timestamp,
 668                                         swapData.lastref,
 669                                         swapData.lastmod,
 670                                         swapData.refcount,
 671                                         swapData.flags,
 672                                         (int) flags.clean));
 673
 674         storeDirSwapLog(currentEntry(), SWAP_LOG_ADD);
 675     }
 676
 677 }
 678
 679 int
 680 RebuildState::getNextFile(sfileno * filn_p, int *size)
 681 {
 682     int fd = -1;
 683     int dirs_opened = 0;
 684     debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init  << ", " <<
 685            sd->index  << ": /"<< std::setfill('0') << std::hex <<
 686            std::uppercase << std::setw(2) << curlvl1  << "/" << std::setw(2) <<
 687            curlvl2);
 688
 689     if (done)
 690         return -2;
 691
 692     while (fd < 0 && done == 0) {
 693         fd = -1;
 694
 695         if (0 == flags.init) {  /* initialize, open first file */
 696             done = 0;
 697             curlvl1 = 0;
 698             curlvl2 = 0;
 699             in_dir = 0;
 700             flags.init = 1;
 701             assert(Config.cacheSwap.n_configured > 0);
 702         }
 703
 704         if (0 == in_dir) {      /* we need to read in a new directory */
 705             snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
 706                      sd->path,
 707                      curlvl1, curlvl2);
 708
 709             if (dirs_opened)
 710                 return -1;
 711
 712             td = opendir(fullpath);
 713
 714             dirs_opened++;
 715
 716             if (td == NULL) {
 717                 debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror());
 718             } else {
 719                 entry = readdir(td);    /* skip . and .. */
 720                 entry = readdir(td);
 721
 722                 if (entry == NULL && errno == ENOENT)
 723                 debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!.");
 724                 debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath);
 725             }
 726         }
 727
 728         if (td != NULL && (entry = readdir(td)) != NULL) {
 729             in_dir++;
 730
 731             if (sscanf(entry->d_name, "%x", &fn) != 1) {
 732                 debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name);
 733                 continue;
 734             }
 735
 736             if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) {
 737                 debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') <<
 738                        std::hex << std::uppercase << std::setw(8) << fn  <<
 739                        " does not belong in " << std::dec << sd->index  << "/" <<
 740                        curlvl1  << "/" << curlvl2);
 741
 742                 continue;
 743             }
 744
 745             if (sd->mapBitTest(fn)) {
 746                 debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next.");
 747                 continue;
 748             }
 749
 750             snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s",
 751                      fullpath, entry->d_name);
 752             debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename);
 753             fd = file_open(fullfilename, O_RDONLY | O_BINARY);
 754
 755             if (fd < 0)
 756                 debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror());
 757             else
 758                 store_open_disk_fd++;
 759
 760             continue;
 761         }
 762
 763         if (td != NULL)
 764             closedir(td);
 765
 766         td = NULL;
 767
 768         in_dir = 0;
 769
 770         if (sd->validL2(++curlvl2))
 771             continue;
 772
 773         curlvl2 = 0;
 774
 775         if (sd->validL1(++curlvl1))
 776             continue;
 777
 778         curlvl1 = 0;
 779
 780         done = 1;
 781     }
 782
 783     *filn_p = fn;
 784     return fd;
 785 }
 786
 787 void
 788 RebuildState::next(void (callback)(void *cbdata), void *cbdata)
 789 {
 790     /* for now, we don't cache at all */
 791     speed = 1;
 792     currentEntry(NULL);
 793
 794     while (!isDone() && currentEntry() == NULL)
 795         rebuildStep();
 796
 797     callback(cbdata);
 798 }
 799
 800 bool
 801 RebuildState::next()
 802 {
 803     return false;
 804 }
 805
 806 bool
 807 RebuildState::error() const
 808 {
 809     return false;
 810 }
 811
 812 bool
 813 RebuildState::isDone() const
 814 {
 815     return _done;
 816 }
 817
 818 StoreEntry *
 819 RebuildState::currentItem()
 820 {
 821     return currentEntry();
 822 }
 823
 824 #ifndef _USE_INLINE_
 825 #include "ufscommon.cci"
 826 #endif