src/fs/rock/RockRebuild.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 79    Disk IO Routines
   5  */
   6
   7 #include "config.h"
   8 #include "fs/rock/RockRebuild.h"
   9 #include "fs/rock/RockSwapDir.h"
  10 #include "fs/rock/RockDbCell.h"
  11 #include "SquidTime.h"
  12
  13 CBDATA_NAMESPACED_CLASS_INIT(Rock, Rebuild);
  14
  15 Rock::Rebuild::Rebuild(SwapDir *dir): AsyncJob("Rock::Rebuild"),
  16         sd(dir),
  17         dbSize(0),
  18         dbEntrySize(0),
  19         dbEntryLimit(0),
  20         fd(-1),
  21         dbOffset(0),
  22         filen(0)
  23 {
  24     assert(sd);
  25     memset(&counts, 0, sizeof(counts));
  26     dbSize = sd->diskOffsetLimit(); // we do not care about the trailer waste
  27     dbEntrySize = sd->max_objsize;
  28     dbEntryLimit = sd->entryLimit();
  29 }
  30
  31 Rock::Rebuild::~Rebuild()
  32 {
  33     if (fd >= 0)
  34         file_close(fd);
  35 }
  36
  37 /// prepares and initiates entry loading sequence
  38 void
  39 Rock::Rebuild::start()
  40 {
  41     // in SMP mode, only the disker is responsible for populating the map
  42     if (UsingSmp() && !IamDiskProcess()) {
  43         debugs(47, 2, "Non-disker skips rebuilding of cache_dir #" <<
  44                sd->index << " from " << sd->filePath);
  45         mustStop("non-disker");
  46         return;
  47     }
  48
  49     debugs(47, DBG_IMPORTANT, "Loading cache_dir #" << sd->index <<
  50            " from " << sd->filePath);
  51
  52     fd = file_open(sd->filePath, O_RDONLY | O_BINARY);
  53     if (fd < 0)
  54         failure("cannot open db", errno);
  55
  56     char buf[SwapDir::HeaderSize];
  57     if (read(fd, buf, sizeof(buf)) != SwapDir::HeaderSize)
  58         failure("cannot read db header", errno);
  59
  60     dbOffset = SwapDir::HeaderSize;
  61     filen = 0;
  62
  63     checkpoint();
  64 }
  65
  66 /// continues after a pause if not done
  67 void
  68 Rock::Rebuild::checkpoint()
  69 {
  70     if (!done())
  71         eventAdd("Rock::Rebuild", Rock::Rebuild::Steps, this, 0.01, 1, true);
  72 }
  73
  74 bool
  75 Rock::Rebuild::doneAll() const
  76 {
  77     return dbOffset >= dbSize && AsyncJob::doneAll();
  78 }
  79
  80 void
  81 Rock::Rebuild::Steps(void *data)
  82 {
  83     // use async call to enable job call protection that time events lack
  84     CallJobHere(47, 5, static_cast<Rebuild*>(data), Rock::Rebuild, steps);
  85 }
  86
  87 void
  88 Rock::Rebuild::steps()
  89 {
  90     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
  91            dbOffset << " <= " << dbSize);
  92
  93     // Balance our desire to maximize the number of entries processed at once
  94     // (and, hence, minimize overheads and total rebuild time) with a
  95     // requirement to also process Coordinator events, disk I/Os, etc.
  96     const int maxSpentMsec = 50; // keep small: most RAM I/Os are under 1ms
  97     const timeval loopStart = current_time;
  98
  99     int loaded = 0;
 100     while (loaded < dbEntryLimit && dbOffset < dbSize) {
 101         doOneEntry();
 102         dbOffset += dbEntrySize;
 103         ++filen;
 104         ++loaded;
 105
 106         if (counts.scancount % 1000 == 0)
 107             storeRebuildProgress(sd->index, dbEntryLimit, counts.scancount);
 108
 109         if (opt_foreground_rebuild)
 110             continue; // skip "few entries at a time" check below
 111
 112         getCurrentTime();
 113         const double elapsedMsec = tvSubMsec(loopStart, current_time);
 114         if (elapsedMsec > maxSpentMsec || elapsedMsec < 0) {
 115             debugs(47, 5, HERE << "pausing after " << loaded << " entries in " <<
 116                    elapsedMsec << "ms; " << (elapsedMsec/loaded) << "ms per entry");
 117             break;
 118         }
 119     }
 120
 121     checkpoint();
 122 }
 123
 124 void
 125 Rock::Rebuild::doOneEntry()
 126 {
 127     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
 128            dbOffset << " <= " << dbSize);
 129
 130     ++counts.scancount;
 131
 132     if (lseek(fd, dbOffset, SEEK_SET) < 0)
 133         failure("cannot seek to db entry", errno);
 134
 135     MemBuf buf;
 136     buf.init(SM_PAGE_SIZE, SM_PAGE_SIZE);
 137
 138     if (!storeRebuildLoadEntry(fd, sd->index, buf, counts))
 139         return;
 140
 141     // get our header
 142     DbCellHeader header;
 143     if (buf.contentSize() < static_cast<mb_size_t>(sizeof(header))) {
 144         debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << sd->index << "]: " <<
 145                "Ignoring truncated cache entry meta data at " << dbOffset);
 146         counts.invalid++;
 147         return;
 148     }
 149     memcpy(&header, buf.content(), sizeof(header));
 150
 151     if (!header.sane()) {
 152         debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << sd->index << "]: " <<
 153                "Ignoring malformed cache entry meta data at " << dbOffset);
 154         counts.invalid++;
 155         return;
 156     }
 157     buf.consume(sizeof(header)); // optimize to avoid memmove()
 158
 159     cache_key key[SQUID_MD5_DIGEST_LENGTH];
 160     StoreEntry loadedE;
 161     if (!storeRebuildParseEntry(buf, loadedE, key, counts, header.payloadSize)) {
 162         // skip empty slots
 163         if (loadedE.swap_filen > 0 || loadedE.swap_file_sz > 0) {
 164             counts.invalid++;
 165             //sd->unlink(filen); leave garbage on disk, it should not hurt
 166         }
 167         return;
 168     }
 169
 170     assert(loadedE.swap_filen < dbEntryLimit);
 171     if (!storeRebuildKeepEntry(loadedE, key, counts))
 172         return;
 173
 174     counts.objcount++;
 175     // loadedE->dump(5);
 176
 177     sd->addEntry(filen, header, loadedE);
 178 }
 179
 180 void
 181 Rock::Rebuild::swanSong()
 182 {
 183     debugs(47,3, HERE << "cache_dir #" << sd->index << " rebuild level: " <<
 184            StoreController::store_dirs_rebuilding);
 185     --StoreController::store_dirs_rebuilding;
 186     storeRebuildComplete(&counts);
 187 }
 188
 189 void
 190 Rock::Rebuild::failure(const char *msg, int errNo)
 191 {
 192     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
 193            dbOffset << " <= " << dbSize);
 194
 195     if (errNo)
 196         debugs(47, DBG_CRITICAL, "ERROR: Rock cache_dir rebuild failure: " << xstrerr(errNo));
 197     debugs(47, DBG_CRITICAL, "Do you need to run 'squid -z' to initialize storage?");
 198
 199     assert(sd);
 200     fatalf("Rock cache_dir[%d] rebuild of %s failed: %s.",
 201            sd->index, sd->filePath, msg);
 202 }