src/fs/rock/RockRebuild.cc

   1 /*
   2  * DEBUG: section 79    Disk IO Routines
   3  */
   4
   5 #include "squid.h"
   6 #include "disk.h"
   7 #include "fs/rock/RockDbCell.h"
   8 #include "fs/rock/RockRebuild.h"
   9 #include "fs/rock/RockSwapDir.h"
  10 #include "globals.h"
  11 #include "md5.h"
  12 #include "SquidTime.h"
  13 #include "store_rebuild.h"
  14 #include "tools.h"
  15 #include "typedefs.h"
  16
  17 #if HAVE_ERRNO_H
  18 #include <errno.h>
  19 #endif
  20
  21 CBDATA_NAMESPACED_CLASS_INIT(Rock, Rebuild);
  22
  23 Rock::Rebuild::Rebuild(SwapDir *dir): AsyncJob("Rock::Rebuild"),
  24         sd(dir),
  25         dbSize(0),
  26         dbEntrySize(0),
  27         dbEntryLimit(0),
  28         fd(-1),
  29         dbOffset(0),
  30         filen(0)
  31 {
  32     assert(sd);
  33     memset(&counts, 0, sizeof(counts));
  34     dbSize = sd->diskOffsetLimit(); // we do not care about the trailer waste
  35     dbEntrySize = sd->max_objsize;
  36     dbEntryLimit = sd->entryLimit();
  37 }
  38
  39 Rock::Rebuild::~Rebuild()
  40 {
  41     if (fd >= 0)
  42         file_close(fd);
  43 }
  44
  45 /// prepares and initiates entry loading sequence
  46 void
  47 Rock::Rebuild::start()
  48 {
  49     // in SMP mode, only the disker is responsible for populating the map
  50     if (UsingSmp() && !IamDiskProcess()) {
  51         debugs(47, 2, "Non-disker skips rebuilding of cache_dir #" <<
  52                sd->index << " from " << sd->filePath);
  53         mustStop("non-disker");
  54         return;
  55     }
  56
  57     debugs(47, DBG_IMPORTANT, "Loading cache_dir #" << sd->index <<
  58            " from " << sd->filePath);
  59
  60     fd = file_open(sd->filePath, O_RDONLY | O_BINARY);
  61     if (fd < 0)
  62         failure("cannot open db", errno);
  63
  64     char buf[SwapDir::HeaderSize];
  65     if (read(fd, buf, sizeof(buf)) != SwapDir::HeaderSize)
  66         failure("cannot read db header", errno);
  67
  68     dbOffset = SwapDir::HeaderSize;
  69     filen = 0;
  70
  71     checkpoint();
  72 }
  73
  74 /// continues after a pause if not done
  75 void
  76 Rock::Rebuild::checkpoint()
  77 {
  78     if (!done())
  79         eventAdd("Rock::Rebuild", Rock::Rebuild::Steps, this, 0.01, 1, true);
  80 }
  81
  82 bool
  83 Rock::Rebuild::doneAll() const
  84 {
  85     return dbOffset >= dbSize && AsyncJob::doneAll();
  86 }
  87
  88 void
  89 Rock::Rebuild::Steps(void *data)
  90 {
  91     // use async call to enable job call protection that time events lack
  92     CallJobHere(47, 5, static_cast<Rebuild*>(data), Rock::Rebuild, steps);
  93 }
  94
  95 void
  96 Rock::Rebuild::steps()
  97 {
  98     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
  99            dbOffset << " <= " << dbSize);
 100
 101     // Balance our desire to maximize the number of entries processed at once
 102     // (and, hence, minimize overheads and total rebuild time) with a
 103     // requirement to also process Coordinator events, disk I/Os, etc.
 104     const int maxSpentMsec = 50; // keep small: most RAM I/Os are under 1ms
 105     const timeval loopStart = current_time;
 106
 107     int loaded = 0;
 108     while (loaded < dbEntryLimit && dbOffset < dbSize) {
 109         doOneEntry();
 110         dbOffset += dbEntrySize;
 111         ++filen;
 112         ++loaded;
 113
 114         if (counts.scancount % 1000 == 0)
 115             storeRebuildProgress(sd->index, dbEntryLimit, counts.scancount);
 116
 117         if (opt_foreground_rebuild)
 118             continue; // skip "few entries at a time" check below
 119
 120         getCurrentTime();
 121         const double elapsedMsec = tvSubMsec(loopStart, current_time);
 122         if (elapsedMsec > maxSpentMsec || elapsedMsec < 0) {
 123             debugs(47, 5, HERE << "pausing after " << loaded << " entries in " <<
 124                    elapsedMsec << "ms; " << (elapsedMsec/loaded) << "ms per entry");
 125             break;
 126         }
 127     }
 128
 129     checkpoint();
 130 }
 131
 132 void
 133 Rock::Rebuild::doOneEntry()
 134 {
 135     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
 136            dbOffset << " <= " << dbSize);
 137
 138     ++counts.scancount;
 139
 140     if (lseek(fd, dbOffset, SEEK_SET) < 0)
 141         failure("cannot seek to db entry", errno);
 142
 143     MemBuf buf;
 144     buf.init(SM_PAGE_SIZE, SM_PAGE_SIZE);
 145
 146     if (!storeRebuildLoadEntry(fd, sd->index, buf, counts))
 147         return;
 148
 149     // get our header
 150     DbCellHeader header;
 151     if (buf.contentSize() < static_cast<mb_size_t>(sizeof(header))) {
 152         debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << sd->index << "]: " <<
 153                "Ignoring truncated cache entry meta data at " << dbOffset);
 154         ++counts.invalid;
 155         return;
 156     }
 157     memcpy(&header, buf.content(), sizeof(header));
 158
 159     if (!header.sane()) {
 160         debugs(47, DBG_IMPORTANT, "WARNING: cache_dir[" << sd->index << "]: " <<
 161                "Ignoring malformed cache entry meta data at " << dbOffset);
 162         ++counts.invalid;
 163         return;
 164     }
 165     buf.consume(sizeof(header)); // optimize to avoid memmove()
 166
 167     cache_key key[SQUID_MD5_DIGEST_LENGTH];
 168     StoreEntry loadedE;
 169     if (!storeRebuildParseEntry(buf, loadedE, key, counts, header.payloadSize)) {
 170         // skip empty slots
 171         if (loadedE.swap_filen > 0 || loadedE.swap_file_sz > 0) {
 172             ++counts.invalid;
 173             //sd->unlink(filen); leave garbage on disk, it should not hurt
 174         }
 175         return;
 176     }
 177
 178     assert(loadedE.swap_filen < dbEntryLimit);
 179     if (!storeRebuildKeepEntry(loadedE, key, counts))
 180         return;
 181
 182     ++counts.objcount;
 183     // loadedE->dump(5);
 184
 185     sd->addEntry(filen, header, loadedE);
 186 }
 187
 188 void
 189 Rock::Rebuild::swanSong()
 190 {
 191     debugs(47,3, HERE << "cache_dir #" << sd->index << " rebuild level: " <<
 192            StoreController::store_dirs_rebuilding);
 193     --StoreController::store_dirs_rebuilding;
 194     storeRebuildComplete(&counts);
 195 }
 196
 197 void
 198 Rock::Rebuild::failure(const char *msg, int errNo)
 199 {
 200     debugs(47,5, HERE << sd->index << " filen " << filen << " at " <<
 201            dbOffset << " <= " << dbSize);
 202
 203     if (errNo)
 204         debugs(47, DBG_CRITICAL, "ERROR: Rock cache_dir rebuild failure: " << xstrerr(errNo));
 205     debugs(47, DBG_CRITICAL, "Do you need to run 'squid -z' to initialize storage?");
 206
 207     assert(sd);
 208     fatalf("Rock cache_dir[%d] rebuild of %s failed: %s.",
 209            sd->index, sd->filePath, msg);
 210 }