From: Dmitry Kurochkin Date: Mon, 31 Jan 2011 04:08:41 +0000 (+0300) Subject: Shared Rock::DirMap version 5. X-Git-Tag: take01~14 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=f1eaa2540453dccde688d900e75f900c67f0dd7e;p=thirdparty%2Fsquid.git Shared Rock::DirMap version 5. --- diff --git a/src/fs/rock/RockDirMap.cc b/src/fs/rock/RockDirMap.cc index d6377671d5..e4b5e86bc9 100644 --- a/src/fs/rock/RockDirMap.cc +++ b/src/fs/rock/RockDirMap.cc @@ -5,20 +5,22 @@ */ #include "squid.h" + +#include "Store.h" #include "fs/rock/RockDirMap.h" static const char SharedMemoryName[] = "RockDirMap"; -Rock::DirMap::DirMap(const int id, const int limit): - shm(SharedMemoryName, id) +Rock::DirMap::DirMap(const char *const path, const int limit): + shm(SharedMemoryName(path)) { shm.create(limit); assert(shm.mem()); shared = new (shm.mem()) Shared(limit); } -Rock::DirMap::DirMap(const int id): - shm(SharedMemoryName, id) +Rock::DirMap::DirMap(const char *const path): + shm(SharedMemoryName(path)) { shm.open(); assert(shm.mem()); @@ -26,19 +28,11 @@ Rock::DirMap::DirMap(const int id): } StoreEntryBasics * -Rock::DirMap::add(const cache_key *const key) -{ - return add(key, slotIdx(key)); -} - -StoreEntryBasics * -Rock::DirMap::add(const cache_key *const key, const sfileno fileno) +Rock::DirMap::openForWriting(const cache_key *const key, sfileno &fileno) { - if (fileno != slotIdx(key)) - return 0; - - free(fileno); - Slot &s = shared->slots[fileno]; + const int idx = slotIdx(key); + free(idx); + Slot &s = shared->slots[idx]; if (s.state.swap_if(Slot::Empty, Slot::Writing)) { s.setKey(key); return &s.seBasics; @@ -47,25 +41,19 @@ Rock::DirMap::add(const cache_key *const key, const sfileno fileno) } void -Rock::DirMap::added(const cache_key *const key) +Rock::DirMap::closeForWriting(const sfileno fileno) { - Slot &s = slot(key); - assert(s.checkKey(key)); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; assert(s.state == Slot::Writing); ++shared->count; assert(s.state.swap_if(Slot::Writing, Slot::Usable)); } bool -Rock::DirMap::free(const cache_key *const key) -{ - return free(slotIdx(key)); -} - -bool -Rock::DirMap::free(const int fileno) +Rock::DirMap::free(const sfileno fileno) { - if (open(fileno)) { + if (openForReadingAt(fileno)) { Slot &s = shared->slots[fileno]; s.state.swap_if(Slot::Usable, Slot::WaitingToBeFreed); --s.readLevel; @@ -76,8 +64,21 @@ Rock::DirMap::free(const int fileno) } const StoreEntryBasics * -Rock::DirMap::open(const sfileno fileno) +Rock::DirMap::openForReading(const cache_key *const key, sfileno &fileno) +{ + const int idx = slotIdx(key); + const StoreEntryBasics *const seBasics = openForReadingAt(idx); + if (seBasics && shared->slots[idx].checkKey(key)) { + fileno = idx; + return seBasics; + } + return 0; +} + +const StoreEntryBasics * +Rock::DirMap::openForReadingAt(const sfileno fileno) { + assert(valid(fileno)); Slot &s = shared->slots[fileno]; ++s.readLevel; if (s.state == Slot::Usable) @@ -87,23 +88,11 @@ Rock::DirMap::open(const sfileno fileno) return 0; } -const StoreEntryBasics * -Rock::DirMap::open(const cache_key *const key, sfileno &fileno) -{ - const int idx = slotIdx(key); - const StoreEntryBasics *const seBasics = open(idx); - if (seBasics && shared->slots[fileno].checkKey(key)) { - fileno = idx; - return seBasics; - } - return 0; -} - void -Rock::DirMap::close(const cache_key *const key) +Rock::DirMap::closeForReading(const sfileno fileno) { - Slot &s = slot(key); - assert(s.checkKey(key)); + assert(valid(fileno)); + Slot &s = shared->slots[fileno]; assert(s.readLevel > 0); --s.readLevel; freeIfNeeded(s); @@ -175,6 +164,21 @@ Rock::DirMap::SharedSize(const int limit) return sizeof(Shared) + limit * sizeof(Slot); } +String +Rock::DirMap::SharedMemoryName(const char *path) +{ + String result; + for (const char *p = strchr(path, '/'); p; p = strchr(path, '/')) { + if (path != p) { + result.append('.'); + result.append(path, p - path); + } + path = p + 1; + } + result.append(path); + return result; +} + void Rock::DirMap::Slot::setKey(const cache_key *const aKey) { @@ -191,3 +195,16 @@ Rock::DirMap::Slot::checkKey(const cache_key *const aKey) const Rock::DirMap::Shared::Shared(const int aLimit): limit(aLimit), count(0) { } + +void +StoreEntryBasics::set(const StoreEntry &from) +{ + memset(this, 0, sizeof(*this)); + timestamp = from.timestamp; + lastref = from.lastref; + expires = from.expires; + lastmod = from.lastmod; + swap_file_sz = from.swap_file_sz; + refcount = from.refcount; + flags = from.flags; +} diff --git a/src/fs/rock/RockDirMap.h b/src/fs/rock/RockDirMap.h index f9ac56f14a..15a7f069b7 100644 --- a/src/fs/rock/RockDirMap.h +++ b/src/fs/rock/RockDirMap.h @@ -6,6 +6,8 @@ class StoreEntryBasics { public: + void set(const StoreEntry &from); + /* START OF ON-DISK STORE_META_STD TLV field */ time_t timestamp; time_t lastref; @@ -24,23 +26,23 @@ namespace Rock { class DirMap { public: - DirMap(const int id, const int limit); ///< create a new shared DirMap - DirMap(const int id); ///< open an existing shared DirMap + DirMap(const char *const path, const int limit); ///< create a new shared DirMap + DirMap(const char *const path); ///< open an existing shared DirMap - /// start adding a new entry - StoreEntryBasics *add(const cache_key *const key); - /// start adding a new entry, with fileno check - StoreEntryBasics *add(const cache_key *const key, const sfileno fileno); - /// finish adding a new entry - void added(const cache_key *const key); + /// start writing a new entry + StoreEntryBasics *openForWriting(const cache_key *const key, sfileno &fileno); + /// finish writing a new entry + void closeForWriting(const sfileno fileno); /// mark slot as waiting to be freed, will be freed when no one uses it - bool free(const cache_key *const key); + bool free(const sfileno fileno); /// open slot for reading, increments read level - const StoreEntryBasics *open(const cache_key *const key, sfileno &fileno); + const StoreEntryBasics *openForReading(const cache_key *const key, sfileno &fileno); + /// open slot for reading, increments read level + const StoreEntryBasics *openForReadingAt(const sfileno fileno); /// close slot after reading, decrements read level - void close(const cache_key *const key); + void closeForReading(const sfileno fileno); bool full() const; ///< there are no empty slots left bool valid(int n) const; ///< whether n is a valid slot coordinate @@ -79,11 +81,11 @@ private: int slotIdx(const cache_key *const key) const; Slot &slot(const cache_key *const key); - bool free(const sfileno fileno); - const StoreEntryBasics *open(const sfileno fileno); + const StoreEntryBasics *openForReading(Slot &s); void freeIfNeeded(Slot &s); static int SharedSize(const int limit); + static String SharedMemoryName(const char *path); SharedMemory shm; ///< shared memory segment Shared *shared; ///< pointer to shared memory diff --git a/src/fs/rock/RockSwapDir.cc b/src/fs/rock/RockSwapDir.cc index 7820c6131a..413579a0da 100644 --- a/src/fs/rock/RockSwapDir.cc +++ b/src/fs/rock/RockSwapDir.cc @@ -20,13 +20,14 @@ // must be divisible by 1024 due to cur_size and max_size KB madness const int64_t Rock::SwapDir::HeaderSize = 16*1024; -Rock::SwapDir::SwapDir(): ::SwapDir("rock"), filePath(NULL), io(NULL) +Rock::SwapDir::SwapDir(): ::SwapDir("rock"), filePath(NULL), io(NULL), map(NULL) { } Rock::SwapDir::~SwapDir() { delete io; + delete map; safe_free(filePath); } @@ -41,7 +42,7 @@ StoreEntry * Rock::SwapDir::get(const cache_key *key) { sfileno fileno; - const StoreEntryBasics *const basics = map.open(key, fileno); + const StoreEntryBasics *const basics = map->openForReading(key, fileno); if (!basics) return NULL; @@ -173,6 +174,10 @@ Rock::SwapDir::parse(int anIndex, char *aPath) repl = createRemovalPolicy(Config.replPolicy); + // map size is set when shared memory segment is created + if (!map) + map = new DirMap(path); + validateOptions(); } @@ -210,12 +215,14 @@ Rock::SwapDir::validateOptions() if (ps > 0 && (max_objsize % ps != 0)) fatal("Rock store max-size should be a multiple of page size"); + /* const int64_t eLimitHi = 0xFFFFFF; // Core sfileno maximum const int64_t eLimitLo = map.entryLimit(); // dynamic shrinking unsupported const int64_t eWanted = (maximumSize() - HeaderSize)/max_objsize; const int64_t eAllowed = min(max(eLimitLo, eWanted), eLimitHi); map.resize(eAllowed); // the map may decide to use an even lower limit + */ // Note: We could try to shrink max_size now. It is stored in KB so we // may not be able to make it match the end of the last entry exactly. @@ -226,10 +233,10 @@ Rock::SwapDir::validateOptions() assert(diskOffsetLimit() <= maximumSize()); // warn if maximum db size is not reachable due to sfileno limit - if (map.entryLimit() == map.AbsoluteEntryLimit() && + if (map->entryLimit() == map->AbsoluteEntryLimit() && totalWaste > roundingWasteMx) { debugs(47, 0, "Rock store cache_dir[" << index << "]:"); - debugs(47, 0, "\tmaximum number of entries: " << map.entryLimit()); + debugs(47, 0, "\tmaximum number of entries: " << map->entryLimit()); debugs(47, 0, "\tmaximum entry size: " << max_objsize << " bytes"); debugs(47, 0, "\tmaximum db size: " << maximumSize() << " bytes"); debugs(47, 0, "\tusable db size: " << diskOffsetLimit() << " bytes"); @@ -243,7 +250,7 @@ Rock::SwapDir::validateOptions() // not fatal because it can be added later } - cur_size = (HeaderSize + max_objsize * map.entryCount()) >> 10; + cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; } void @@ -259,7 +266,7 @@ Rock::SwapDir::rebuild() { /* Add a new object to the cache with empty memory copy and pointer to disk * use to rebuild store from disk. XXX: dupes UFSSwapDir::addDiskRestore */ -void +bool Rock::SwapDir::addEntry(const int fileno, const StoreEntry &from) { const cache_key *const key = reinterpret_cast(from.key); @@ -267,21 +274,19 @@ Rock::SwapDir::addEntry(const int fileno, const StoreEntry &from) << ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << fileno); - StoreEntryBasics *const basics = map.add(key, fileno); - if (!basics) { - debugs(47, 5, HERE << "Rock::SwapDir::addEntry: map.add failed"); + int idx; + StoreEntryBasics *const basics = map->openForWriting(key, idx); + if (!basics || fileno != idx) { + debugs(47, 5, HERE << "Rock::SwapDir::addEntry: map->add failed"); + if (basics) { + map->closeForWriting(idx); + map->free(idx); + } + return false; } - - memset(basics, 0, sizeof(*basics)); - basics->timestamp = from.timestamp; - basics->lastref = from.lastref; - basics->expires = from.expires; - basics->lastmod = from.lastmod; - basics->swap_file_sz = from.swap_file_sz; - basics->refcount = from.refcount; - basics->flags = from.flags; - - map.added(key); + basics->set(from); + map->closeForWriting(fileno); + return true; } @@ -307,16 +312,19 @@ Rock::SwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB *cbFile, StoreI return NULL; } - if (full()) { - maintain(); - if (full()) // maintain() above warns when it fails - return NULL; + sfileno fileno; + StoreEntryBasics *const basics = + map->openForWriting(reinterpret_cast(e.key), fileno); + if (!basics) { + debugs(47, 5, HERE << "Rock::SwapDir::createStoreIO: map->add failed"); + return NULL; } + basics->set(e); IoState *sio = new IoState(this, &e, cbFile, cbIo, data); sio->swap_dirn = index; - sio->swap_filen = map.useNext(); + sio->swap_filen = fileno; sio->offset_ = diskOffset(sio->swap_filen); sio->entrySize = e.objectLen() + e.mem_obj->swap_hdr_sz; @@ -343,7 +351,7 @@ Rock::SwapDir::diskOffset(int filen) const int64_t Rock::SwapDir::diskOffsetLimit() const { - return diskOffset(map.entryLimit()); + return diskOffset(map->entryLimit()); } StoreIOState::Pointer @@ -354,6 +362,13 @@ Rock::SwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB *cbFile, StoreIOS return NULL; } + if (!map->openForReadingAt(e.swap_filen)) { + debugs(47,1, HERE << "bug: dir " << index << " lost fileno: " << + std::setfill('0') << std::hex << std::uppercase << std::setw(8) << + e.swap_filen); + return NULL; + } + IoState *sio = new IoState(this, &e, cbFile, cbIo, data); sio->swap_dirn = index; @@ -362,18 +377,11 @@ Rock::SwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB *cbFile, StoreIOS std::setfill('0') << std::hex << std::uppercase << std::setw(8) << sio->swap_filen); - assert(map.valid(sio->swap_filen)); + assert(map->valid(sio->swap_filen)); sio->offset_ = diskOffset(sio->swap_filen); sio->entrySize = e.swap_file_sz; assert(sio->entrySize <= max_objsize); - if (!map.has(sio->swap_filen)) { - debugs(47,1, HERE << "bug: dir " << index << " lost fileno: " << - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << - sio->swap_filen); - return NULL; - } - assert(sio->offset_ + sio->entrySize <= diskOffsetLimit()); sio->file(theFile); @@ -394,12 +402,12 @@ Rock::SwapDir::ioCompletedNotification() fatalf("Rock cache_dir failed to open db file: %s", filePath); } - cur_size = (HeaderSize + max_objsize * map.entryCount()) >> 10; + cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; // TODO: lower debugging level debugs(47,1, "Rock cache_dir[" << index << "] limits: " << std::setw(12) << maximumSize() << " disk bytes and " << - std::setw(7) << map.entryLimit() << " entries"); + std::setw(7) << map->entryLimit() << " entries"); } void @@ -414,6 +422,7 @@ Rock::SwapDir::readCompleted(const char *buf, int rlen, int errflag, RefCount< : ReadRequest *request = dynamic_cast(r.getRaw()); assert(request); IoState::Pointer sio = request->sio; + map->closeForReading(sio->swap_filen); // do not increment sio->offset_: callers always supply relative offset @@ -432,12 +441,13 @@ Rock::SwapDir::writeCompleted(int errflag, size_t rlen, RefCount< ::WriteRequest assert(request); assert(request->sio != NULL); IoState &sio = *request->sio; + map->closeForWriting(sio.swap_filen); if (errflag != DISK_OK) - map.clear(sio.swap_filen); // TODO: test by forcing failure + map->free(sio.swap_filen); // TODO: test by forcing failure // else sio.offset_ += rlen; // TODO: always compute cur_size based on map, do not store it - cur_size = (HeaderSize + max_objsize * map.entryCount()) >> 10; + cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; assert(sio.offset_ <= diskOffsetLimit()); // post-factum check sio.finishedWriting(errflag); @@ -446,7 +456,7 @@ Rock::SwapDir::writeCompleted(int errflag, size_t rlen, RefCount< ::WriteRequest bool Rock::SwapDir::full() const { - return map.full(); + return map->full(); } void @@ -454,7 +464,7 @@ Rock::SwapDir::updateSize(int64_t size, int sign) { // it is not clear what store_swap_size really is; TODO: move low-level // size maintenance to individual store dir types - cur_size = (HeaderSize + max_objsize * map.entryCount()) >> 10; + cur_size = (HeaderSize + max_objsize * map->entryCount()) >> 10; store_swap_size = cur_size; if (sign > 0) @@ -489,7 +499,7 @@ Rock::SwapDir::maintain() return; // no need (to find a victim) debugs(47,3, HERE << "cache_dir[" << index << "] state: " << - map.full() << ' ' << currentSize() << " < " << diskOffsetLimit()); + map->full() << ' ' << currentSize() << " < " << diskOffsetLimit()); // Hopefully, we find a removable entry much sooner (TODO: use time?) const int maxProbed = 10000; @@ -538,7 +548,7 @@ Rock::SwapDir::unlink(StoreEntry &e) { debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen); ignoreReferences(e); - map.free(e.key); + map->free(e.swap_filen); } void @@ -564,9 +574,9 @@ Rock::SwapDir::statfs(StoreEntry &e) const storeAppendPrintf(&e, "Current Size: %"PRIu64" KB %.2f%%\n", cur_size, 100.0 * cur_size / max_size); - storeAppendPrintf(&e, "Maximum entries: %9d\n", map.entryLimit()); + storeAppendPrintf(&e, "Maximum entries: %9d\n", map->entryLimit()); storeAppendPrintf(&e, "Current entries: %9d %.2f%%\n", - map.entryCount(), (100.0 * map.entryCount() / map.entryLimit())); + map->entryCount(), (100.0 * map->entryCount() / map->entryLimit())); storeAppendPrintf(&e, "Pending operations: %d out of %d\n", store_open_disk_fd, Config.max_open_disk_fds); diff --git a/src/fs/rock/RockSwapDir.h b/src/fs/rock/RockSwapDir.h index 7e92bf0064..6637169b71 100644 --- a/src/fs/rock/RockSwapDir.h +++ b/src/fs/rock/RockSwapDir.h @@ -53,7 +53,7 @@ protected: void rebuild(); ///< starts loading and validating stored entry metadata ///< used to add entries successfully loaded during rebuild - void addEntry(const int fileno, const StoreEntry &from); + bool addEntry(const int fileno, const StoreEntry &from); bool full() const; ///< no more entries can be stored without purging void trackReferences(StoreEntry &e); ///< add to replacement policy scope @@ -64,7 +64,7 @@ protected: int64_t maximumSize() const { return static_cast(max_size) << 10;} int64_t diskOffset(int filen) const; int64_t diskOffsetLimit() const; - int entryLimit() const { return map.entryLimit(); } + int entryLimit() const { return map->entryLimit(); } friend class Rebuild; const char *filePath; ///< location of cache storage file inside path/ @@ -72,7 +72,7 @@ protected: private: DiskIOStrategy *io; RefCount theFile; ///< cache storage for this cache_dir - DirMap map; + DirMap *map; static const int64_t HeaderSize; ///< on-disk db header size }; diff --git a/src/ipc/SharedMemory.cc b/src/ipc/SharedMemory.cc index 0bb0dad9ac..c4438a1704 100644 --- a/src/ipc/SharedMemory.cc +++ b/src/ipc/SharedMemory.cc @@ -16,8 +16,8 @@ #include #include -SharedMemory::SharedMemory(const String &id, const int magic): - theName(GenerateName(id, magic)), theFD(-1), theSize(-1), theMem(NULL) +SharedMemory::SharedMemory(const String &id): + theName(GenerateName(id)), theFD(-1), theSize(-1), theMem(NULL) { } @@ -35,7 +35,7 @@ SharedMemory::create(const int aSize) assert(aSize > 0); assert(theFD < 0); - theFD = shm_open(theName.termedBuf(), O_CREAT | O_EXCL | O_RDWR, + theFD = shm_open(theName.termedBuf(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); if (theFD < 0) { debugs(54, 5, "SharedMemory::create: shm_open: " << xstrerror()); @@ -60,7 +60,9 @@ SharedMemory::open() theFD = shm_open(theName.termedBuf(), O_RDWR, 0); if (theFD < 0) { debugs(54, 5, "SharedMemory::open: shm_open: " << xstrerror()); - fatal("SharedMemory::open failed"); + String s = "SharedMemory::open failed 1 "; + s.append(theName); + fatal(s.termedBuf()); } { @@ -68,7 +70,9 @@ SharedMemory::open() memset(&s, 0, sizeof(s)); if (fstat(theFD, &s)) { debugs(54, 5, "SharedMemory::open: fstat: " << xstrerror()); - fatal("SharedMemory::open failed"); + String s = "SharedMemory::open failed 2 "; + s.append(theName); + fatal(s.termedBuf()); } theSize = s.st_size; @@ -108,21 +112,11 @@ SharedMemory::detach() theMem = 0; } -/// Generate name for shared memory segment. Uses the master process -/// PID to avoid conflicts with other Squid instances. +/// Generate name for shared memory segment. String -SharedMemory::GenerateName(const String &id, const int magic) +SharedMemory::GenerateName(const String &id) { String name("/squid-"); name.append(id); - name.append('-'); - { - const int pid = IamMasterProcess() ? getpid() : getppid(); - name.append(pid); - } - if (magic) { - name.append('-'); - name.append(magic); - } return name; } diff --git a/src/ipc/SharedMemory.h b/src/ipc/SharedMemory.h index 3d39050ecb..8fd8d1c4ea 100644 --- a/src/ipc/SharedMemory.h +++ b/src/ipc/SharedMemory.h @@ -11,9 +11,8 @@ /// POSIX shared memory segment class SharedMemory { public: - /// Create a shared memory segment. Id is a human-readable name, - /// optional magic is unique key (e.g. kid id). - SharedMemory(const String &id, const int magic = 0); + /// Create a shared memory segment. + SharedMemory(const String &id); ~SharedMemory(); /// Create a new shared memory segment. Fails if a segment with @@ -29,7 +28,7 @@ private: void attach(); void detach(); - static String GenerateName(const String &id, const int magic); + static String GenerateName(const String &id); const String theName; ///< shared memory segment file name int theFD; ///< shared memory segment file descriptor diff --git a/src/main.cc b/src/main.cc index 725515d182..5adfa0edc2 100644 --- a/src/main.cc +++ b/src/main.cc @@ -73,6 +73,7 @@ #include "MemPool.h" #include "icmp/IcmpSquid.h" #include "icmp/net_db.h" +#include "fs/rock/RockDirMap.h" #if USE_LOADABLE_MODULES #include "LoadableModules.h" @@ -763,6 +764,7 @@ mainReconfigureFinish(void *) // parse the config returns a count of errors encountered. const int oldWorkers = Config.workers; + if ( parseConfigFile(ConfigFile) != 0) { // for now any errors are a fatal condition... self_destruct(); @@ -1368,6 +1370,15 @@ SquidMain(int argc, char **argv) Ip::ProbeTransport(); // determine IPv4 or IPv6 capabilities before parsing. + debugs(0,0, HERE << '1'); + if (IamMasterProcess()) { + debugs(0,0, HERE << '2'); + new Rock::DirMap("/home/dikk/projects/factory/squid/rock_smp/rock1", 1000); + new Rock::DirMap("/home/dikk/projects/factory/squid/rock_smp/rock2", 1000); + new Rock::DirMap("/home/dikk/projects/factory/squid/rock_smp/rock3", 1000); + } + debugs(0,0, HERE << '3'); + parse_err = parseConfigFile(ConfigFile); Mem::Report(); @@ -1695,7 +1706,7 @@ watch_child(char *argv[]) } TheKids.init(); -syslog(LOG_NOTICE, "XXX: will start %d kids", TheKids.count()); +syslog(LOG_NOTICE, "XXX: will start %d kids", (int)TheKids.count()); // keep [re]starting kids until it is time to quit for (;;) {