From: Alex Rousskov Date: Fri, 6 Dec 2013 23:52:26 +0000 (-0700) Subject: Merged from trunk 13172. X-Git-Tag: SQUID_3_5_0_1~444^2~12 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dcd84f807886121b9c725dec786a830bd1c8afb4;p=thirdparty%2Fsquid.git Merged from trunk 13172. --- dcd84f807886121b9c725dec786a830bd1c8afb4 diff --cc src/Makefile.am index 2998bf1398,9595c5c7ac..87d14167af --- a/src/Makefile.am +++ b/src/Makefile.am @@@ -1569,11 -1559,8 +1563,9 @@@ tests_testCacheManager_SOURCES = StoreSwapLogData.cc \ tools.h \ tools.cc \ + Transients.cc \ - tunnel.cc \ - SwapDir.cc \ + tests/stub_tunnel.cc \ + tests/stub_SwapDir.cc \ MemStore.cc \ $(UNLINKDSOURCE) \ url.cc \ @@@ -2005,8 -1988,7 +1993,8 @@@ tests_testEvent_SOURCES = time.cc \ tools.h \ tools.cc \ + Transients.cc \ - tunnel.cc \ + tests/stub_tunnel.cc \ MemStore.cc \ $(UNLINKDSOURCE) \ url.cc \ @@@ -2256,8 -2234,7 +2240,8 @@@ tests_testEventLoop_SOURCES = time.cc \ tools.h \ tools.cc \ + Transients.cc \ - tunnel.cc \ + tests/stub_tunnel.cc \ MemStore.cc \ $(UNLINKDSOURCE) \ url.cc \ @@@ -2785,9 -2753,8 +2761,9 @@@ tests_testHttpRequest_SOURCES = event.cc \ tools.h \ tools.cc \ + Transients.cc \ - tunnel.cc \ - SwapDir.cc \ + tests/stub_tunnel.cc \ + tests/stub_SwapDir.cc \ MemStore.cc \ url.cc \ URLScheme.cc \ @@@ -3264,9 -3227,7 +3236,9 @@@ testRefCount_LDADD = tests_testRock_SOURCES = \ cbdata.cc \ CacheDigest.h \ - CacheDigest.cc \ + CollapsedForwarding.h \ + CollapsedForwarding.cc \ + tests/stub_CacheDigest.cc \ ConfigOption.cc \ ConfigParser.cc \ disk.h \ @@@ -3767,8 -3557,7 +3570,8 @@@ tests_testURL_SOURCES = String.cc \ StrList.h \ StrList.cc \ - SwapDir.cc \ + Transients.cc \ + tests/stub_SwapDir.cc \ MemStore.cc \ tests/stub_debug.cc \ tests/stub_DiskIOModule.cc \ diff --cc src/MemObject.cc index b38cbb6805,7e403697a4..851943c229 --- a/src/MemObject.cc +++ b/src/MemObject.cc @@@ -74,49 -74,30 +74,49 @@@ MemObject::inUseCount( return Pool().inUseCount(); } -void -MemObject::resetUrls(char const *aUrl, char const *aLog_url) -{ - safe_free(url); - safe_free(log_url); /* XXX account log_url */ - log_url = xstrdup(aLog_url); - url = xstrdup(aUrl); +const char * +MemObject::storeId() const { - if (!storeId_.defined()) { ++ if (!storeId_.size()) { + debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value"); + dump(); + storeId_ = "[unknown_URI]"; + } + return storeId_.termedBuf(); } -MemObject::MemObject(char const *aUrl, char const *aLog_url) -{ - debugs(20, 3, HERE << "new MemObject " << this); - _reply = new HttpReply; - HTTPMSGLOCK(_reply); +const char * +MemObject::logUri() const { - return logUri_.defined() ? logUri_.termedBuf() : storeId(); ++ return logUri_.size() ? logUri_.termedBuf() : storeId(); +} - url = xstrdup(aUrl); +bool +MemObject::hasUris() const { - return storeId_.defined(); ++ return storeId_.size(); +} -#if URL_CHECKSUM_DEBUG +void +MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod) +{ + storeId_ = aStoreId; - chksum = url_checksum(url); + // fast pointer comparison for a common storeCreateEntry(url,url,...) case + if (!aLogUri || aLogUri == aStoreId) + logUri_.clean(); // use storeId_ by default to minimize copying + else + logUri_ = aLogUri; + method = aMethod; + +#if URL_CHECKSUM_DEBUG + chksum = url_checksum(urlXXX()); #endif +} - log_url = xstrdup(aLog_url); +MemObject::MemObject(): smpCollapsed(false) +{ + debugs(20, 3, HERE << "new MemObject " << this); + _reply = new HttpReply; + HTTPMSGLOCK(_reply); object_sz = -1; diff --cc src/acl/Asn.cc index a401101edb,5ef8091232..33062e8aba --- a/src/acl/Asn.cc +++ b/src/acl/Asn.cc @@@ -98,8 -103,31 +103,31 @@@ public int reqofs; char reqbuf[AS_REQBUF_SZ]; bool dataRead; + private: + CBDATA_CLASS2(ASState); }; + CBDATA_CLASS_INIT(ASState); + + ASState::ASState() : + entry(NULL), + sc(NULL), + request(NULL), + as_number(0), + offset(0), + reqofs(0), + dataRead(false) + { + memset(reqbuf, 0, AS_REQBUF_SZ); + } + + ASState::~ASState() + { + debugs(53, 3, entry->url()); + storeUnregister(sc, entry, this); - entry->unlock(); ++ entry->unlock("~ASState"); + } + /** entry into the radix tree */ struct rtentry_t { struct squid_radix_node e_nodes[2]; @@@ -250,9 -271,9 +271,9 @@@ asnCacheStart(int as if ((e = storeGetPublic(asres, Http::METHOD_GET)) == NULL) { e = storeCreateEntry(asres, asres, RequestFlags(), Http::METHOD_GET); asState->sc = storeClientListAdd(e, asState); - FwdState::fwdStart(Comm::ConnectionPointer(), e, asState->request); + FwdState::fwdStart(Comm::ConnectionPointer(), e, asState->request.getRaw()); } else { - e->lock(); + e->lock("Asn"); asState->sc = storeClientListAdd(e, asState); } diff --cc src/cf.data.pre index 67c17e5b39,533b726ac6..60fcd335f0 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@@ -3423,36 -3475,7 +3467,17 @@@ DOC_STAR and when set to zero, disables the disk I/O rate limit enforcement. Currently supported by IpcIo module only. + slot-size=bytes: The size of a database "record" used for + storing cached responses. A cached response occupies at least + one slot and all database I/O is done using individual slots so + increasing this parameter leads to more disk space waste while + decreasing it leads to more disk I/O overheads. Should be a + multiple of your operating system I/O page size. Defaults to + 16KBytes. A housekeeping header is stored with each slot and + smaller slot-sizes will be rejected. The header is smaller than + 100 bytes. + - ==== The coss store type ==== - - NP: COSS filesystem in Squid-3 has been deemed too unstable for - production use and has thus been removed from this release. - We hope that it can be made usable again soon. - - block-size=n defines the "block size" for COSS cache_dir's. - Squid uses file numbers as block numbers. Since file numbers - are limited to 24 bits, the block size determines the maximum - size of the COSS partition. The default is 512 bytes, which - leads to a maximum cache_dir size of 512<<24, or 8 GB. Note - you should not change the coss block size after Squid - has written some objects to the cache_dir. - - The coss file store has changed from 2.5. Now it uses a file - called 'stripe' in the directory names in the config - and - this will be created by squid -z. - - ==== COMMON OPTIONS ==== no-store no new objects should be stored to this cache_dir. diff --cc src/format/Format.cc index e737280e8e,15bef5e868..79a19ef5cd --- a/src/format/Format.cc +++ b/src/format/Format.cc @@@ -495,6 -495,13 +495,13 @@@ Format::Format::assemble(MemBuf &mb, co break; + case LFT_TIME_START: { + int precision = fmt->widthMax >=0 ? fmt->widthMax : 3; - snprintf(tmp, sizeof(tmp), "%0*" PRId64 ".%0*d", fmt->zero && (fmt->widthMin - precision - 1 >= 0) ? fmt->widthMin - precision - 1 : 0, al->cache.start_time.tv_sec, precision, (int)(al->cache.start_time.tv_usec / fmt->divisor)); ++ snprintf(tmp, sizeof(tmp), "%0*" PRId64 ".%0*d", fmt->zero && (fmt->widthMin - precision - 1 >= 0) ? fmt->widthMin - precision - 1 : 0, (int64_t)al->cache.start_time.tv_sec, precision, (int)(al->cache.start_time.tv_usec / fmt->divisor)); + out = tmp; + } + break; + case LFT_TIME_TO_HANDLE_REQUEST: outint = al->cache.msec; doint = 1; diff --cc src/fs/rock/RockIoState.cc index 4ee24ac55c,8190f28e9c..f1841617dc --- a/src/fs/rock/RockIoState.cc +++ b/src/fs/rock/RockIoState.cc @@@ -3,21 -3,17 +3,21 @@@ */ #include "squid.h" - #include "CollapsedForwarding.h" +#include "base/TextException.h" ++#include "CollapsedForwarding.h" #include "DiskIO/DiskIOModule.h" #include "DiskIO/DiskIOStrategy.h" #include "DiskIO/WriteRequest.h" - #include "fs/rock/RockIoState.h" #include "fs/rock/RockIoRequests.h" + #include "fs/rock/RockIoState.h" #include "fs/rock/RockSwapDir.h" #include "globals.h" - #include "MemObject.h" +#include "Mem.h" + #include "MemObject.h" #include "Parsing.h" +#include "Transients.h" -Rock::IoState::IoState(SwapDir *dir, +Rock::IoState::IoState(Rock::SwapDir::Pointer &aDir, StoreEntry *anEntry, StoreIOState::STFNCB *cbFile, StoreIOState::STIOCB *cbIo, diff --cc src/fs/rock/RockRebuild.cc index 454bf00652,aca7d89059..0457cabae0 --- a/src/fs/rock/RockRebuild.cc +++ b/src/fs/rock/RockRebuild.cc @@@ -4,16 -4,15 +4,16 @@@ #include "squid.h" #include "disk.h" + #include "fs/rock/RockDbCell.h" #include "fs/rock/RockRebuild.h" #include "fs/rock/RockSwapDir.h" - #include "fs/rock/RockDbCell.h" - #include "ipc/StoreMap.h" #include "globals.h" ++#include "ipc/StoreMap.h" #include "md5.h" - #include "tools.h" - #include "typedefs.h" #include "SquidTime.h" #include "store_rebuild.h" + #include "tools.h" + #include "typedefs.h" #if HAVE_ERRNO_H #include diff --cc src/fs/rock/RockSwapDir.h index 1a77a20855,00cb22ca87..2a844ae694 --- a/src/fs/rock/RockSwapDir.h +++ b/src/fs/rock/RockSwapDir.h @@@ -4,11 -4,8 +4,11 @@@ #include "DiskIO/DiskFile.h" #include "DiskIO/IORequestor.h" #include "fs/rock/RockDbCell.h" +#include "fs/rock/RockForward.h" #include "ipc/StoreMap.h" +#include "ipc/mem/Page.h" +#include "ipc/mem/PageStack.h" + #include "SwapDir.h" class DiskIOStrategy; class ReadRequest; diff --cc src/http.cc index 1409a905fb,99773d74f7..e7f853106b --- a/src/http.cc +++ b/src/http.cc @@@ -919,12 -910,9 +919,9 @@@ HttpStateData::haveParsedReplyHeaders( { ServerStateData::haveParsedReplyHeaders(); - Ctx ctx = ctx_enter(entry->mem_obj->url); + Ctx ctx = ctx_enter(entry->mem_obj->urlXXX()); HttpReply *rep = finalReply(); - if (rep->sline.status() == Http::scPartialContent && rep->content_range) - currentOffset = rep->content_range->spec.offset; - entry->timestampsSet(); /* Check if object is cacheable or not based on reply code */ diff --cc src/ipc/Strand.cc index 10dc1befd1,f6939b4b5e..20c57236f3 --- a/src/ipc/Strand.cc +++ b/src/ipc/Strand.cc @@@ -6,20 -6,19 +6,20 @@@ #include "squid.h" #include "base/Subscription.h" #include "base/TextException.h" + #include "CacheManager.h" ++#include "CollapsedForwarding.h" #include "comm/Connection.h" #include "globals.h" - #include "ipc/Strand.h" - #include "ipc/StrandCoord.h" + #include "ipc/Kids.h" #include "ipc/Messages.h" #include "ipc/SharedListen.h" + #include "ipc/Strand.h" + #include "ipc/StrandCoord.h" #include "ipc/StrandSearch.h" - #include "ipc/Kids.h" + #include "mgr/Forwarder.h" #include "mgr/Request.h" #include "mgr/Response.h" - #include "mgr/Forwarder.h" #include "SwapDir.h" /* XXX: scope boundary violation */ - #include "CacheManager.h" - #include "CollapsedForwarding.h" #if USE_DISKIO_IPCIO #include "DiskIO/IpcIo/IpcIoFile.h" /* XXX: scope boundary violation */ #endif diff --cc src/store_client.cc index 87c111a824,b990e385c2..092402094f --- a/src/store_client.cc +++ b/src/store_client.cc @@@ -260,38 -260,35 +260,40 @@@ store_client::copy(StoreEntry * anEntry #if USE_ADAPTATION anEntry->kickProducer(); #endif + anEntry->unlock("store_client::copy"); + - anEntry->unlock(); // after the "++enEntry->lock_count" above + // Add no code here. This object may no longer exist. } -/* - * This function is used below to decide if we have any more data to - * send to the client. If the store_status is STORE_PENDING, then we - * do have more data to send. If its STORE_OK, then - * we continue checking. If the object length is negative, then we - * don't know the real length and must open the swap file to find out. - * If the length is >= 0, then we compare it to the requested copy - * offset. - */ -static int -storeClientNoMoreToSend(StoreEntry * e, store_client * sc) +/// Whether there is (or will be) more entry data for us. +bool +store_client::moreToSend() const { - int64_t len; + if (entry->store_status == STORE_PENDING) + return true; // there may be more coming - if (e->store_status == STORE_PENDING) - return 0; + /* STORE_OK, including aborted entries: no more data is coming */ - if ((len = e->objectLen()) < 0) - return 0; + const int64_t len = entry->objectLen(); - if (sc->copyInto.offset < len) - return 0; + // If we do not know the entry length, then we have to open the swap file, + // which is only possible if there is one AND if we are allowed to use it. + const bool canSwapIn = entry->swap_filen >= 0 && + getType() == STORE_DISK_CLIENT; + if (len < 0) + return canSwapIn; - return 1; + if (copyInto.offset >= len) + return false; // sent everything there is + + if (canSwapIn) + return true; // if we lack prefix, we can swap it in + + // If we cannot swap in, make sure we have what we want in RAM. Otherwise, + // scheduleRead calls scheduleDiskRead which asserts on STORE_MEM_CLIENTs. + const MemObject *mem = entry->mem_obj; + return mem && + mem->inmem_lo <= copyInto.offset && copyInto.offset < mem->endOffset(); } static void diff --cc src/store_dir.cc index 5507e80524,cade545757..009f0424ec --- a/src/store_dir.cc +++ b/src/store_dir.cc @@@ -43,10 -43,9 +43,10 @@@ #include "Store.h" #include "store_key_md5.h" #include "StoreHashIndex.h" - #include "SwapDir.h" #include "swap_log_op.h" + #include "SwapDir.h" #include "tools.h" +#include "Transients.h" #if HAVE_STATVFS #if HAVE_SYS_STATVFS_H diff --cc src/whois.cc index d5f6156230,2d98a33438..4f5201611b --- a/src/whois.cc +++ b/src/whois.cc @@@ -204,6 -198,6 +197,6 @@@ whoisClose(const CommCloseCbParams &par { WhoisState *p = (WhoisState *)params.data; debugs(75, 3, "whoisClose: FD " << params.fd); - p->entry->unlock(); + p->entry->unlock("whoisClose"); - cbdataFree(p); + delete p; }