src/servers/Makefile
src/snmp/Makefile
src/ssl/Makefile
+ src/store/Makefile
test-suite/Makefile
tools/Makefile
tools/helper-mux/Makefile
#include "squid.h"
#include "Debug.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/AIO/AIODiskFile.h"
#include "DiskIO/AIO/AIODiskIOStrategy.h"
#include "DiskIO/IORequestor.h"
#include "BlockingFile.h"
#include "Debug.h"
#include "defines.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/IORequestor.h"
#include "DiskIO/ReadRequest.h"
#include "DiskIO/WriteRequest.h"
/* DEBUG: section 79 Disk IO Routines */
#include "squid.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/IORequestor.h"
#include "DiskIO/ReadRequest.h"
#include "DiskIO/WriteRequest.h"
#include "squid.h"
#include "base/RunnersRegistry.h"
#include "base/TextException.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/IORequestor.h"
#include "DiskIO/IpcIo/IpcIoFile.h"
#include "DiskIO/ReadRequest.h"
#include "squid.h"
#include "Debug.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/IORequestor.h"
#include "DiskIO/Mmapped/MmappedFile.h"
#include "DiskIO/ReadRequest.h"
#ifndef FILEMAP_H_
#define FILEMAP_H_
-#include "fs/forward.h"
+#include "store/forward.h"
/** A bitmap used for managing UFS StoreEntry "file numbers".
*
LoadableModules.h \
LoadableModules.cc
-SUBDIRS = mem base anyp helper dns ftp parser comm eui acl format clients servers fs repl DiskIO
-DIST_SUBDIRS = mem base anyp helper dns ftp parser comm eui acl format clients servers fs repl DiskIO
+SUBDIRS = mem base anyp helper dns ftp parser comm eui acl format clients servers fs repl store DiskIO
+DIST_SUBDIRS = mem base anyp helper dns ftp parser comm eui acl format clients servers fs repl store DiskIO
if ENABLE_AUTH
SUBDIRS += auth
Debug.h \
defines.h \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
Generic.h \
Store.h \
StoreFileSystem.cc \
StoreFileSystem.h \
- StoreHashIndex.h \
store_io.cc \
StoreIOBuffer.h \
StoreIOState.cc \
StoreClient.h \
store_digest.h \
store_digest.cc \
- store_dir.cc \
store_key_md5.h \
store_key_md5.cc \
store_log.h \
StoreSwapLogData.cc \
StoreSwapLogData.h \
swap_log_op.h \
- SwapDir.cc \
- SwapDir.h \
Transients.cc \
Transients.h \
MemStore.cc \
$(ESI_LIBS) \
$(SNMP_LIBS) \
mem/libmem.la \
+ store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
## ACLProxyAuth.cc wants ACLUserData
## ACLProxyAuth.cc wants ACLRegexData
## cache_cf.cc wants $(AUTH_LIBS)
-## cache_cf.cc wants Swapdir
+## cache_cf.cc wants store/libstore.la
## cache_cf.cc wants AnyP::PortCfg
## client_side wants client_db
## client_db wants SNMP_SOURCE
## snmp_core wants ACLStringData
-## SwapDir wants ConfigOption
## tools.cc wants ip/libip.la
## client_side.cc wants ip/libip.la
## libbase.la wants cbdata.*
SBufDetailedStats.h \
tests/stub_SBufDetailedStats.cc \
String.cc \
- store_dir.cc \
StoreIOState.cc \
tests/stub_StoreMeta.cc \
StoreMetaUnpacker.cc \
swap_log_op.cc \
swap_log_op.h \
tests/stub_SwapDir.cc \
- SwapDir.h \
Transients.cc \
log/access_log.h \
tests/stub_access_log.cc \
CpuAffinitySet.cc \
CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
mgr/libmgr.la \
$(SNMP_LIBS) \
mem/libmem.la \
+ store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
ConfigOption.cc \
ConfigParser.cc \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
tests/stub_ETag.cc \
EventLoop.cc \
event.cc \
fde.cc \
FileMap.h \
filemap.cc \
- fs/forward.h \
HttpBody.h \
HttpBody.cc \
HttpHeaderFieldStat.h \
tests/stub_StoreMeta.cc \
StoreMetaUnpacker.cc \
StoreSwapLogData.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
tests/stub_Port.cc \
tests/stub_stat.cc \
tests/stub_store_client.cc \
+ tests/stub_store_search.cc \
tests/stub_store_stats.cc \
store_rebuild.h \
tests/stub_store_rebuild.cc \
dns/libdns.la \
base/libbase.la \
mem/libmem.la \
+ store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
CpuAffinitySet.h \
debug.cc \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
$(top_builddir)/lib/libmiscutil.la \
ipc/libipc.la \
mgr/libmgr.la \
+ store/libstore.la \
$(SNMP_LIBS) \
$(NETTLELIB) \
$(REGEXLIB) \
CpuAffinitySet.h \
debug.cc \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
$(top_builddir)/lib/libmiscutil.la \
ipc/libipc.la \
mgr/libmgr.la \
+ store/libstore.la \
$(SNMP_LIBS) \
$(NETTLELIB) \
$(REGEXLIB) \
CpuAffinitySet.h \
debug.cc \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_key_md5.h \
store_key_md5.cc \
store_io.cc \
dns/libdns.la \
base/libbase.la \
mgr/libmgr.la \
+ store/libstore.la \
$(SNMP_LIBS) \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
CpuAffinitySet.cc \
CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
fde.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
comm/libcomm.la \
log/liblog.la \
format/libformat.la \
+ store/libstore.la \
$(REPL_OBJS) \
$(ADAPTATION_LIBS) \
$(ESI_LIBS) \
ConfigOption.cc \
ConfigParser.cc \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
ETag.cc \
event.cc \
EventLoop.cc \
tests/stub_fatal.cc \
FileMap.h \
filemap.cc \
- fs/forward.h \
HttpHeaderFieldStat.h \
HttpHdrCc.h \
HttpHdrCc.cc \
stmem.cc \
repl_modules.h \
store.cc \
- store_dir.cc \
store_io.cc \
store_swapout.cc \
StoreIOState.cc \
String.cc \
StrList.h \
StrList.cc \
- SwapDir.cc \
tests/CapturingStoreEntry.h \
log/access_log.h \
tests/stub_access_log.cc \
ipc/libipc.la \
anyp/libanyp.la \
mem/libmem.la \
+ store/libstore.la \
DiskIO/libdiskio.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
fde.h \
fde.cc \
client_db.h \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
FileMap.h \
filemap.cc \
- fs/forward.h \
HttpBody.h \
HttpBody.cc \
HttpReply.cc \
EventLoop.cc \
HttpMsg.cc \
RemovalPolicy.cc \
- store_dir.cc \
repl_modules.h \
store.cc \
store_key_md5.h \
store_key_md5.cc \
Parsing.cc \
ConfigOption.cc \
- SwapDir.cc \
tests/stub_acl.cc \
cache_cf.h \
YesNoNone.h \
base/libbase.la \
ip/libip.la \
mem/libmem.la \
+ store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
tests/stub_CacheDigest.cc \
ConfigOption.cc \
ConfigParser.cc \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
ETag.cc \
EventLoop.cc \
event.cc \
fde.cc \
FileMap.h \
filemap.cc \
- fs/forward.h \
HttpHeaderFieldStat.h \
HttpBody.h \
HttpBody.cc \
StoreMetaUnpacker.cc \
$(STOREMETA_SOURCE) \
StoreSwapLogData.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
String.cc \
StrList.h \
StrList.cc \
- SwapDir.cc \
Transients.h \
Transients.cc \
tests/testRock.cc \
ipc/libipc.la \
base/libbase.la \
mem/libmem.la \
+ store/libstore.la \
$(top_builddir)/lib/libmisccontainers.la \
$(top_builddir)/lib/libmiscencoding.la \
$(top_builddir)/lib/libmiscutil.la \
CpuAffinitySet.cc \
CpuAffinitySet.h \
$(DELAY_POOL_SOURCE) \
- disk.h \
- disk.cc \
+ fs_io.h \
+ fs_io.cc \
dlink.h \
dlink.cc \
$(DNSSOURCE) \
filemap.cc \
fqdncache.h \
fqdncache.cc \
- fs/forward.h \
FwdState.cc \
FwdState.h \
gopher.h \
store_client.cc \
store_digest.h \
tests/stub_store_digest.cc \
- store_dir.cc \
store_io.cc \
store_key_md5.h \
store_key_md5.cc \
comm/libcomm.la \
log/liblog.la \
format/libformat.la \
+ store/libstore.la \
$(REGEXLIB) \
$(REPL_OBJS) \
$(ADAPTATION_LIBS) \
}
bool
-MemStore::dereference(StoreEntry &, bool)
+MemStore::dereference(StoreEntry &)
{
// no need to keep e in the global store_table for us; we have our own map
return false;
}
-int
-MemStore::callback()
-{
- return 0;
-}
-
-StoreSearch *
-MemStore::search(String const, HttpRequest *)
-{
- fatal("not implemented");
- return NULL;
-}
-
StoreEntry *
MemStore::get(const cache_key *key)
{
return NULL;
}
-void
-MemStore::get(String const, STOREGETCLIENT, void *)
-{
- // XXX: not needed but Store parent forces us to implement this
- fatal("MemStore::get(key,callback,data) should not be called");
-}
-
bool
MemStore::anchorCollapsed(StoreEntry &collapsed, bool &inSync)
{
#include "ipc/mem/PageStack.h"
#include "ipc/StoreMap.h"
#include "Store.h"
+#include "store/Controlled.h"
// StoreEntry restoration info not already stored by Ipc::StoreMap
struct MemStoreMapExtraItem {
/// Stores HTTP entities in RAM. Current implementation uses shared memory.
/// Unlike a disk store (SwapDir), operations are synchronous (and fast).
-class MemStore: public Store, public Ipc::StoreMapCleaner
+class MemStore: public Store::Controlled, public Ipc::StoreMapCleaner
{
public:
MemStore();
/// all data has been received; there will be no more write() calls
void completeWriting(StoreEntry &e);
- /// remove from the cache
- void unlink(StoreEntry &e);
-
/// called when the entry is about to forget its association with mem cache
void disconnect(StoreEntry &e);
- /* Store API */
- virtual int callback();
- virtual StoreEntry * get(const cache_key *);
- virtual void get(String const key , STOREGETCLIENT callback, void *cbdata);
- virtual void init();
- virtual uint64_t maxSize() const;
- virtual uint64_t minSize() const;
- virtual uint64_t currentSize() const;
- virtual uint64_t currentCount() const;
- virtual int64_t maxObjectSize() const;
- virtual void getStats(StoreInfoStats &stats) const;
- virtual void stat(StoreEntry &) const;
- virtual StoreSearch *search(String const url, HttpRequest *);
- virtual void markForUnlink(StoreEntry &e);
- virtual void reference(StoreEntry &);
- virtual bool dereference(StoreEntry &, bool);
- virtual void maintain();
- virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync);
- virtual bool updateCollapsed(StoreEntry &collapsed);
+ /* Storage API */
+ virtual void create() override {}
+ virtual void init() override;
+ virtual StoreEntry *get(const cache_key *) override;
+ virtual uint64_t maxSize() const override;
+ virtual uint64_t minSize() const override;
+ virtual uint64_t currentSize() const override;
+ virtual uint64_t currentCount() const override;
+ virtual int64_t maxObjectSize() const override;
+ virtual void getStats(StoreInfoStats &stats) const override;
+ virtual void stat(StoreEntry &e) const override;
+ virtual void reference(StoreEntry &e) override;
+ virtual bool dereference(StoreEntry &e) override;
+ virtual void maintain() override;
+ virtual bool anchorCollapsed(StoreEntry &e, bool &inSync) override;
+ virtual bool updateCollapsed(StoreEntry &e) override;
+ virtual void markForUnlink(StoreEntry &) override;
+ virtual void unlink(StoreEntry &e) override;
static int64_t EntryLimit();
#include "base/InstanceId.h"
#include "Debug.h"
+#include "globals.h"
#include "MemBlob.h"
#include "SBufExceptions.h"
#include "SquidString.h"
#include "Notes.h"
#include "security/forward.h"
#include "SquidTime.h"
+#include "store/forward.h"
#include "YesNoNone.h"
#if USE_OPENSSL
class HeaderManglers;
class RefreshPattern;
class RemovalPolicySettings;
-class SwapDir;
namespace AnyP
{
class PortCfg;
}
+namespace Store {
+class DiskConfig {
+public:
+ RefCount<SwapDir> *swapDirs;
+ int n_allocated;
+ int n_configured;
+ /// number of disk processes required to support all cache_dirs
+ int n_strands;
+};
+#define INDEXSD(i) (Config.cacheSwap.swapDirs[i].getRaw())
+}
+
/// the representation of the configuration. POD.
class SquidConfig
{
} Ftp;
RefreshPattern *Refresh;
- struct _cacheSwap {
- RefCount<SwapDir> *swapDirs;
- int n_allocated;
- int n_configured;
- /// number of disk processes required to support all cache_dirs
- int n_strands;
- } cacheSwap;
- /*
- * I'm sick of having to keep doing this ..
- */
-#define INDEXSD(i) (Config.cacheSwap.swapDirs[(i)].getRaw())
+ Store::DiskConfig cacheSwap;
struct {
char *directory;
#include "MemObject.h"
#include "Range.h"
#include "RemovalPolicy.h"
+#include "store/Controller.h"
+#include "store/forward.h"
#include "store_key_md5.h"
#include "StoreIOBuffer.h"
#include "StoreStats.h"
class AsyncCall;
class HttpRequest;
class RequestFlags;
-class StoreClient;
-class StoreSearch;
-class SwapDir;
extern StoreIoStats store_io_stats;
-/// maximum number of entries per cache_dir
-enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
-
class StoreEntry : public hash_link, public Packable
{
MEMPROXY_CLASS(StoreEntry);
void swapOutDecision(const MemObject::SwapOut::Decision &decision);
void abort();
- void unlink();
void makePublic();
void makePrivate();
void setPublicKey();
/// whether this entry has an ETag; if yes, puts ETag value into parameter
bool hasEtag(ETag &etag) const;
- /** What store does this entry belong too ? */
- virtual RefCount<SwapDir> store() const;
+ /// the disk this entry is [being] cached on; asserts for entries w/o a disk
+ Store::Disk &disk() const;
MemObject *mem_obj;
RemovalPolicyNode repl;
/// \ingroup StoreAPI
typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
-/**
- \ingroup StoreAPI
- * Abstract base class that will replace the whole store and swapdir interface.
- */
-class Store : public RefCountable
-{
-
-public:
- /** The root store */
- static Store &Root() {
- if (CurrentRoot == NULL)
- fatal("No Store Root has been set");
- return *CurrentRoot;
- }
- static void Root(Store *);
- static void Root(RefCount<Store>);
- static void Stats(StoreEntry * output);
- static void Maintain(void *unused);
-
- virtual ~Store() {}
-
- /** Handle pending callbacks - called by the event loop. */
- virtual int callback() = 0;
-
- /** create the resources needed for this store to operate */
- virtual void create();
-
- /**
- * Notify this store that its disk is full.
- \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
- */
- virtual void diskFull();
-
- /** Retrieve a store entry from the store */
- virtual StoreEntry * get(const cache_key *) = 0;
-
- /** \todo imeplement the async version */
- virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
-
- /* prepare the store for use. The store need not be usable immediately,
- * it should respond to readable() and writable() with true as soon
- * as it can provide those services
- */
- virtual void init() = 0;
-
- /**
- * The maximum size the store will support in normal use. Inaccuracy is permitted,
- * but may throw estimates for memory etc out of whack.
- */
- virtual uint64_t maxSize() const = 0;
-
- /** The minimum size the store will shrink to via normal housekeeping */
- virtual uint64_t minSize() const = 0;
-
- /** current store size */
- virtual uint64_t currentSize() const = 0;
-
- /** the total number of objects stored */
- virtual uint64_t currentCount() const = 0;
-
- /** the maximum object size that can be stored, -1 if unlimited */
- virtual int64_t maxObjectSize() const = 0;
-
- /// collect cache storage-related statistics
- virtual void getStats(StoreInfoStats &stats) const = 0;
-
- /**
- * Output stats to the provided store entry.
- \todo make these calls asynchronous
- */
- virtual void stat(StoreEntry &) const = 0;
-
- /** Sync the store prior to shutdown */
- virtual void sync();
-
- /** remove a Store entry from the store */
- virtual void unlink (StoreEntry &);
-
- /* search in the store */
- virtual StoreSearch *search(String const url, HttpRequest *) = 0;
-
- /* pulled up from SwapDir for migration.... probably do not belong here */
- virtual void reference(StoreEntry &) = 0; /* Reference this object */
-
- /// Undo reference(), returning false iff idle e should be destroyed
- virtual bool dereference(StoreEntry &, bool wantsLocalMemory) = 0;
-
- virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// informs stores that this entry will be eventually unlinked
- virtual void markForUnlink(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // because test cases use non-StoreController derivatives as Root
- /// called when the entry is no longer needed by any transaction
- virtual void handleIdleEntry(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // because test cases use non-StoreController derivatives as Root
- /// called to get rid of no longer needed entry data in RAM, if any
- virtual void memoryOut(StoreEntry &, const bool /*preserveSwappable*/) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// makes the entry available for collapsing future requests
- virtual void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// marks the entry completed for collapsed requests
- virtual void transientsCompleteWriting(StoreEntry &) {}
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// Update local intransit entry after changes made by appending worker.
- virtual void syncCollapsed(const sfileno) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// calls Root().transients->abandon() if transients are tracked
- virtual void transientsAbandon(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// number of the transient entry readers some time ago
- virtual int transientReaders(const StoreEntry &) const { return 0; }
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// disassociates the entry from the intransit table
- virtual void transientsDisconnect(MemObject &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// removes the entry from the memory cache
- virtual void memoryUnlink(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// disassociates the entry from the memory cache, preserving cached data
- virtual void memoryDisconnect(StoreEntry &) {}
-
- /// If the entry is not found, return false. Otherwise, return true after
- /// tying the entry to this cache and setting inSync to updateCollapsed().
- virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
-
- /// update a local collapsed entry with fresh info from this cache (if any)
- virtual bool updateCollapsed(StoreEntry &) { return false; }
-
-private:
- static RefCount<Store> CurrentRoot;
+namespace Store {
+ void Stats(StoreEntry *output);
+ void Maintain(void *unused);
};
-/// \ingroup StoreAPI
-typedef RefCount<Store> StorePointer;
-
/// \ingroup StoreAPI
size_t storeEntryInUse();
/// \ingroup StoreAPI
int storeTooManyDiskFilesOpen(void);
-class SwapDir;
/// \ingroup StoreAPI
void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
#ifndef SQUID_STOREFILESYSTEM_H
#define SQUID_STOREFILESYSTEM_H
+#include "store/forward.h"
#include <vector>
/* ****** DOCUMENTATION ***** */
* given StoreEntry. A maxobjsize of -1 means 'any size'.
*/
-class SwapDir;
-
/**
\ingroup FileSystems
*
+++ /dev/null
-/*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREHASHINDEX_H
-#define SQUID_STOREHASHINDEX_H
-
-#include "Store.h"
-#include "StoreSearch.h"
-
-/* A summary store that indexs all its children
- * into a memory hash
- */
-
-class StoreSearch;
-
-class StoreHashIndex : public Store
-{
-
-public:
- StoreHashIndex();
- StoreHashIndex(StoreHashIndex const &); /* to cause link failures */
- virtual ~StoreHashIndex();
- virtual int callback();
- virtual void create();
-
- virtual StoreEntry * get
- (const cache_key *);
-
- virtual void get
- (String const, STOREGETCLIENT, void * cbdata);
-
- virtual void init();
-
- virtual void sync();
-
- virtual uint64_t maxSize() const;
-
- virtual uint64_t minSize() const;
-
- virtual uint64_t currentSize() const;
-
- virtual uint64_t currentCount() const;
-
- virtual int64_t maxObjectSize() const;
-
- virtual void getStats(StoreInfoStats &stats) const;
- virtual void stat(StoreEntry&) const;
-
- virtual void reference(StoreEntry&);
-
- virtual bool dereference(StoreEntry&, bool);
-
- virtual void maintain();
-
- virtual StoreSearch *search(String const url, HttpRequest *);
-
-private:
- /* migration logic */
- StorePointer store(int const x) const;
- SwapDir &dir(int const idx) const;
-};
-
-class StoreHashIndexEntry : public StoreEntry
-{};
-
-class StoreSearchHashIndex : public StoreSearch
-{
- CBDATA_CLASS(StoreSearchHashIndex);
-
-public:
- StoreSearchHashIndex(RefCount<StoreHashIndex> sd);
- StoreSearchHashIndex(StoreSearchHashIndex const &);
- virtual ~StoreSearchHashIndex();
- /* Iterator API - garh, wrong place */
- /* callback the client when a new StoreEntry is available
- * or an error occurs
- */
- virtual void next(void (callback)(void *cbdata), void *cbdata);
- /* return true if a new StoreEntry is immediately available */
- virtual bool next();
- virtual bool error() const;
- virtual bool isDone() const;
- virtual StoreEntry *currentItem();
-
- RefCount<StoreHashIndex> sd;
-
-private:
- void copyBucket();
- void (*callback)(void *cbdata);
- void *cbdata;
- bool _done;
- int bucket;
- std::vector<StoreEntry *> entries;
-};
-
-#endif /* SQUID_STOREHASHINDEX_H */
-
#include "base/RefCount.h"
#include "cbdata.h"
-#include "fs/forward.h"
#include "mem/forward.h"
+#include "store/forward.h"
class StoreIOState : public RefCountable
{
* the value for MemObject->swap_hdr_sz.
*/
-#include "fs/forward.h"
#include "md5.h"
#include "mem/forward.h"
+#include "store/forward.h"
/// maintains a 24-bit checksum over integer fields
class SwapChecksum24
+++ /dev/null
-/*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_SWAPDIR_H
-#define SQUID_SWAPDIR_H
-
-#include "mgr/forward.h"
-#include "SquidConfig.h"
-#include "Store.h"
-#include "StoreIOState.h"
-
-/* forward decls */
-class RemovalPolicy;
-class MemStore;
-class Transients;
-class RequestFlags;
-class HttpRequestMethod;
-
-/* Store dir configuration routines */
-/* SwapDir *sd, char *path ( + char *opt later when the strtok mess is gone) */
-
-typedef int STDIRSELECT(const StoreEntry *);
-
-class ConfigOption;
-
-/// hides memory/disk cache distinction from callers
-class StoreController : public Store
-{
-
-public:
- StoreController();
- virtual ~StoreController();
- virtual int callback();
- virtual void create();
-
- virtual StoreEntry * get(const cache_key *);
-
- virtual void get(String const, STOREGETCLIENT, void * cbdata);
-
- /* Store parent API */
- virtual void markForUnlink(StoreEntry &e);
- virtual void handleIdleEntry(StoreEntry &e);
- virtual void transientsCompleteWriting(StoreEntry &e);
- virtual void transientsAbandon(StoreEntry &e);
- virtual int transientReaders(const StoreEntry &e) const;
- virtual void transientsDisconnect(MemObject &mem_obj);
- virtual void memoryOut(StoreEntry &e, const bool preserveSwappable);
- virtual void memoryUnlink(StoreEntry &e);
- virtual void memoryDisconnect(StoreEntry &e);
- virtual void allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod);
- virtual void syncCollapsed(const sfileno xitIndex);
-
- virtual void init();
-
- virtual void maintain(); /* perform regular maintenance should be private and self registered ... */
-
- virtual uint64_t maxSize() const;
-
- virtual uint64_t minSize() const;
-
- virtual uint64_t currentSize() const;
-
- virtual uint64_t currentCount() const;
-
- virtual int64_t maxObjectSize() const;
-
- virtual void getStats(StoreInfoStats &stats) const;
- virtual void stat(StoreEntry &) const;
-
- virtual void sync(); /* Sync the store prior to shutdown */
-
- virtual StoreSearch *search(String const url, HttpRequest *);
-
- virtual void reference(StoreEntry &); /* Reference this object */
-
- virtual bool dereference(StoreEntry &, bool); /* Unreference this object */
-
- /* the number of store dirs being rebuilt. */
- static int store_dirs_rebuilding;
-
-private:
- void createOneStore(Store &aStore);
- StoreEntry *find(const cache_key *key);
- bool keepForLocalMemoryCache(StoreEntry &e) const;
- bool anchorCollapsed(StoreEntry &collapsed, bool &inSync);
- bool anchorCollapsedOnDisk(StoreEntry &collapsed, bool &inSync);
-
- StorePointer swapDir; ///< summary view of all disk caches
- MemStore *memStore; ///< memory cache
-
- /// A shared table of public store entries that do not know whether they
- /// will belong to a memory cache, a disk cache, or will be uncachable
- /// when the response header comes. Used for SMP collapsed forwarding.
- Transients *transients;
-};
-
-/* migrating from the Config based list of swapdirs */
-void allocate_new_swapdir(SquidConfig::_cacheSwap *);
-void free_cachedir(SquidConfig::_cacheSwap * swap);
-extern OBJH storeDirStats;
-char *storeDirSwapLogFile(int, const char *);
-char *storeSwapFullPath(int, char *);
-char *storeSwapSubSubDir(int, char *);
-const char *storeSwapPath(int);
-int storeDirWriteCleanLogs(int reopen);
-extern STDIRSELECT *storeDirSelectSwapDir;
-int storeVerifySwapDirs(void);
-void storeDirCloseSwapLogs(void);
-void storeDirCloseTmpSwapLog(int dirn);
-void storeDirDiskFull(sdirno);
-void storeDirOpenSwapLogs(void);
-void storeDirSwapLog(const StoreEntry *, int op);
-void storeDirLRUDelete(StoreEntry *);
-void storeDirLRUAdd(StoreEntry *);
-int storeDirGetBlkSize(const char *path, int *blksize);
-int storeDirGetUFSStats(const char *, int *, int *, int *, int *);
-
-/// manages a single cache_dir
-class SwapDir : public Store
-{
-
-public:
- typedef RefCount<SwapDir> Pointer;
-
- SwapDir(char const *aType);
- virtual ~SwapDir();
- virtual void reconfigure() = 0;
- char const *type() const;
-
- virtual bool needsDiskStrand() const; ///< needs a dedicated kid process
- virtual bool active() const; ///< may be used in this strand
- /// whether stat should be reported by this SwapDir
- virtual bool doReportStat() const { return active(); }
- /// whether SwapDir may benefit from unlinkd
- virtual bool unlinkdUseful() const = 0;
-
- /* official Store interface functions */
- virtual void diskFull();
-
- virtual StoreEntry * get(const cache_key *);
-
- virtual void get(String const, STOREGETCLIENT, void * cbdata);
-
- virtual uint64_t maxSize() const { return max_size;}
-
- virtual uint64_t minSize() const;
-
- /// The maximum size of object which may be stored here.
- /// Larger objects will not be added and may be purged.
- virtual int64_t maxObjectSize() const;
-
- /// configure the maximum object size for this storage area.
- /// May be any size up to the total storage area.
- void maxObjectSize(int64_t newMax);
-
- virtual void getStats(StoreInfoStats &stats) const;
- virtual void stat (StoreEntry &anEntry) const;
- virtual StoreSearch *search(String const url, HttpRequest *) = 0;
-
- /* migrated from store_dir.cc */
- bool objectSizeIsAcceptable(int64_t objsize) const;
-
- /// called when the entry is about to forget its association with cache_dir
- virtual void disconnect(StoreEntry &) {}
-
- /// called when entry swap out is complete
- virtual void swappedOut(const StoreEntry &e) = 0;
-
-protected:
- void parseOptions(int reconfiguring);
- void dumpOptions(StoreEntry * e) const;
- virtual ConfigOption *getOptionTree() const;
- virtual bool allowOptionReconfigure(const char *const) const { return true; }
-
- int64_t sizeInBlocks(const int64_t size) const { return (size + fs.blksize - 1) / fs.blksize; }
-
-private:
- bool optionReadOnlyParse(char const *option, const char *value, int reconfiguring);
- void optionReadOnlyDump(StoreEntry * e) const;
- bool optionObjectSizeParse(char const *option, const char *value, int reconfiguring);
- void optionObjectSizeDump(StoreEntry * e) const;
- char const *theType;
-
-protected:
- uint64_t max_size; ///< maximum allocatable size of the storage area
- int64_t min_objsize; ///< minimum size of any object stored here (-1 for no limit)
- int64_t max_objsize; ///< maximum size of any object stored here (-1 for no limit)
-
-public:
- char *path;
- int index; /* This entry's index into the swapDirs array */
- int disker; ///< disker kid id dedicated to this SwapDir or -1
- RemovalPolicy *repl;
- int removals;
- int scanned;
-
- struct Flags {
- Flags() : selected(false), read_only(false) {}
- bool selected;
- bool read_only;
- } flags;
- virtual void init() = 0; /* Initialise the fs */
- virtual void create(); /* Create a new fs */
- virtual void dump(StoreEntry &)const; /* Dump fs config snippet */
- virtual bool doubleCheck(StoreEntry &); /* Double check the obj integrity */
- virtual void statfs(StoreEntry &) const; /* Dump fs statistics */
- virtual void maintain(); /* Replacement maintainence */
- /// check whether we can store the entry; if we can, report current load
- virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const = 0;
- /* These two are notifications */
- virtual void reference(StoreEntry &); /* Reference this object */
- virtual bool dereference(StoreEntry &, bool); /* Unreference this object */
- virtual int callback(); /* Handle pending callbacks */
- virtual void sync(); /* Sync the store prior to shutdown */
- virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) = 0;
- virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) = 0;
- virtual void unlink (StoreEntry &);
- bool canLog(StoreEntry const &e)const;
- virtual void openLog();
- virtual void closeLog();
- virtual void logEntry(const StoreEntry & e, int op) const;
-
- class CleanLog
- {
-
- public:
- virtual ~CleanLog() {}
-
- virtual const StoreEntry *nextEntry() = 0;
- virtual void write(StoreEntry const &) = 0;
- };
-
- CleanLog *cleanLog;
- virtual int writeCleanStart();
- virtual void writeCleanDone();
- virtual void parse(int index, char *path) = 0;
-
- struct {
- int blksize;
- } fs;
-};
-
-#endif /* SQUID_SWAPDIR_H */
-
}
bool
-Transients::dereference(StoreEntry &, bool)
+Transients::dereference(StoreEntry &)
{
// no need to keep e in the global store_table for us; we have our own map
return false;
}
-int
-Transients::callback()
-{
- return 0;
-}
-
-StoreSearch *
-Transients::search(String const, HttpRequest *)
-{
- fatal("not implemented");
- return NULL;
-}
-
StoreEntry *
Transients::get(const cache_key *key)
{
return e;
}
-void
-Transients::get(String const, STOREGETCLIENT, void *)
-{
- // XXX: not needed but Store parent forces us to implement this
- fatal("Transients::get(key,callback,data) should not be called");
-}
-
StoreEntry *
Transients::findCollapsed(const sfileno index)
{
void
Transients::markForUnlink(StoreEntry &e)
+{
+ unlink(e);
+}
+
+void
+Transients::unlink(StoreEntry &e)
{
if (e.mem_obj && e.mem_obj->xitTable.io == MemObject::ioWriting)
abandon(e);
#include "ipc/mem/PageStack.h"
#include "ipc/StoreMap.h"
#include "Store.h"
+#include "store/Controlled.h"
#include <vector>
// StoreEntry restoration info not already stored by Ipc::StoreMap
/// Keeps track of store entries being delivered to clients that arrived before
/// those entries were [fully] cached. This shared table is necessary to sync
/// the entry-writing worker with entry-reading worker(s).
-class Transients: public Store, public Ipc::StoreMapCleaner
+class Transients: public Store::Controlled, public Ipc::StoreMapCleaner
{
public:
Transients();
void disconnect(MemObject &mem_obj);
/* Store API */
- virtual int callback();
- virtual StoreEntry * get(const cache_key *);
- virtual void get(String const key , STOREGETCLIENT callback, void *cbdata);
- virtual void init();
- virtual uint64_t maxSize() const;
- virtual uint64_t minSize() const;
- virtual uint64_t currentSize() const;
- virtual uint64_t currentCount() const;
- virtual int64_t maxObjectSize() const;
- virtual void getStats(StoreInfoStats &stats) const;
- virtual void stat(StoreEntry &) const;
- virtual StoreSearch *search(String const url, HttpRequest *);
- virtual void reference(StoreEntry &);
- virtual bool dereference(StoreEntry &, bool);
- virtual void markForUnlink(StoreEntry &e);
- virtual void maintain();
+ virtual StoreEntry *get(const cache_key *) override;
+ virtual void create() override {}
+ virtual void init() override;
+ virtual uint64_t maxSize() const override;
+ virtual uint64_t minSize() const override;
+ virtual uint64_t currentSize() const override;
+ virtual uint64_t currentCount() const override;
+ virtual int64_t maxObjectSize() const override;
+ virtual void getStats(StoreInfoStats &stats) const override;
+ virtual void stat(StoreEntry &e) const override;
+ virtual void reference(StoreEntry &e) override;
+ virtual bool dereference(StoreEntry &e) override;
+ virtual void markForUnlink(StoreEntry &e) override;
+ virtual void unlink(StoreEntry &e) override;
+ virtual void maintain() override;
static int64_t EntryLimit();
#include "SquidString.h"
#include "ssl/ProxyCerts.h"
#include "Store.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
#include "StoreFileSystem.h"
-#include "SwapDir.h"
#include "tools.h"
#include "util.h"
#include "wordlist.h"
#endif
static void
-dump_cachedir(StoreEntry * entry, const char *name, SquidConfig::_cacheSwap swap)
+dump_cachedir(StoreEntry * entry, const char *name, const Store::DiskConfig &swap)
{
SwapDir *s;
int i;
}
static void
-parse_cachedir(SquidConfig::_cacheSwap * swap)
+parse_cachedir(Store::DiskConfig *swap)
{
char *type_str;
char *path_str;
#include "clients/forward.h"
#include "comm/Connection.h"
#include "comm/Write.h"
-#include "disk.h"
+#include "fs_io.h"
#include "err_detail_type.h"
#include "errorpage.h"
#include "fde.h"
#include "Debug.h"
#include "fatal.h"
#include "globals.h"
-#include "SwapDir.h"
+#include "store/Disks.h"
#include "tools.h"
static void
shutting_down = 1;
releaseServerSockets();
- /* check for store_dirs_rebuilding because fatal() is often
- * used in early initialization phases, long before we ever
- * get to the store log. */
/* XXX: this should be turned into a callback-on-fatal, or
* a mandatory-shutdown-event or something like that.
*/
leave_suid();
- if (0 == StoreController::store_dirs_rebuilding)
- storeDirWriteCleanLogs(0);
+ storeDirWriteCleanLogs(0);
fatal_common(message);
#ifndef SQUID_FS_ROCK_DB_CELL_H
#define SQUID_FS_ROCK_DB_CELL_H
-#include "fs/forward.h"
+#include "store/forward.h"
namespace Rock
{
/* DEBUG: section 79 Disk IO Routines */
#include "squid.h"
-#include "disk.h"
+#include "fs_io.h"
+#include "base/AsyncJobCalls.h"
#include "fs/rock/RockDbCell.h"
#include "fs/rock/RockRebuild.h"
#include "fs/rock/RockSwapDir.h"
#include "md5.h"
#include "SquidTime.h"
#include "store_rebuild.h"
+#include "Store.h"
#include "tools.h"
#include <cerrno>
safe_free(filePath);
}
-StoreSearch *
-Rock::SwapDir::search(String const, HttpRequest *)
-{
- assert(false);
- return NULL; // XXX: implement
-}
-
-void
-Rock::SwapDir::get(String const key, STOREGETCLIENT cb, void *data)
-{
- ::SwapDir::get(key, cb, data);
-}
-
// called when Squid core needs a StoreEntry with a given key
StoreEntry *
Rock::SwapDir::get(const cache_key *key)
}
bool
-Rock::SwapDir::dereference(StoreEntry &e, bool)
+Rock::SwapDir::dereference(StoreEntry &e)
{
debugs(47, 5, HERE << &e << ' ' << e.swap_dirn << ' ' << e.swap_filen);
if (repl && repl->Dereferenced)
#include "ipc/mem/Page.h"
#include "ipc/mem/PageStack.h"
#include "ipc/StoreMap.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
class DiskIOStrategy;
class ReadRequest;
/* public ::SwapDir API */
virtual void reconfigure();
- virtual StoreSearch *search(String const url, HttpRequest *);
virtual StoreEntry *get(const cache_key *key);
- virtual void get(String const, STOREGETCLIENT, void * cbdata);
virtual void markForUnlink(StoreEntry &e);
virtual void disconnect(StoreEntry &e);
virtual uint64_t currentSize() const;
virtual void maintain();
virtual void diskFull();
virtual void reference(StoreEntry &e);
- virtual bool dereference(StoreEntry &e, bool);
+ virtual bool dereference(StoreEntry &e);
virtual bool unlinkdUseful() const;
virtual void unlink(StoreEntry &e);
virtual void statfs(StoreEntry &e) const;
#ifndef SQUID_FS_ROCK_FORWARD_H
#define SQUID_FS_ROCK_FORWARD_H
-#include "fs/forward.h"
+#include "store/forward.h"
namespace Ipc
{
/* DEBUG: section 47 Store Directory Routines */
#include "squid.h"
-#include "disk.h"
+#include "fs_io.h"
#include "globals.h"
#include "RebuildState.h"
#include "SquidConfig.h"
#include "SquidTime.h"
+#include "store/Disks.h"
#include "store_key_md5.h"
#include "store_rebuild.h"
#include "StoreSwapLogData.h"
currentEntry()->lastmod = swapData.lastmod;
currentEntry()->flags = swapData.flags;
currentEntry()->refcount += swapData.refcount;
- sd->dereference(*currentEntry(), false);
+ sd->dereference(*currentEntry());
} else {
debug_trap("commonUfsDirRebuildFromSwapLog: bad condition");
debugs(47, DBG_IMPORTANT, HERE << "bad condition");
#include "Generic.h"
#include "SquidList.h"
#include "Store.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
#include "UFSStoreState.h"
#include "UFSStrategy.h"
#include "squid.h"
#include "cache_cf.h"
#include "ConfigOption.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/DiskIOModule.h"
#include "DiskIO/DiskIOStrategy.h"
#include "fde.h"
started_clean_event = 1;
}
- (void) storeDirGetBlkSize(path, &fs.blksize);
+ (void) fsBlockSize(path, &fs.blksize);
}
void
storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n",
map->numFilesInMap(), map->capacity(),
Math::intPercent(map->numFilesInMap(), map->capacity()));
- x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
+ x = fsStats(path, &totl_kb, &free_kb, &totl_in, &free_in);
if (0 == x) {
storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n",
}
bool
-Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e, bool)
+Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e)
{
debugs(47, 3, HERE << "dereferencing " << &e << " " <<
e.swap_dirn << "/" << e.swap_filen);
replacementRemove(&e);
mapBitReset(e.swap_filen);
UFSSwapDir::unlinkFile(e.swap_filen);
+ e.swap_filen = -1;
+ e.swap_dirn = -1;
+ e.swap_status = SWAPOUT_NONE;
}
void
void
Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e)
{
- StorePointer SD;
-
if (e->swap_dirn < 0)
return;
- SD = INDEXSD(e->swap_dirn);
+ SwapDirPointer SD = INDEXSD(e->swap_dirn);
assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this);
++n_disk_objects;
}
-StoreSearch *
-Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *)
-{
- if (url.size())
- fatal ("Cannot search by url yet\n");
-
- return new Fs::Ufs::StoreSearchUFS (this);
-}
-
void
Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const
{
#include "SquidString.h"
#include "Store.h"
+#include "store/Disk.h"
#include "StoreIOState.h"
#include "StoreSearch.h"
#include "swap_log_op.h"
-#include "SwapDir.h"
#include "UFSStrategy.h"
class HttpRequest;
virtual void create();
virtual void dump(StoreEntry &) const;
~UFSSwapDir();
- virtual StoreSearch *search(String const url, HttpRequest *);
/** double-check swap during rebuild (-S command-line option)
*
* called by storeCleanup if needed
virtual void unlink(StoreEntry &);
virtual void statfs(StoreEntry &)const;
virtual void maintain();
+ virtual bool anchorCollapsed(StoreEntry &, bool &) override { return false; }
+ virtual bool updateCollapsed(StoreEntry &) override { return false; }
+ virtual void markForUnlink(StoreEntry &) override {}
+
/** check whether this filesystem can store the given object
*
* UFS filesystems will happily store anything as long as
* This routine is called whenever the last reference to an object is
* removed, to maintain replacement information within the storage fs.
*/
- virtual bool dereference(StoreEntry &, bool);
+ virtual bool dereference(StoreEntry &);
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void openLog();
#include "squid.h"
#include "comm/Loops.h"
-#include "disk.h"
+#include "fs_io.h"
#include "fd.h"
#include "fde.h"
#include "globals.h"
#endif
-void
-disk_init(void)
-{
- (void) 0;
-}
-
/* hack needed on SunStudio to avoid linkage convention mismatch */
static void cxx_xfree(void *ptr)
{
return -1;
}
+int
+fsBlockSize(const char *path, int *blksize)
+{
+ struct statvfs sfs;
+
+ if (xstatvfs(path, &sfs)) {
+ debugs(50, DBG_IMPORTANT, "" << path << ": " << xstrerror());
+ *blksize = 2048;
+ return 1;
+ }
+
+ *blksize = (int) sfs.f_frsize;
+
+ // Sanity check; make sure we have a meaningful value.
+ if (*blksize < 512)
+ *blksize = 2048;
+
+ return 0;
+}
+
+#define fsbtoblk(num, fsbs, bs) \
+ (((fsbs) != 0 && (fsbs) < (bs)) ? \
+ (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
+int
+fsStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
+{
+ struct statvfs sfs;
+
+ if (xstatvfs(path, &sfs)) {
+ debugs(50, DBG_IMPORTANT, "" << path << ": " << xstrerror());
+ return 1;
+ }
+
+ *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024);
+ *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_frsize, 1024);
+ *totl_in = (int) sfs.f_files;
+ *free_in = (int) sfs.f_ffree;
+ return 0;
+}
/* DEBUG: section 06 Disk I/O Routines */
-#ifndef SQUID_DISK_H_
-#define SQUID_DISK_H_
+#ifndef SQUID_FS_IO_H_
+#define SQUID_FS_IO_H_
#include "mem/forward.h"
#include "typedefs.h" //DRCB, DWCB
int file_open(const char *path, int mode);
void file_close(int fd);
-
-/* Adapter file_write for object callbacks */
-template <class O>
-void
-FreeObject(void *address)
-{
- O *anObject = static_cast <O *>(address);
- delete anObject;
-}
-
void file_write(int, off_t, void const *, int len, DWCB *, void *, FREE *);
void file_write_mbuf(int fd, off_t, MemBuf mb, DWCB * handler, void *handler_data);
void file_read(int, char *, int, off_t, DRCB *, void *);
-void disk_init(void);
void safeunlink(const char *path, int quiet);
-int xrename(const char *from, const char *to); //disk.cc
+int xrename(const char *from, const char *to);
+
+int fsBlockSize(const char *path, int *blksize);
+int fsStats(const char *, int *, int *, int *, int *);
-#endif /* SQUID_DISK_H_ */
+#endif /* SQUID_FS_IO_H_ */
#include "squid.h"
#include "CachePeer.h"
#include "cbdata.h"
-#include "disk.h"
+#include "fs_io.h"
#include "event.h"
#include "fde.h"
#include "FwdState.h"
#include "SquidTime.h"
#include "Store.h"
#include "StoreClient.h"
-#include "SwapDir.h"
#include "tools.h"
#include "URL.h"
#include "wordlist.h"
#include "StatCounters.h"
#include "Store.h"
#include "store_key_md5.h"
-#include "SwapDir.h"
#include "tools.h"
#include "wordlist.h"
#define SQUID_IPC_STORE_MAP_H
#include "Debug.h"
-#include "fs/forward.h"
#include "ipc/mem/FlexibleArray.h"
#include "ipc/mem/Pointer.h"
#include "ipc/ReadWriteLock.h"
#include "SBuf.h"
+#include "store/forward.h"
#include "store_key_md5.h"
#include "tools.h"
#ifndef SQUID_IPC_STORE_MAP_H
#define SQUID_IPC_STORE_MAP_H
-#include "fs/forward.h"
#include "ipc/mem/FlexibleArray.h"
#include "ipc/mem/Pointer.h"
#include "ipc/ReadWriteLock.h"
#include "SBuf.h"
+#include "store/forward.h"
#include "store_key_md5.h"
namespace Ipc
#include "CacheManager.h"
#include "CollapsedForwarding.h"
#include "comm/Connection.h"
+#include "fatal.h"
#include "globals.h"
#include "ipc/Kids.h"
#include "ipc/Messages.h"
#include "mgr/Forwarder.h"
#include "mgr/Request.h"
#include "mgr/Response.h"
-#include "SwapDir.h" /* XXX: scope boundary violation */
#if HAVE_DISKIO_MODULE_IPCIO
#include "DiskIO/IpcIo/IpcIoFile.h" /* XXX: scope boundary violation */
#endif
#include "base/TextException.h"
#include "ipc/mem/PagePool.h"
#include "ipc/mem/Pages.h"
-#include "SwapDir.h"
#include "tools.h"
// Uses a single PagePool instance, for now.
/* DEBUG: section 50 Log file handling */
#include "squid.h"
-#include "disk.h"
+#include "fs_io.h"
#include "fatal.h"
#include "fd.h"
#include "fde.h"
#include "squid.h"
#include "comm.h"
#include "comm/Connection.h"
-#include "disk.h"
+#include "fs_io.h"
#include "fatal.h"
#include "fd.h"
#include "log/File.h"
#include "comm.h"
#include "ConfigParser.h"
#include "CpuAffinity.h"
-#include "disk.h"
+#include "fs_io.h"
#include "DiskIO/DiskIOModule.h"
#include "dns/forward.h"
#include "errorpage.h"
#include "stat.h"
#include "StatCounters.h"
#include "Store.h"
+#include "store/Disks.h"
#include "store_log.h"
#include "StoreFileSystem.h"
-#include "SwapDir.h"
#include "tools.h"
#include "unlinkd.h"
#include "URL.h"
#endif
- if (!configured_once)
- disk_init(); /* disk_init must go before ipcache_init() */
-
ipcache_init();
fqdncache_init();
StoreFileSystem::SetupAllFs();
/* we may want the parsing process to set this up in the future */
- Store::Root(new StoreController);
+ Store::Init();
Auth::Init(); /* required for config parsing. NOP if !USE_AUTH */
Ip::ProbeTransport(); // determine IPv4 or IPv6 capabilities before parsing.
mimeFreeMemory();
errorClean();
#endif
- // clear StoreController
- Store::Root(NULL);
+ Store::FreeMemory();
fdDumpOpen();
#include "base/PackableStream.h"
#include "CacheDigest.h"
#include "ClientInfo.h"
-#include "disk.h"
+#include "fs_io.h"
#include "dlink.h"
#include "event.h"
#include "icmp/net_db.h"
/* DEBUG: section 25 MIME Parsing and Internal Icons */
#include "squid.h"
-#include "disk.h"
+#include "fs_io.h"
#include "fde.h"
#include "globals.h"
#include "HttpHdrCc.h"
#include "squid.h"
#include "anyp/PortCfg.h"
#include "comm/Connection.h"
-#include "disk.h"
+#include "fs_io.h"
#include "event.h"
#include "fd.h"
#include "fde.h"
#include "ssl/cert_validate_message.h"
#include "ssl/Config.h"
#include "ssl/helper.h"
-#include "SwapDir.h"
+#include "fs_io.h"
#include "wordlist.h"
LruMap<Ssl::CertValidationResponse> *Ssl::CertValidationHelper::HelperCache = NULL;
} else if (db_path_was_found) {
db_path_was_found = false;
int fs_block_size = 0;
- storeDirGetBlkSize(token, &fs_block_size);
+ fsBlockSize(token, &fs_block_size);
snprintf(buffer, sizeof(buffer), "%i", fs_block_size);
}
}
state->filter = filter;
sentry->lock("statObjects");
- state->theSearch = Store::Root().search(NULL, NULL);
+ state->theSearch = Store::Root().search();
eventAdd("statObjects", statObjects, state, 0.0, 1);
}
#include "store_key_md5.h"
#include "store_log.h"
#include "store_rebuild.h"
+#include "store/Controller.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
#include "StoreClient.h"
#include "StoreIOState.h"
#include "StoreMeta.h"
#include "StrList.h"
#include "swap_log_op.h"
-#include "SwapDir.h"
#include "tools.h"
#if USE_DELAY_POOLS
#include "DelayPools.h"
static std::stack<StoreEntry*> LateReleaseStack;
MemAllocator *StoreEntry::pool = NULL;
-StorePointer Store::CurrentRoot = NULL;
-
-void
-Store::Root(Store * aRoot)
-{
- CurrentRoot = aRoot;
-}
-
-void
-Store::Root(StorePointer aRoot)
-{
- Root(aRoot.getRaw());
-}
-
void
Store::Stats(StoreEntry * output)
{
Root().stat(*output);
}
-void
-Store::create()
-{}
-
-void
-Store::diskFull()
-{}
-
-void
-Store::sync()
-{}
-
-void
-Store::unlink(StoreEntry &)
-{
- fatal("Store::unlink on invalid Store\n");
-}
-
void
StoreEntry::makePublic()
{
return;
// Store::Root() is FATALly missing during shutdown
- if (e->swap_filen >= 0 && !shutting_down) {
- SwapDir &sd = dynamic_cast<SwapDir&>(*e->store());
- sd.disconnect(*e);
- }
+ if (e->swap_filen >= 0 && !shutting_down)
+ e->disk().disconnect(*e);
e->destroyMemObject();
StoreEntry::touch()
{
lastref = squid_curtime;
- Store::Root().reference(*this);
}
void
#define MAINTAIN_MAX_SCAN 1024
#define MAINTAIN_MAX_REMOVE 64
-/*
- * This routine is to be called by main loop in main.c.
- * It removes expired objects on only one bucket for each time called.
- *
- * This should get called 1/s from main().
- */
-void
-StoreController::maintain()
-{
- static time_t last_warn_time = 0;
-
- PROF_start(storeMaintainSwapSpace);
- swapDir->maintain();
-
- /* this should be emitted by the oversize dir, not globally */
-
- if (Store::Root().currentSize() > Store::Root().maxSize()) {
- if (squid_curtime - last_warn_time > 10) {
- debugs(20, DBG_CRITICAL, "WARNING: Disk space over limit: "
- << Store::Root().currentSize() / 1024.0 << " KB > "
- << (Store::Root().maxSize() >> 10) << " KB");
- last_warn_time = squid_curtime;
- }
- }
-
- PROF_stop(storeMaintainSwapSpace);
-}
-
/* release an object from a cache */
void
StoreEntry::release()
return;
}
- Store::Root().memoryUnlink(*this);
+ if (Store::Controller::store_dirs_rebuilding && swap_filen > -1) {
+ /* TODO: Teach disk stores to handle releases during rebuild instead. */
- if (StoreController::store_dirs_rebuilding && swap_filen > -1) {
- setPrivateKey();
+ Store::Root().memoryUnlink(*this);
- if (swap_filen > -1) {
- // lock the entry until rebuilding is done
- lock("storeLateRelease");
- setReleaseFlag();
- LateReleaseStack.push(this);
- } else {
- destroyStoreEntry(static_cast<hash_link *>(this));
- // "this" is no longer valid
- }
+ setPrivateKey();
- PROF_stop(storeRelease);
+ // lock the entry until rebuilding is done
+ lock("storeLateRelease");
+ setReleaseFlag();
+ LateReleaseStack.push(this);
return;
}
storeLog(STORE_LOG_RELEASE, this);
-
- if (swap_filen > -1) {
+ if (swap_filen > -1 && !EBIT_TEST(flags, KEY_PRIVATE)) {
// log before unlink() below clears swap_filen
- if (!EBIT_TEST(flags, KEY_PRIVATE))
- storeDirSwapLog(this, SWAP_LOG_DEL);
-
- unlink();
+ storeDirSwapLog(this, SWAP_LOG_DEL);
}
+ Store::Root().unlink(*this);
destroyStoreEntry(static_cast<hash_link *>(this));
PROF_stop(storeRelease);
}
StoreEntry *e;
static int n = 0;
- if (StoreController::store_dirs_rebuilding) {
+ if (Store::Controller::store_dirs_rebuilding) {
eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
return;
}
void
storeFreeMemory(void)
{
- Store::Root(NULL);
+ Store::FreeMemory();
#if USE_CACHE_DIGESTS
if (store_digest)
return matched;
}
-SwapDir::Pointer
-StoreEntry::store() const
+Store::Disk &
+StoreEntry::disk() const
{
assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured);
- return INDEXSD(swap_dirn);
-}
-
-void
-StoreEntry::unlink()
-{
- store()->unlink(*this); // implies disconnect()
- swap_filen = -1;
- swap_dirn = -1;
- swap_status = SWAPOUT_NONE;
+ const RefCount<Store::Disk> &sd = INDEXSD(swap_dirn);
+ assert(sd);
+ return *sd;
}
/*
{
return NULL;
}
-
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_CONTROLLED_H
+#define SQUID_STORE_CONTROLLED_H
+
+#include "store/Storage.h"
+
+namespace Store {
+
+/// Storage controlled by a Controller.
+/// This API is shared among Disks, Disk, Memory caches and Transients.
+class Controlled: public Storage
+{
+public:
+ /// somebody needs this entry (many cache replacement policies need to know)
+ virtual void reference(StoreEntry &e) = 0;
+
+ /// somebody no longer needs this entry (usually after calling reference())
+ /// return false iff the idle entry should be destroyed
+ virtual bool dereference(StoreEntry &e) = 0;
+
+ /// If this storage cannot cache collapsed entries, return false.
+ /// If the entry is not found, return false. Otherwise, return true after
+ /// tying the entry to this cache and setting inSync to updateCollapsed().
+ virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
+
+ /// Update a local collapsed entry with fresh info from this cache (if any).
+ /// Return true iff the cache supports collapsed entries and
+ /// the given local collapsed entry is now in sync with this storage.
+ virtual bool updateCollapsed(StoreEntry &) { return false; }
+};
+
+} // namespace Store
+
+#endif /* SQUID_STORE_CONTROLLED_H */
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+/* DEBUG: section 20 Store Controller */
+
+#include "squid.h"
+#include "mem_node.h"
+#include "MemStore.h"
+#include "profiler/Profiler.h"
+#include "SquidConfig.h"
+#include "SquidMath.h"
+#include "store/Controller.h"
+#include "store/Disks.h"
+#include "store/LocalSearch.h"
+#include "tools.h"
+#include "Transients.h"
+
+#if HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
+/*
+ * store_dirs_rebuilding is initialized to _1_ as a hack so that
+ * storeDirWriteCleanLogs() doesn't try to do anything unless _all_
+ * cache_dirs have been read. For example, without this hack, Squid
+ * will try to write clean log files if -kparse fails (becasue it
+ * calls fatal()).
+ */
+int Store::Controller::store_dirs_rebuilding = 1;
+
+Store::Controller::Controller() :
+ swapDir(new Disks),
+ memStore(NULL),
+ transients(NULL)
+{
+ assert(!store_table);
+}
+
+Store::Controller::~Controller()
+{
+ delete memStore;
+ delete transients;
+ delete swapDir;
+
+ if (store_table) {
+ hashFreeItems(store_table, destroyStoreEntry);
+ hashFreeMemory(store_table);
+ store_table = nullptr;
+ }
+}
+
+void
+Store::Controller::init()
+{
+ if (Config.memShared && IamWorkerProcess()) {
+ memStore = new MemStore;
+ memStore->init();
+ }
+
+ swapDir->init();
+
+ if (UsingSmp() && IamWorkerProcess() && Config.onoff.collapsed_forwarding) {
+ transients = new Transients;
+ transients->init();
+ }
+}
+
+void
+Store::Controller::create()
+{
+ swapDir->create();
+
+#if !_SQUID_WINDOWS_
+
+ pid_t pid;
+
+ do {
+ int status;
+#if _SQUID_NEXT_
+
+ pid = wait3(&status, WNOHANG, NULL);
+#else
+
+ pid = waitpid(-1, &status, 0);
+#endif
+
+ } while (pid > 0 || (pid < 0 && errno == EINTR));
+
+#endif
+}
+
+void
+Store::Controller::maintain()
+{
+ static time_t last_warn_time = 0;
+
+ PROF_start(storeMaintainSwapSpace);
+ swapDir->maintain();
+
+ /* this should be emitted by the oversize dir, not globally */
+
+ if (Root().currentSize() > Store::Root().maxSize()) {
+ if (squid_curtime - last_warn_time > 10) {
+ debugs(20, DBG_CRITICAL, "WARNING: Disk space over limit: "
+ << Store::Root().currentSize() / 1024.0 << " KB > "
+ << (Store::Root().maxSize() >> 10) << " KB");
+ last_warn_time = squid_curtime;
+ }
+ }
+
+ PROF_stop(storeMaintainSwapSpace);
+}
+
+void
+Store::Controller::getStats(StoreInfoStats &stats) const
+{
+ if (memStore)
+ memStore->getStats(stats);
+ else {
+ // move this code to a non-shared memory cache class when we have it
+ stats.mem.shared = false;
+ stats.mem.capacity = Config.memMaxSize;
+ stats.mem.size = mem_node::StoreMemSize();
+ stats.mem.count = hot_obj_count;
+ }
+
+ swapDir->getStats(stats);
+
+ // low-level info not specific to memory or disk cache
+ stats.store_entry_count = StoreEntry::inUseCount();
+ stats.mem_object_count = MemObject::inUseCount();
+}
+
+void
+Store::Controller::stat(StoreEntry &output) const
+{
+ storeAppendPrintf(&output, "Store Directory Statistics:\n");
+ storeAppendPrintf(&output, "Store Entries : %lu\n",
+ (unsigned long int)StoreEntry::inUseCount());
+ storeAppendPrintf(&output, "Maximum Swap Size : %" PRIu64 " KB\n",
+ maxSize() >> 10);
+ storeAppendPrintf(&output, "Current Store Swap Size: %.2f KB\n",
+ currentSize() / 1024.0);
+ storeAppendPrintf(&output, "Current Capacity : %.2f%% used, %.2f%% free\n",
+ Math::doublePercent(currentSize(), maxSize()),
+ Math::doublePercent((maxSize() - currentSize()), maxSize()));
+
+ if (memStore)
+ memStore->stat(output);
+
+ /* now the swapDir */
+ swapDir->stat(output);
+}
+
+/* if needed, this could be taught to cache the result */
+uint64_t
+Store::Controller::maxSize() const
+{
+ /* TODO: include memory cache ? */
+ return swapDir->maxSize();
+}
+
+uint64_t
+Store::Controller::minSize() const
+{
+ /* TODO: include memory cache ? */
+ return swapDir->minSize();
+}
+
+uint64_t
+Store::Controller::currentSize() const
+{
+ /* TODO: include memory cache ? */
+ return swapDir->currentSize();
+}
+
+uint64_t
+Store::Controller::currentCount() const
+{
+ /* TODO: include memory cache ? */
+ return swapDir->currentCount();
+}
+
+int64_t
+Store::Controller::maxObjectSize() const
+{
+ /* TODO: include memory cache ? */
+ return swapDir->maxObjectSize();
+}
+
+StoreSearch *
+Store::Controller::search()
+{
+ // this is the only kind of search we currently support
+ return NewLocalSearch();
+}
+
+void
+Store::Controller::sync(void)
+{
+ if (memStore)
+ memStore->sync();
+ swapDir->sync();
+}
+
+/*
+ * handle callbacks all avaliable fs'es
+ */
+int
+Store::Controller::callback()
+{
+ /* This will likely double count. Thats ok. */
+ PROF_start(storeDirCallback);
+
+ /* mem cache callbacks ? */
+ int result = swapDir->callback();
+
+ PROF_stop(storeDirCallback);
+
+ return result;
+}
+
+void
+Store::Controller::referenceBusy(StoreEntry &e)
+{
+ // special entries do not belong to any specific Store, but are IN_MEMORY
+ if (EBIT_TEST(e.flags, ENTRY_SPECIAL))
+ return;
+
+ /* Notify the fs that we're referencing this object again */
+
+ if (e.swap_dirn > -1)
+ swapDir->reference(e);
+
+ // Notify the memory cache that we're referencing this object again
+ if (memStore && e.mem_status == IN_MEMORY)
+ memStore->reference(e);
+
+ // TODO: move this code to a non-shared memory cache class when we have it
+ if (e.mem_obj) {
+ if (mem_policy->Referenced)
+ mem_policy->Referenced(mem_policy, &e, &e.mem_obj->repl);
+ }
+}
+
+bool
+Store::Controller::dereferenceIdle(StoreEntry &e, bool wantsLocalMemory)
+{
+ // special entries do not belong to any specific Store, but are IN_MEMORY
+ if (EBIT_TEST(e.flags, ENTRY_SPECIAL))
+ return true;
+
+ bool keepInStoreTable = false; // keep only if somebody needs it there
+
+ /* Notify the fs that we're not referencing this object any more */
+
+ if (e.swap_filen > -1)
+ keepInStoreTable = swapDir->dereference(e) || keepInStoreTable;
+
+ // Notify the memory cache that we're not referencing this object any more
+ if (memStore && e.mem_status == IN_MEMORY)
+ keepInStoreTable = memStore->dereference(e) || keepInStoreTable;
+
+ // TODO: move this code to a non-shared memory cache class when we have it
+ if (e.mem_obj) {
+ if (mem_policy->Dereferenced)
+ mem_policy->Dereferenced(mem_policy, &e, &e.mem_obj->repl);
+ // non-shared memory cache relies on store_table
+ if (!memStore)
+ keepInStoreTable = wantsLocalMemory || keepInStoreTable;
+ }
+
+ return keepInStoreTable;
+}
+
+StoreEntry *
+Store::Controller::get(const cache_key *key)
+{
+ if (StoreEntry *e = find(key)) {
+ // this is not very precise: some get()s are not initiated by clients
+ e->touch();
+ referenceBusy(*e);
+ return e;
+ }
+ return NULL;
+}
+
+/// Internal method to implements the guts of the Store::get() API:
+/// returns an in-transit or cached object with a given key, if any.
+StoreEntry *
+Store::Controller::find(const cache_key *key)
+{
+ debugs(20, 3, storeKeyText(key));
+
+ if (StoreEntry *e = static_cast<StoreEntry*>(hash_lookup(store_table, key))) {
+ // TODO: ignore and maybe handleIdleEntry() unlocked intransit entries
+ // because their backing store slot may be gone already.
+ debugs(20, 3, HERE << "got in-transit entry: " << *e);
+ return e;
+ }
+
+ // Must search transients before caches because we must sync those we find.
+ if (transients) {
+ if (StoreEntry *e = transients->get(key)) {
+ debugs(20, 3, "got shared in-transit entry: " << *e);
+ bool inSync = false;
+ const bool found = anchorCollapsed(*e, inSync);
+ if (!found || inSync)
+ return e;
+ assert(!e->locked()); // ensure release will destroyStoreEntry()
+ e->release(); // do not let others into the same trap
+ return NULL;
+ }
+ }
+
+ if (memStore) {
+ if (StoreEntry *e = memStore->get(key)) {
+ debugs(20, 3, HERE << "got mem-cached entry: " << *e);
+ return e;
+ }
+ }
+
+ if (swapDir) {
+ if (StoreEntry *e = swapDir->get(key)) {
+ debugs(20, 3, "got disk-cached entry: " << *e);
+ return e;
+ }
+ }
+
+ debugs(20, 4, "cannot locate " << storeKeyText(key));
+ return nullptr;
+}
+
+void
+Store::Controller::markForUnlink(StoreEntry &e)
+{
+ if (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0)
+ transients->markForUnlink(e);
+ if (memStore && e.mem_obj && e.mem_obj->memCache.index >= 0)
+ memStore->markForUnlink(e);
+ if (swapDir && e.swap_filen >= 0)
+ swapDir->markForUnlink(e);
+}
+
+void
+Store::Controller::unlink(StoreEntry &e)
+{
+ memoryUnlink(e);
+ if (swapDir && e.swap_filen >= 0)
+ swapDir->unlink(e);
+}
+
+// move this into [non-shared] memory cache class when we have one
+/// whether e should be kept in local RAM for possible future caching
+bool
+Store::Controller::keepForLocalMemoryCache(StoreEntry &e) const
+{
+ if (!e.memoryCachable())
+ return false;
+
+ // does the current and expected size obey memory caching limits?
+ assert(e.mem_obj);
+ const int64_t loadedSize = e.mem_obj->endOffset();
+ const int64_t expectedSize = e.mem_obj->expectedReplySize(); // may be < 0
+ const int64_t ramSize = max(loadedSize, expectedSize);
+ const int64_t ramLimit = min(
+ static_cast<int64_t>(Config.memMaxSize),
+ static_cast<int64_t>(Config.Store.maxInMemObjSize));
+ return ramSize <= ramLimit;
+}
+
+void
+Store::Controller::memoryOut(StoreEntry &e, const bool preserveSwappable)
+{
+ bool keepInLocalMemory = false;
+ if (memStore)
+ memStore->write(e); // leave keepInLocalMemory false
+ else
+ keepInLocalMemory = keepForLocalMemoryCache(e);
+
+ debugs(20, 7, HERE << "keepInLocalMemory: " << keepInLocalMemory);
+
+ if (!keepInLocalMemory)
+ e.trimMemory(preserveSwappable);
+}
+
+void
+Store::Controller::memoryUnlink(StoreEntry &e)
+{
+ if (memStore)
+ memStore->unlink(e);
+ else // TODO: move into [non-shared] memory cache class when we have one
+ e.destroyMemObject();
+}
+
+void
+Store::Controller::memoryDisconnect(StoreEntry &e)
+{
+ if (memStore)
+ memStore->disconnect(e);
+ // else nothing to do for non-shared memory cache
+}
+
+void
+Store::Controller::transientsAbandon(StoreEntry &e)
+{
+ if (transients) {
+ assert(e.mem_obj);
+ if (e.mem_obj->xitTable.index >= 0)
+ transients->abandon(e);
+ }
+}
+
+void
+Store::Controller::transientsCompleteWriting(StoreEntry &e)
+{
+ if (transients) {
+ assert(e.mem_obj);
+ if (e.mem_obj->xitTable.index >= 0)
+ transients->completeWriting(e);
+ }
+}
+
+int
+Store::Controller::transientReaders(const StoreEntry &e) const
+{
+ return (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0) ?
+ transients->readers(e) : 0;
+}
+
+void
+Store::Controller::transientsDisconnect(MemObject &mem_obj)
+{
+ if (transients)
+ transients->disconnect(mem_obj);
+}
+
+void
+Store::Controller::handleIdleEntry(StoreEntry &e)
+{
+ bool keepInLocalMemory = false;
+
+ if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) {
+ // Icons (and cache digests?) should stay in store_table until we
+ // have a dedicated storage for them (that would not purge them).
+ // They are not managed [well] by any specific Store handled below.
+ keepInLocalMemory = true;
+ } else if (memStore) {
+ // leave keepInLocalMemory false; memStore maintains its own cache
+ } else {
+ keepInLocalMemory = keepForLocalMemoryCache(e) && // in good shape and
+ // the local memory cache is not overflowing
+ (mem_node::InUseCount() <= store_pages_max);
+ }
+
+ // An idle, unlocked entry that only belongs to a SwapDir which controls
+ // its own index, should not stay in the global store_table.
+ if (!dereferenceIdle(e, keepInLocalMemory)) {
+ debugs(20, 5, HERE << "destroying unlocked entry: " << &e << ' ' << e);
+ destroyStoreEntry(static_cast<hash_link*>(&e));
+ return;
+ }
+
+ debugs(20, 5, HERE << "keepInLocalMemory: " << keepInLocalMemory);
+
+ // TODO: move this into [non-shared] memory cache class when we have one
+ if (keepInLocalMemory) {
+ e.setMemStatus(IN_MEMORY);
+ e.mem_obj->unlinkRequest();
+ } else {
+ e.purgeMem(); // may free e
+ }
+}
+
+void
+Store::Controller::allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags,
+ const HttpRequestMethod &reqMethod)
+{
+ e->makePublic(); // this is needed for both local and SMP collapsing
+ if (transients)
+ transients->startWriting(e, reqFlags, reqMethod);
+ debugs(20, 3, "may " << (transients && e->mem_obj->xitTable.index >= 0 ?
+ "SMP-" : "locally-") << "collapse " << *e);
+}
+
+void
+Store::Controller::syncCollapsed(const sfileno xitIndex)
+{
+ assert(transients);
+
+ StoreEntry *collapsed = transients->findCollapsed(xitIndex);
+ if (!collapsed) { // the entry is no longer locally active, ignore update
+ debugs(20, 7, "not SMP-syncing not-transient " << xitIndex);
+ return;
+ }
+ assert(collapsed->mem_obj);
+ assert(collapsed->mem_obj->smpCollapsed);
+
+ debugs(20, 7, "syncing " << *collapsed);
+
+ bool abandoned = transients->abandoned(*collapsed);
+ bool found = false;
+ bool inSync = false;
+ if (memStore && collapsed->mem_obj->memCache.io == MemObject::ioDone) {
+ found = true;
+ inSync = true;
+ debugs(20, 7, "fully mem-loaded " << *collapsed);
+ } else if (memStore && collapsed->mem_obj->memCache.index >= 0) {
+ found = true;
+ inSync = memStore->updateCollapsed(*collapsed);
+ } else if (swapDir && collapsed->swap_filen >= 0) {
+ found = true;
+ inSync = swapDir->updateCollapsed(*collapsed);
+ } else {
+ found = anchorCollapsed(*collapsed, inSync);
+ }
+
+ if (abandoned && collapsed->store_status == STORE_PENDING) {
+ debugs(20, 3, "aborting abandoned but STORE_PENDING " << *collapsed);
+ collapsed->abort();
+ return;
+ }
+
+ if (inSync) {
+ debugs(20, 5, "synced " << *collapsed);
+ collapsed->invokeHandlers();
+ } else if (found) { // unrecoverable problem syncing this entry
+ debugs(20, 3, "aborting unsyncable " << *collapsed);
+ collapsed->abort();
+ } else { // the entry is still not in one of the caches
+ debugs(20, 7, "waiting " << *collapsed);
+ }
+}
+
+/// Called for in-transit entries that are not yet anchored to a cache.
+/// For cached entries, return true after synchronizing them with their cache
+/// (making inSync true on success). For not-yet-cached entries, return false.
+bool
+Store::Controller::anchorCollapsed(StoreEntry &collapsed, bool &inSync)
+{
+ // this method is designed to work with collapsed transients only
+ assert(collapsed.mem_obj);
+ assert(collapsed.mem_obj->xitTable.index >= 0);
+ assert(collapsed.mem_obj->smpCollapsed);
+
+ debugs(20, 7, "anchoring " << collapsed);
+
+ bool found = false;
+ if (memStore)
+ found = memStore->anchorCollapsed(collapsed, inSync);
+ if (!found && swapDir)
+ found = swapDir->anchorCollapsed(collapsed, inSync);
+
+ if (found) {
+ if (inSync)
+ debugs(20, 7, "anchored " << collapsed);
+ else
+ debugs(20, 5, "failed to anchor " << collapsed);
+ } else {
+ debugs(20, 7, "skipping not yet cached " << collapsed);
+ }
+
+ return found;
+}
+
+namespace Store {
+ static RefCount<Controller> TheRoot;
+}
+
+Store::Controller&
+Store::Root()
+{
+ assert(TheRoot);
+ return *TheRoot;
+}
+
+void
+Store::Init(Controller *root)
+{
+ TheRoot = root ? root : new Controller;
+}
+
+void
+Store::FreeMemory()
+{
+ TheRoot = nullptr;
+}
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_CONTROLLER_H
+#define SQUID_STORE_CONTROLLER_H
+
+#include "store/Storage.h"
+
+class MemObject;
+class RequestFlags;
+class HttpRequestMethod;
+
+namespace Store {
+
+/// Public Store interface. Coordinates the work of memory/disk/transient stores
+/// and hides their individual existence/differences from the callers.
+class Controller: public Storage
+{
+public:
+ Controller();
+ virtual ~Controller() override;
+
+ /* Storage API */
+ virtual void create() override;
+ virtual void init() override;
+ virtual StoreEntry *get(const cache_key *) override;
+ virtual uint64_t maxSize() const override;
+ virtual uint64_t minSize() const override;
+ virtual uint64_t currentSize() const override;
+ virtual uint64_t currentCount() const override;
+ virtual int64_t maxObjectSize() const override;
+ virtual void getStats(StoreInfoStats &stats) const override;
+ virtual void stat(StoreEntry &) const override;
+ virtual void sync() override;
+ virtual void maintain() override;
+ virtual void markForUnlink(StoreEntry &) override;
+ virtual void unlink(StoreEntry &) override;
+ virtual int callback() override;
+
+ /// called when the entry is no longer needed by any transaction
+ void handleIdleEntry(StoreEntry &);
+
+ /// called to get rid of no longer needed entry data in RAM, if any
+ void memoryOut(StoreEntry &, const bool preserveSwappable);
+
+ /// makes the entry available for collapsing future requests
+ void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &);
+
+ /// marks the entry completed for collapsed requests
+ void transientsCompleteWriting(StoreEntry &);
+
+ /// Update local intransit entry after changes made by appending worker.
+ void syncCollapsed(const sfileno);
+
+ /// calls Root().transients->abandon() if transients are tracked
+ void transientsAbandon(StoreEntry &);
+
+ /// number of the transient entry readers some time ago
+ int transientReaders(const StoreEntry &) const;
+
+ /// disassociates the entry from the intransit table
+ void transientsDisconnect(MemObject &);
+
+ /// removes the entry from the memory cache
+ void memoryUnlink(StoreEntry &);
+
+ /// disassociates the entry from the memory cache, preserving cached data
+ void memoryDisconnect(StoreEntry &);
+
+ /// \returns an iterator for all Store entries
+ StoreSearch *search();
+
+ /// the number of cache_dirs being rebuilt; TODO: move to Disks::Rebuilding
+ static int store_dirs_rebuilding;
+
+private:
+ /// update reference counters of the recently touched entry
+ void referenceBusy(StoreEntry &e);
+ /// dereference() an idle entry and return true if the entry should be deleted
+ bool dereferenceIdle(StoreEntry &, bool wantsLocalMemory);
+
+ StoreEntry *find(const cache_key *key);
+ bool keepForLocalMemoryCache(StoreEntry &e) const;
+ bool anchorCollapsed(StoreEntry &, bool &inSync);
+
+ Disks *swapDir; ///< summary view of all disk caches
+ Memory *memStore; ///< memory cache
+
+ /// A shared table of public store entries that do not know whether they
+ /// will belong to a memory cache, a disk cache, or will be uncachable
+ /// when the response header comes. Used for SMP collapsed forwarding.
+ Transients *transients;
+};
+
+/// safely access controller singleton
+extern Controller &Root();
+
+/// initialize the storage module; a custom root is used by unit tests only
+extern void Init(Controller *root = nullptr);
+
+/// undo Init()
+extern void FreeMemory();
+
+} // namespace Store
+
+#endif /* SQUID_STORE_CONTROLLER_H */
#include "Parsing.h"
#include "SquidConfig.h"
#include "StoreFileSystem.h"
-#include "SwapDir.h"
+#include "Store.h"
+#include "store/Disk.h"
#include "tools.h"
-SwapDir::SwapDir(char const *aType): theType(aType),
+Store::Disk::Disk(char const *aType): theType(aType),
max_size(0), min_objsize(0), max_objsize (-1),
path(NULL), index(-1), disker(-1),
repl(NULL), removals(0), scanned(0),
fs.blksize = 1024;
}
-SwapDir::~SwapDir()
+Store::Disk::~Disk()
{
// TODO: should we delete repl?
xfree(path);
}
void
-SwapDir::create() {}
+Store::Disk::create() {}
void
-SwapDir::dump(StoreEntry &)const {}
+Store::Disk::dump(StoreEntry &)const {}
bool
-SwapDir::doubleCheck(StoreEntry &)
+Store::Disk::doubleCheck(StoreEntry &)
{
return false;
}
void
-SwapDir::unlink(StoreEntry &) {}
-
-void
-SwapDir::getStats(StoreInfoStats &stats) const
+Store::Disk::getStats(StoreInfoStats &stats) const
{
if (!doReportStat())
return;
}
void
-SwapDir::stat(StoreEntry &output) const
+Store::Disk::stat(StoreEntry &output) const
{
if (!doReportStat())
return;
}
void
-SwapDir::statfs(StoreEntry &)const {}
+Store::Disk::statfs(StoreEntry &)const {}
void
-SwapDir::maintain() {}
+Store::Disk::maintain() {}
uint64_t
-SwapDir::minSize() const
+Store::Disk::minSize() const
{
+ // XXX: Not all disk stores use Config.Swap.lowWaterMark
return ((maxSize() * Config.Swap.lowWaterMark) / 100);
}
int64_t
-SwapDir::maxObjectSize() const
+Store::Disk::maxObjectSize() const
{
// per-store max-size=N value is authoritative
if (max_objsize > -1)
}
void
-SwapDir::maxObjectSize(int64_t newMax)
+Store::Disk::maxObjectSize(int64_t newMax)
{
// negative values mean no limit (-1)
if (newMax < 0) {
}
void
-SwapDir::reference(StoreEntry &) {}
+Store::Disk::reference(StoreEntry &) {}
bool
-SwapDir::dereference(StoreEntry &, bool)
+Store::Disk::dereference(StoreEntry &)
{
return true; // keep in global store_table
}
-int
-SwapDir::callback()
+void
+Store::Disk::diskFull()
{
- return 0;
+ if (currentSize() >= maxSize())
+ return;
+
+ max_size = currentSize();
+
+ debugs(20, DBG_IMPORTANT, "WARNING: Shrinking cache_dir #" << index << " to " << currentSize() / 1024.0 << " KB");
+}
+
+bool
+Store::Disk::objectSizeIsAcceptable(int64_t objsize) const
+{
+ // without limits, all object sizes are acceptable, including unknown ones
+ if (min_objsize <= 0 && max_objsize == -1)
+ return true;
+
+ // with limits, objects with unknown sizes are not acceptable
+ if (objsize == -1)
+ return false;
+
+ // without the upper limit, just check the lower limit
+ if (max_objsize == -1)
+ return min_objsize <= objsize;
+
+ return min_objsize <= objsize && objsize < max_objsize;
}
bool
-SwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
+Store::Disk::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const
{
debugs(47,8, HERE << "cache_dir[" << index << "]: needs " <<
diskSpaceNeeded << " <? " << max_objsize);
return true; // kids may provide more tests and should report true load
}
-void
-SwapDir::sync() {}
-
/* Move to StoreEntry ? */
bool
-SwapDir::canLog(StoreEntry const &e)const
+Store::Disk::canLog(StoreEntry const &e)const
{
if (e.swap_filen < 0)
return false;
}
void
-SwapDir::openLog() {}
+Store::Disk::openLog() {}
void
-SwapDir::closeLog() {}
+Store::Disk::closeLog() {}
int
-SwapDir::writeCleanStart()
+Store::Disk::writeCleanStart()
{
return 0;
}
void
-SwapDir::writeCleanDone() {}
+Store::Disk::writeCleanDone() {}
void
-SwapDir::logEntry(const StoreEntry &, int) const {}
+Store::Disk::logEntry(const StoreEntry &, int) const {}
char const *
-SwapDir::type() const
+Store::Disk::type() const
{
return theType;
}
bool
-SwapDir::active() const
+Store::Disk::active() const
{
if (IamWorkerProcess())
return true;
}
bool
-SwapDir::needsDiskStrand() const
+Store::Disk::needsDiskStrand() const
{
return false;
}
* - RBC 20030718
*/
ConfigOption *
-SwapDir::getOptionTree() const
+Store::Disk::getOptionTree() const
{
ConfigOptionVector *result = new ConfigOptionVector;
- result->options.push_back(new ConfigOptionAdapter<SwapDir>(*const_cast<SwapDir *>(this), &SwapDir::optionReadOnlyParse, &SwapDir::optionReadOnlyDump));
- result->options.push_back(new ConfigOptionAdapter<SwapDir>(*const_cast<SwapDir *>(this), &SwapDir::optionObjectSizeParse, &SwapDir::optionObjectSizeDump));
+ result->options.push_back(new ConfigOptionAdapter<Disk>(*const_cast<Disk*>(this), &Store::Disk::optionReadOnlyParse, &Store::Disk::optionReadOnlyDump));
+ result->options.push_back(new ConfigOptionAdapter<Disk>(*const_cast<Disk*>(this), &Store::Disk::optionObjectSizeParse, &Store::Disk::optionObjectSizeDump));
return result;
}
void
-SwapDir::parseOptions(int isaReconfig)
+Store::Disk::parseOptions(int isaReconfig)
{
const bool old_read_only = flags.read_only;
char *name, *value;
++value;
}
- debugs(3,2, "SwapDir::parseOptions: parsing store option '" << name << "'='" << (value ? value : "") << "'");
+ debugs(3,2, "cache_dir " << name << '=' << (value ? value : ""));
if (newOption)
if (!newOption->parse(name, value, isaReconfig))
}
void
-SwapDir::dumpOptions(StoreEntry * entry) const
+Store::Disk::dumpOptions(StoreEntry * entry) const
{
ConfigOption *newOption = getOptionTree();
}
bool
-SwapDir::optionReadOnlyParse(char const *option, const char *value, int)
+Store::Disk::optionReadOnlyParse(char const *option, const char *value, int)
{
if (strcmp(option, "no-store") != 0 && strcmp(option, "read-only") != 0)
return false;
}
void
-SwapDir::optionReadOnlyDump(StoreEntry * e) const
+Store::Disk::optionReadOnlyDump(StoreEntry * e) const
{
if (flags.read_only)
storeAppendPrintf(e, " no-store");
}
bool
-SwapDir::optionObjectSizeParse(char const *option, const char *value, int isaReconfig)
+Store::Disk::optionObjectSizeParse(char const *option, const char *value, int isaReconfig)
{
int64_t *val;
if (strcmp(option, "max-size") == 0) {
}
void
-SwapDir::optionObjectSizeDump(StoreEntry * e) const
+Store::Disk::optionObjectSizeDump(StoreEntry * e) const
{
if (min_objsize != 0)
storeAppendPrintf(e, " min-size=%" PRId64, min_objsize);
// some SwapDirs may maintain their indexes and be able to lookup an entry key
StoreEntry *
-SwapDir::get(const cache_key *)
+Store::Disk::get(const cache_key *)
{
return NULL;
}
-
-void
-SwapDir::get(String const, STOREGETCLIENT, void *)
-{
- fatal("not implemented");
-}
-
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_DISK_H
+#define SQUID_STORE_DISK_H
+
+#include "store/Controlled.h"
+#include "StoreIOState.h"
+
+
+class ConfigOption;
+class RemovalPolicy;
+
+namespace Store {
+
+/// manages a single cache_dir
+class Disk: public Controlled
+{
+
+public:
+ typedef RefCount<Disk> Pointer;
+
+ explicit Disk(char const *aType);
+ virtual ~Disk();
+ virtual void reconfigure() = 0;
+ char const *type() const;
+
+ virtual bool needsDiskStrand() const; ///< needs a dedicated kid process
+ virtual bool active() const; ///< may be used in this strand
+ /// whether stat should be reported by this SwapDir
+ virtual bool doReportStat() const { return active(); }
+ /// whether SwapDir may benefit from unlinkd
+ virtual bool unlinkdUseful() const = 0;
+
+ /**
+ * Notify this disk that it is full.
+ \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
+ */
+ virtual void diskFull();
+
+ /* Controlled API */
+ virtual void create() override;
+ virtual void init() override = 0;
+ virtual StoreEntry *get(const cache_key *) override;
+ virtual uint64_t maxSize() const override { return max_size; }
+ virtual uint64_t minSize() const override;
+ virtual uint64_t currentSize() const override = 0;
+ virtual uint64_t currentCount() const override = 0;
+ virtual int64_t maxObjectSize() const override;
+ virtual void getStats(StoreInfoStats &stats) const override;
+ virtual void stat(StoreEntry &) const override;
+ virtual void reference(StoreEntry &e) override;
+ virtual bool dereference(StoreEntry &e) override;
+ virtual void maintain() override;
+ virtual bool anchorCollapsed(StoreEntry &e, bool &inSync) override = 0;
+ virtual bool updateCollapsed(StoreEntry &e) override = 0;
+ virtual void markForUnlink(StoreEntry &) override = 0;
+ virtual void unlink(StoreEntry &) override = 0;
+
+ /// configure the maximum object size for this storage area.
+ /// May be any size up to the total storage area.
+ void maxObjectSize(int64_t newMax);
+
+ /// whether we can store an object of the given size
+ /// negative objSize means the object size is currently unknown
+ bool objectSizeIsAcceptable(int64_t objSize) const;
+
+ /// called when the entry is about to forget its association with cache_dir
+ virtual void disconnect(StoreEntry &) {}
+
+ /// called when entry swap out is complete
+ virtual void swappedOut(const StoreEntry &e) = 0;
+
+protected:
+ void parseOptions(int reconfiguring);
+ void dumpOptions(StoreEntry * e) const;
+ virtual ConfigOption *getOptionTree() const;
+ virtual bool allowOptionReconfigure(const char *const) const { return true; }
+
+ int64_t sizeInBlocks(const int64_t size) const { return (size + fs.blksize - 1) / fs.blksize; }
+
+private:
+ bool optionReadOnlyParse(char const *option, const char *value, int reconfiguring);
+ void optionReadOnlyDump(StoreEntry * e) const;
+ bool optionObjectSizeParse(char const *option, const char *value, int reconfiguring);
+ void optionObjectSizeDump(StoreEntry * e) const;
+ char const *theType;
+
+protected:
+ uint64_t max_size; ///< maximum allocatable size of the storage area
+ int64_t min_objsize; ///< minimum size of any object stored here (-1 for no limit)
+ int64_t max_objsize; ///< maximum size of any object stored here (-1 for no limit)
+
+public:
+ char *path;
+ int index; /* This entry's index into the swapDirs array */
+ int disker; ///< disker kid id dedicated to this SwapDir or -1
+ RemovalPolicy *repl;
+ int removals;
+ int scanned;
+
+ struct Flags {
+ Flags() : selected(false), read_only(false) {}
+ bool selected;
+ bool read_only;
+ } flags;
+
+ virtual void dump(StoreEntry &)const; /* Dump fs config snippet */
+ virtual bool doubleCheck(StoreEntry &); /* Double check the obj integrity */
+ virtual void statfs(StoreEntry &) const; /* Dump fs statistics */
+
+ /// check whether we can store the entry; if we can, report current load
+ virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const = 0;
+
+ virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) = 0;
+ virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) = 0;
+
+ bool canLog(StoreEntry const &e)const;
+ virtual void openLog();
+ virtual void closeLog();
+ virtual void logEntry(const StoreEntry & e, int op) const;
+
+ class CleanLog
+ {
+
+ public:
+ virtual ~CleanLog() {}
+
+ virtual const StoreEntry *nextEntry() = 0;
+ virtual void write(StoreEntry const &) = 0;
+ };
+
+ CleanLog *cleanLog;
+ virtual int writeCleanStart();
+ virtual void writeCleanDone();
+ virtual void parse(int index, char *path) = 0;
+
+ struct {
+ int blksize;
+ } fs;
+};
+
+} // namespace Store
+
+#endif /* SQUID_STORE_DISK_H */
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+/* DEBUG: section 47 Store Directory Routines */
+
+#include "squid.h"
+#include "Debug.h"
+#include "globals.h"
+#include "profiler/Profiler.h"
+#include "SquidConfig.h"
+#include "Store.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
+#include "swap_log_op.h"
+#include "util.h" // for tvSubDsec() which should be in SquidTime.h
+
+static STDIRSELECT storeDirSelectSwapDirRoundRobin;
+static STDIRSELECT storeDirSelectSwapDirLeastLoad;
+/*
+ * This function pointer is set according to 'store_dir_select_algorithm'
+ * in squid.conf.
+ */
+STDIRSELECT *storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
+
+/*
+ * This new selection scheme simply does round-robin on all SwapDirs.
+ * A SwapDir is skipped if it is over the max_size (100%) limit, or
+ * overloaded.
+ */
+static int
+storeDirSelectSwapDirRoundRobin(const StoreEntry * e)
+{
+ // e->objectLen() is negative at this point when we are still STORE_PENDING
+ ssize_t objsize = e->mem_obj->expectedReplySize();
+ if (objsize != -1)
+ objsize += e->mem_obj->swap_hdr_sz;
+
+ // Increment the first candidate once per selection (not once per
+ // iteration) to reduce bias when some disk(s) attract more entries.
+ static int firstCandidate = 0;
+ if (++firstCandidate >= Config.cacheSwap.n_configured)
+ firstCandidate = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ const int dirn = (firstCandidate + i) % Config.cacheSwap.n_configured;
+ const SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(dirn));
+
+ int load = 0;
+ if (!sd->canStore(*e, objsize, load))
+ continue;
+
+ if (load < 0 || load > 1000) {
+ continue;
+ }
+
+ return dirn;
+ }
+
+ return -1;
+}
+
+/*
+ * Spread load across all of the store directories
+ *
+ * Note: We should modify this later on to prefer sticking objects
+ * in the *tightest fit* swapdir to conserve space, along with the
+ * actual swapdir usage. But for now, this hack will do while
+ * testing, so you should order your swapdirs in the config file
+ * from smallest max-size= to largest max-size=.
+ *
+ * We also have to choose nleast == nconf since we need to consider
+ * ALL swapdirs, regardless of state. Again, this is a hack while
+ * we sort out the real usefulness of this algorithm.
+ */
+static int
+storeDirSelectSwapDirLeastLoad(const StoreEntry * e)
+{
+ int64_t most_free = 0;
+ ssize_t least_objsize = -1;
+ int least_load = INT_MAX;
+ int load;
+ int dirn = -1;
+ int i;
+ RefCount<SwapDir> SD;
+
+ // e->objectLen() is negative at this point when we are still STORE_PENDING
+ ssize_t objsize = e->mem_obj->expectedReplySize();
+
+ if (objsize != -1)
+ objsize += e->mem_obj->swap_hdr_sz;
+
+ for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ SD = dynamic_cast<SwapDir *>(INDEXSD(i));
+ SD->flags.selected = false;
+
+ if (!SD->canStore(*e, objsize, load))
+ continue;
+
+ if (load < 0 || load > 1000)
+ continue;
+
+ if (load > least_load)
+ continue;
+
+ const int64_t cur_free = SD->maxSize() - SD->currentSize();
+
+ /* If the load is equal, then look in more details */
+ if (load == least_load) {
+ /* closest max-size fit */
+
+ if (least_objsize != -1)
+ if (SD->maxObjectSize() > least_objsize)
+ continue;
+
+ /* most free */
+ if (cur_free < most_free)
+ continue;
+ }
+
+ least_load = load;
+ least_objsize = SD->maxObjectSize();
+ most_free = cur_free;
+ dirn = i;
+ }
+
+ if (dirn >= 0)
+ dynamic_cast<SwapDir *>(INDEXSD(dirn))->flags.selected = true;
+
+ return dirn;
+}
+
+SwapDir *
+Store::Disks::store(int const x) const
+{
+ return INDEXSD(x);
+}
+
+SwapDir &
+Store::Disks::dir(const int i) const
+{
+ SwapDir *sd = INDEXSD(i);
+ assert(sd);
+ return *sd;
+}
+
+int
+Store::Disks::callback()
+{
+ int result = 0;
+ int j;
+ static int ndir = 0;
+
+ do {
+ j = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (ndir >= Config.cacheSwap.n_configured)
+ ndir = ndir % Config.cacheSwap.n_configured;
+
+ int temp_result = store(ndir)->callback();
+
+ ++ndir;
+
+ j += temp_result;
+
+ result += temp_result;
+
+ if (j > 100)
+ fatal ("too much io\n");
+ }
+ } while (j > 0);
+
+ ++ndir;
+
+ return result;
+}
+
+void
+Store::Disks::create()
+{
+ if (Config.cacheSwap.n_configured == 0) {
+ debugs(0, DBG_PARSE_NOTE(DBG_CRITICAL), "No cache_dir stores are configured.");
+ }
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).active())
+ store(i)->create();
+ }
+}
+
+StoreEntry *
+Store::Disks::get(const cache_key *key)
+{
+ if (const int cacheDirs = Config.cacheSwap.n_configured) {
+ // ask each cache_dir until the entry is found; use static starting
+ // point to avoid asking the same subset of disks more often
+ // TODO: coordinate with put() to be able to guess the right disk often
+ static int idx = 0;
+ for (int n = 0; n < cacheDirs; ++n) {
+ idx = (idx + 1) % cacheDirs;
+ SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(idx));
+ if (!sd->active())
+ continue;
+
+ if (StoreEntry *e = sd->get(key)) {
+ debugs(20, 7, "cache_dir " << idx << " has: " << *e);
+ return e;
+ }
+ }
+ }
+
+ debugs(20, 6, "none of " << Config.cacheSwap.n_configured <<
+ " cache_dirs have " << storeKeyText(key));
+ return nullptr;
+}
+
+void
+Store::Disks::init()
+{
+ if (Config.Store.objectsPerBucket <= 0)
+ fatal("'store_objects_per_bucket' should be larger than 0.");
+
+ if (Config.Store.avgObjectSize <= 0)
+ fatal("'store_avg_object_size' should be larger than 0.");
+
+ /* Calculate size of hash table (maximum currently 64k buckets). */
+ /* this is very bogus, its specific to the any Store maintaining an
+ * in-core index, not global */
+ size_t buckets = (Store::Root().maxSize() + Config.memMaxSize) / Config.Store.avgObjectSize;
+ debugs(20, DBG_IMPORTANT, "Swap maxSize " << (Store::Root().maxSize() >> 10) <<
+ " + " << ( Config.memMaxSize >> 10) << " KB, estimated " << buckets << " objects");
+ buckets /= Config.Store.objectsPerBucket;
+ debugs(20, DBG_IMPORTANT, "Target number of buckets: " << buckets);
+ /* ideally the full scan period should be configurable, for the
+ * moment it remains at approximately 24 hours. */
+ store_hash_buckets = storeKeyHashBuckets(buckets);
+ debugs(20, DBG_IMPORTANT, "Using " << store_hash_buckets << " Store buckets");
+ debugs(20, DBG_IMPORTANT, "Max Mem size: " << ( Config.memMaxSize >> 10) << " KB" <<
+ (Config.memShared ? " [shared]" : ""));
+ debugs(20, DBG_IMPORTANT, "Max Swap size: " << (Store::Root().maxSize() >> 10) << " KB");
+
+ store_table = hash_create(storeKeyHashCmp,
+ store_hash_buckets, storeKeyHashHash);
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ /* this starts a search of the store dirs, loading their
+ * index. under the new Store api this should be
+ * driven by the StoreHashIndex, not by each store.
+ *
+ * That is, the HashIndex should perform a search of each dir it is
+ * indexing to do the hash insertions. The search is then able to
+ * decide 'from-memory', or 'from-clean-log' or 'from-dirty-log' or
+ * 'from-no-log'.
+ *
+ * Step 1: make the store rebuilds use a search internally
+ * Step 2: change the search logic to use the four modes described
+ * above
+ * Step 3: have the hash index walk the searches itself.
+ */
+ if (dir(i).active())
+ store(i)->init();
+ }
+
+ if (strcasecmp(Config.store_dir_select_algorithm, "round-robin") == 0) {
+ storeDirSelectSwapDir = storeDirSelectSwapDirRoundRobin;
+ debugs(47, DBG_IMPORTANT, "Using Round Robin store dir selection");
+ } else {
+ storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
+ debugs(47, DBG_IMPORTANT, "Using Least Load store dir selection");
+ }
+}
+
+uint64_t
+Store::Disks::maxSize() const
+{
+ uint64_t result = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).doReportStat())
+ result += store(i)->maxSize();
+ }
+
+ return result;
+}
+
+uint64_t
+Store::Disks::minSize() const
+{
+ uint64_t result = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).doReportStat())
+ result += store(i)->minSize();
+ }
+
+ return result;
+}
+
+uint64_t
+Store::Disks::currentSize() const
+{
+ uint64_t result = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).doReportStat())
+ result += store(i)->currentSize();
+ }
+
+ return result;
+}
+
+uint64_t
+Store::Disks::currentCount() const
+{
+ uint64_t result = 0;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).doReportStat())
+ result += store(i)->currentCount();
+ }
+
+ return result;
+}
+
+int64_t
+Store::Disks::maxObjectSize() const
+{
+ int64_t result = -1;
+
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ if (dir(i).active() && store(i)->maxObjectSize() > result)
+ result = store(i)->maxObjectSize();
+ }
+
+ return result;
+}
+
+void
+Store::Disks::getStats(StoreInfoStats &stats) const
+{
+ // accumulate per-disk cache stats
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ StoreInfoStats dirStats;
+ store(i)->getStats(dirStats);
+ stats += dirStats;
+ }
+
+ // common to all disks
+ stats.swap.open_disk_fd = store_open_disk_fd;
+
+ // memory cache stats are collected in StoreController::getStats(), for now
+}
+
+void
+Store::Disks::stat(StoreEntry & output) const
+{
+ int i;
+
+ /* Now go through each store, calling its stat routine */
+
+ for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ storeAppendPrintf(&output, "\n");
+ store(i)->stat(output);
+ }
+}
+
+void
+Store::Disks::reference(StoreEntry &e)
+{
+ e.disk().reference(e);
+}
+
+bool
+Store::Disks::dereference(StoreEntry &e)
+{
+ return e.disk().dereference(e);
+}
+
+void
+Store::Disks::maintain()
+{
+ int i;
+ /* walk each fs */
+
+ for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
+ /* XXX FixMe: This should be done "in parallell" on the different
+ * cache_dirs, not one at a time.
+ */
+ /* call the maintain function .. */
+ store(i)->maintain();
+ }
+}
+
+void
+Store::Disks::sync()
+{
+ for (int i = 0; i < Config.cacheSwap.n_configured; ++i)
+ store(i)->sync();
+}
+
+void
+Store::Disks::markForUnlink(StoreEntry &e) {
+ if (e.swap_filen >= 0)
+ store(e.swap_dirn)->markForUnlink(e);
+}
+
+void
+Store::Disks::unlink(StoreEntry &e) {
+ if (e.swap_filen >= 0)
+ store(e.swap_dirn)->unlink(e);
+}
+
+bool
+Store::Disks::anchorCollapsed(StoreEntry &collapsed, bool &inSync)
+{
+ if (const int cacheDirs = Config.cacheSwap.n_configured) {
+ // ask each cache_dir until the entry is found; use static starting
+ // point to avoid asking the same subset of disks more often
+ // TODO: coordinate with put() to be able to guess the right disk often
+ static int idx = 0;
+ for (int n = 0; n < cacheDirs; ++n) {
+ idx = (idx + 1) % cacheDirs;
+ SwapDir &sd = dir(idx);
+ if (!sd.active())
+ continue;
+
+ if (sd.anchorCollapsed(collapsed, inSync)) {
+ debugs(20, 3, "cache_dir " << idx << " anchors " << collapsed);
+ return true;
+ }
+ }
+ }
+
+ debugs(20, 4, "none of " << Config.cacheSwap.n_configured <<
+ " cache_dirs have " << collapsed);
+ return false;
+}
+
+bool
+Store::Disks::updateCollapsed(StoreEntry &collapsed)
+{
+ return collapsed.swap_filen >= 0 &&
+ dir(collapsed.swap_dirn).updateCollapsed(collapsed);
+}
+
+
+/* Store::Disks globals that should be converted to use RegisteredRunner */
+
+void
+storeDirOpenSwapLogs()
+{
+ for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
+ INDEXSD(dirn)->openLog();
+}
+
+void
+storeDirCloseSwapLogs()
+{
+ for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
+ INDEXSD(dirn)->closeLog();
+}
+
+/*
+ * storeDirWriteCleanLogs
+ *
+ * Writes a "clean" swap log file from in-memory metadata.
+ * This is a rewrite of the original function to troll each
+ * StoreDir and write the logs, and flush at the end of
+ * the run. Thanks goes to Eric Stern, since this solution
+ * came out of his COSS code.
+ */
+int
+storeDirWriteCleanLogs(int reopen)
+{
+ const StoreEntry *e = NULL;
+ int n = 0;
+
+ struct timeval start;
+ double dt;
+ RefCount<SwapDir> sd;
+ int dirn;
+ int notdone = 1;
+
+ // Check for store_dirs_rebuilding because fatal() often calls us in early
+ // initialization phases, before store log is initialized and ready. Also,
+ // some stores probably do not support log cleanup during Store rebuilding.
+ if (StoreController::store_dirs_rebuilding) {
+ debugs(20, DBG_IMPORTANT, "Not currently OK to rewrite swap log.");
+ debugs(20, DBG_IMPORTANT, "storeDirWriteCleanLogs: Operation aborted.");
+ return 0;
+ }
+
+ debugs(20, DBG_IMPORTANT, "storeDirWriteCleanLogs: Starting...");
+ getCurrentTime();
+ start = current_time;
+
+ for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) {
+ sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
+
+ if (sd->writeCleanStart() < 0) {
+ debugs(20, DBG_IMPORTANT, "log.clean.start() failed for dir #" << sd->index);
+ continue;
+ }
+ }
+
+ /*
+ * This may look inefficient as CPU wise it is more efficient to do this
+ * sequentially, but I/O wise the parallellism helps as it allows more
+ * hdd spindles to be active.
+ */
+ while (notdone) {
+ notdone = 0;
+
+ for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) {
+ sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
+
+ if (NULL == sd->cleanLog)
+ continue;
+
+ e = sd->cleanLog->nextEntry();
+
+ if (!e)
+ continue;
+
+ notdone = 1;
+
+ if (!sd->canLog(*e))
+ continue;
+
+ sd->cleanLog->write(*e);
+
+ if ((++n & 0xFFFF) == 0) {
+ getCurrentTime();
+ debugs(20, DBG_IMPORTANT, " " << std::setw(7) << n <<
+ " entries written so far.");
+ }
+ }
+ }
+
+ /* Flush */
+ for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
+ dynamic_cast<SwapDir *>(INDEXSD(dirn))->writeCleanDone();
+
+ if (reopen)
+ storeDirOpenSwapLogs();
+
+ getCurrentTime();
+
+ dt = tvSubDsec(start, current_time);
+
+ debugs(20, DBG_IMPORTANT, " Finished. Wrote " << n << " entries.");
+ debugs(20, DBG_IMPORTANT, " Took "<< std::setw(3)<< std::setprecision(2) << dt <<
+ " seconds ("<< std::setw(6) << ((double) n / (dt > 0.0 ? dt : 1.0)) << " entries/sec).");
+
+ return n;
+}
+
+/* Globals that should be converted to static Store::Disks methods */
+
+void
+allocate_new_swapdir(Store::DiskConfig *swap)
+{
+ if (swap->swapDirs == NULL) {
+ swap->n_allocated = 4;
+ swap->swapDirs = static_cast<SwapDir::Pointer *>(xcalloc(swap->n_allocated, sizeof(SwapDir::Pointer)));
+ }
+
+ if (swap->n_allocated == swap->n_configured) {
+ swap->n_allocated <<= 1;
+ SwapDir::Pointer *const tmp = static_cast<SwapDir::Pointer *>(xcalloc(swap->n_allocated, sizeof(SwapDir::Pointer)));
+ memcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *));
+ xfree(swap->swapDirs);
+ swap->swapDirs = tmp;
+ }
+}
+
+void
+free_cachedir(Store::DiskConfig *swap)
+{
+ int i;
+ /* DON'T FREE THESE FOR RECONFIGURE */
+
+ if (reconfiguring)
+ return;
+
+ for (i = 0; i < swap->n_configured; ++i) {
+ /* TODO XXX this lets the swapdir free resources asynchronously
+ * swap->swapDirs[i]->deactivate();
+ * but there may be such a means already.
+ * RBC 20041225
+ */
+ swap->swapDirs[i] = NULL;
+ }
+
+ safe_free(swap->swapDirs);
+ swap->swapDirs = NULL;
+ swap->n_allocated = 0;
+ swap->n_configured = 0;
+}
+
+/* Globals that should be moved to some Store::UFS-specific logging module */
+
+/*
+ * An entry written to the swap log MUST have the following
+ * properties.
+ * 1. It MUST be a public key. It does no good to log
+ * a public ADD, change the key, then log a private
+ * DEL. So we need to log a DEL before we change a
+ * key from public to private.
+ * 2. It MUST have a valid (> -1) swap_filen.
+ */
+void
+storeDirSwapLog(const StoreEntry * e, int op)
+{
+ assert (e);
+ assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
+ assert(e->swap_filen >= 0);
+ /*
+ * icons and such; don't write them to the swap log
+ */
+
+ if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
+ return;
+
+ assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
+
+ debugs(20, 3, "storeDirSwapLog: " <<
+ swap_log_op_str[op] << " " <<
+ e->getMD5Text() << " " <<
+ e->swap_dirn << " " <<
+ std::hex << std::uppercase << std::setfill('0') << std::setw(8) << e->swap_filen);
+
+ dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn))->logEntry(*e, op);
+}
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_DISKS_H
+#define SQUID_STORE_DISKS_H
+
+#include "store/Controlled.h"
+#include "store/forward.h"
+
+namespace Store {
+
+/// summary view of all disk caches (cache_dirs) combined
+class Disks: public Controlled
+{
+public:
+ /* Storage API */
+ virtual void create() override;
+ virtual void init() override;
+ virtual StoreEntry *get(const cache_key *) override;
+ virtual uint64_t maxSize() const override;
+ virtual uint64_t minSize() const override;
+ virtual uint64_t currentSize() const override;
+ virtual uint64_t currentCount() const override;
+ virtual int64_t maxObjectSize() const override;
+ virtual void getStats(StoreInfoStats &stats) const override;
+ virtual void stat(StoreEntry &) const override;
+ virtual void sync() override;
+ virtual void reference(StoreEntry &) override;
+ virtual bool dereference(StoreEntry &e) override;
+ virtual void maintain() override;
+ virtual bool anchorCollapsed(StoreEntry &e, bool &inSync) override;
+ virtual bool updateCollapsed(StoreEntry &e) override;
+ virtual void markForUnlink(StoreEntry &) override;
+ virtual void unlink(StoreEntry &) override;
+ virtual int callback() override;
+
+private:
+ /* migration logic */
+ SwapDir *store(int const x) const;
+ SwapDir &dir(int const idx) const;
+};
+
+} // namespace Store
+
+/* Store::Disks globals that should be converted to use RegisteredRunner */
+void storeDirOpenSwapLogs(void);
+int storeDirWriteCleanLogs(int reopen);
+void storeDirCloseSwapLogs(void);
+
+/* Globals that should be converted to static Store::Disks methods */
+void allocate_new_swapdir(Store::DiskConfig *swap);
+void free_cachedir(Store::DiskConfig *swap);
+
+/* Globals that should be converted to Store::Disks private data members */
+typedef int STDIRSELECT(const StoreEntry *e);
+extern STDIRSELECT *storeDirSelectSwapDir;
+
+/* Globals that should be moved to some Store::UFS-specific logging module */
+void storeDirSwapLog(const StoreEntry *e, int op);
+
+#endif /* SQUID_STORE_DISKS_H */
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+/* DEBUG: section 47 Store Search */
+
+#include "squid.h"
+#include "Debug.h"
+#include "globals.h"
+#include "store/LocalSearch.h"
+#include "StoreSearch.h"
+
+namespace Store {
+
+/// iterates local store_table
+class LocalSearch : public StoreSearch
+{
+ CBDATA_CLASS(LocalSearch);
+
+public:
+ LocalSearch();
+
+ /* StoreSearch API */
+ virtual void next(void (callback)(void *cbdata), void *cbdata) override;
+ virtual bool next() override;
+ virtual bool error() const override;
+ virtual bool isDone() const override;
+ virtual StoreEntry *currentItem() override;
+
+private:
+ void copyBucket();
+ void (*callback)(void *cbdata);
+ void *cbdata;
+ bool _done;
+ int bucket;
+ std::vector<StoreEntry *> entries;
+};
+
+} // namespace Store
+
+CBDATA_NAMESPACED_CLASS_INIT(Store, LocalSearch);
+
+StoreSearch *
+Store::NewLocalSearch()
+{
+ return new LocalSearch;
+}
+
+Store::LocalSearch::LocalSearch() :
+ callback(NULL),
+ cbdata(NULL),
+ _done(false),
+ bucket(0)
+{}
+
+void
+Store::LocalSearch::next(void (aCallback)(void *), void *aCallbackData)
+{
+ next();
+ aCallback (aCallbackData);
+}
+
+bool
+Store::LocalSearch::next()
+{
+ if (!entries.empty())
+ entries.pop_back();
+
+ while (!isDone() && !entries.size())
+ copyBucket();
+
+ return currentItem() != NULL;
+}
+
+bool
+Store::LocalSearch::error() const
+{
+ return false;
+}
+
+bool
+Store::LocalSearch::isDone() const
+{
+ return bucket >= store_hash_buckets || _done;
+}
+
+StoreEntry *
+Store::LocalSearch::currentItem()
+{
+ if (!entries.size())
+ return NULL;
+
+ return entries.back();
+}
+
+void
+Store::LocalSearch::copyBucket()
+{
+ /* probably need to lock the store entries...
+ * we copy them all to prevent races on the links. */
+ debugs(47, 3, "Store::LocalSearch::copyBucket #" << bucket);
+ assert (!entries.size());
+ hash_link *link_ptr = NULL;
+ hash_link *link_next = NULL;
+ link_next = hash_get_bucket(store_table, bucket);
+
+ while (NULL != (link_ptr = link_next)) {
+ link_next = link_ptr->next;
+ StoreEntry *e = (StoreEntry *) link_ptr;
+
+ entries.push_back(e);
+ }
+
+ ++bucket;
+ debugs(47,3, "got entries: " << entries.size());
+}
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_LOCAL_SEARCH_H
+#define SQUID_STORE_LOCAL_SEARCH_H
+
+#include "store/forward.h"
+
+namespace Store {
+ StoreSearch *NewLocalSearch();
+} // namespace Store
+
+#endif /* SQUID_STORE_LOCAL_SEARCH_H */
--- /dev/null
+## Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+##
+## Squid software is distributed under GPLv2+ license and includes
+## contributions from numerous individuals and organizations.
+## Please see the COPYING and CONTRIBUTORS files for details.
+##
+
+include $(top_srcdir)/src/Common.am
+include $(top_srcdir)/src/TestHeaders.am
+
+noinst_LTLIBRARIES = libstore.la
+
+libstore_la_SOURCES= \
+ Controlled.h \
+ Controller.cc \
+ Controller.h \
+ Disk.cc \
+ Disk.h \
+ Disks.cc \
+ Disks.h \
+ forward.h \
+ LocalSearch.cc \
+ LocalSearch.h \
+ Storage.h
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_STORAGE_H
+#define SQUID_STORE_STORAGE_H
+
+#include "base/RefCount.h"
+#include "store/forward.h"
+
+class StoreInfoStats;
+
+namespace Store {
+
+/// A "response storage" abstraction.
+/// This API is shared among Controller and Controlled classes.
+class Storage: public RefCountable
+{
+public:
+ virtual ~Storage() {}
+
+ /// create system resources needed for this store to operate in the future
+ virtual void create() = 0;
+
+ /// Start preparing the store for use. To check readiness, callers should
+ /// use readable() and writable() methods.
+ virtual void init() = 0;
+
+ /// Retrieve a store entry from the store (blocking)
+ virtual StoreEntry *get(const cache_key *) = 0;
+
+ /**
+ * The maximum size the store will support in normal use. Inaccuracy is
+ * permitted, but may throw estimates for memory etc out of whack.
+ */
+ virtual uint64_t maxSize() const = 0;
+
+ /// the minimum size the store will shrink to via normal housekeeping
+ virtual uint64_t minSize() const = 0;
+
+ /// current size
+ virtual uint64_t currentSize() const = 0;
+
+ /// the total number of objects stored right now
+ virtual uint64_t currentCount() const = 0;
+
+ /// the maximum size of a storable object; -1 if unlimited
+ virtual int64_t maxObjectSize() const = 0;
+
+ /// collect statistics
+ virtual void getStats(StoreInfoStats &stats) const = 0;
+
+ /**
+ * Output stats to the provided store entry.
+ \todo make these calls asynchronous
+ */
+ virtual void stat(StoreEntry &e) const = 0;
+
+ /// expect an unlink() call after the entry becomes idle
+ virtual void markForUnlink(StoreEntry &e) = 0;
+
+ /// remove the entry from the store
+ virtual void unlink(StoreEntry &e) = 0;
+
+ /// called once every main loop iteration; TODO: Move to UFS code.
+ virtual int callback() { return 0; }
+
+ /// perform regular periodic maintenance; TODO: move to UFSSwapDir::Maintain
+ virtual void maintain() = 0;
+
+ /// prepare for shutdown
+ virtual void sync() {}
+};
+
+} // namespace Store
+
+#endif /* SQUID_STORE_STORAGE_H */
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_STORE_FORWARD_H
+#define SQUID_STORE_FORWARD_H
+
+typedef int32_t sfileno;
+typedef signed int sdirno;
+
+/// maximum number of entries per cache_dir
+enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
+
+/// Store key.
+typedef unsigned char cache_key;
+
+class StoreSearch;
+class StoreClient;
+class StoreEntry;
+class MemStore;
+class Transients;
+
+namespace Store
+{
+ class Storage;
+ class Controller;
+ class Controlled;
+ class Disks;
+ class Disk;
+ class DiskConfig;
+
+ typedef ::StoreEntry Entry;
+ typedef ::MemStore Memory;
+ typedef ::Transients Transients;
+} // namespace Store
+
+// TODO: Remove these once all code has been transitioned to Store namespace.
+typedef Store::Controller StoreController;
+typedef Store::Disks StoreHashIndex;
+typedef Store::Disk SwapDir;
+template <class C> class RefCount;
+typedef RefCount<Store::Disk> SwapDirPointer;
+
+#endif /* SQUID_STORE_FORWARD_H */
+
{
assert(sd_state.rebuild_lock);
assert(!sd_state.rewrite_lock);
- sd_state.theSearch = Store::Root().search(NULL, NULL);
+ sd_state.theSearch = Store::Root().search();
/* resize or clear */
if (!storeDigestResize())
+++ /dev/null
-/*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 47 Store Directory Routines */
-
-#include "squid.h"
-#include "globals.h"
-#include "mem_node.h"
-#include "MemObject.h"
-#include "MemStore.h"
-#include "profiler/Profiler.h"
-#include "SquidConfig.h"
-#include "SquidMath.h"
-#include "SquidTime.h"
-#include "Store.h"
-#include "store_key_md5.h"
-#include "StoreHashIndex.h"
-#include "swap_log_op.h"
-#include "SwapDir.h"
-#include "tools.h"
-#include "Transients.h"
-// for tvSubDsec() which should be in SquidTime.h
-#include "util.h"
-
-#include <cerrno>
-#include <climits>
-#if HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-
-static STDIRSELECT storeDirSelectSwapDirRoundRobin;
-static STDIRSELECT storeDirSelectSwapDirLeastLoad;
-
-/*
- * store_dirs_rebuilding is initialized to _1_ as a hack so that
- * storeDirWriteCleanLogs() doesn't try to do anything unless _all_
- * cache_dirs have been read. For example, without this hack, Squid
- * will try to write clean log files if -kparse fails (becasue it
- * calls fatal()).
- */
-int StoreController::store_dirs_rebuilding = 1;
-
-StoreController::StoreController() : swapDir (new StoreHashIndex())
- , memStore(NULL), transients(NULL)
-{}
-
-StoreController::~StoreController()
-{
- delete memStore;
- delete transients;
-}
-
-/*
- * This function pointer is set according to 'store_dir_select_algorithm'
- * in squid.conf.
- */
-STDIRSELECT *storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
-
-void
-StoreController::init()
-{
- if (Config.memShared && IamWorkerProcess()) {
- memStore = new MemStore;
- memStore->init();
- }
-
- swapDir->init();
-
- if (0 == strcasecmp(Config.store_dir_select_algorithm, "round-robin")) {
- storeDirSelectSwapDir = storeDirSelectSwapDirRoundRobin;
- debugs(47, DBG_IMPORTANT, "Using Round Robin store dir selection");
- } else {
- storeDirSelectSwapDir = storeDirSelectSwapDirLeastLoad;
- debugs(47, DBG_IMPORTANT, "Using Least Load store dir selection");
- }
-
- if (UsingSmp() && IamWorkerProcess() && Config.onoff.collapsed_forwarding) {
- transients = new Transients;
- transients->init();
- }
-}
-
-void
-StoreController::createOneStore(Store &aStore)
-{
- /*
- * On Windows, fork() is not available.
- * The following is a workaround for create store directories sequentially
- * when running on native Windows port.
- */
-#if !_SQUID_WINDOWS_
-
- if (fork())
- return;
-
-#endif
-
- aStore.create();
-
-#if !_SQUID_WINDOWS_
-
- exit(0);
-
-#endif
-}
-
-void
-StoreController::create()
-{
- swapDir->create();
-
-#if !_SQUID_WINDOWS_
-
- pid_t pid;
-
- do {
- int status;
-#if _SQUID_NEXT_
-
- pid = wait3(&status, WNOHANG, NULL);
-#else
-
- pid = waitpid(-1, &status, 0);
-#endif
-
- } while (pid > 0 || (pid < 0 && errno == EINTR));
-
-#endif
-}
-
-/**
- * Determine whether the given directory can handle this object
- * size
- *
- * Note: if the object size is -1, then the only swapdirs that
- * will return true here are ones that have min and max unset,
- * ie any-sized-object swapdirs. This is a good thing.
- */
-bool
-SwapDir::objectSizeIsAcceptable(int64_t objsize) const
-{
- // If the swapdir has no range limits, then it definitely can
- if (min_objsize <= 0 && max_objsize == -1)
- return true;
-
- /*
- * If the object size is -1 and the storedir has limits we
- * can't store it there.
- */
- if (objsize == -1)
- return false;
-
- // Else, make sure that the object size will fit.
- if (max_objsize == -1 && min_objsize <= objsize)
- return true;
- else
- return min_objsize <= objsize && max_objsize > objsize;
-}
-
-/*
- * This new selection scheme simply does round-robin on all SwapDirs.
- * A SwapDir is skipped if it is over the max_size (100%) limit, or
- * overloaded.
- */
-static int
-storeDirSelectSwapDirRoundRobin(const StoreEntry * e)
-{
- // e->objectLen() is negative at this point when we are still STORE_PENDING
- ssize_t objsize = e->mem_obj->expectedReplySize();
- if (objsize != -1)
- objsize += e->mem_obj->swap_hdr_sz;
-
- // Increment the first candidate once per selection (not once per
- // iteration) to reduce bias when some disk(s) attract more entries.
- static int firstCandidate = 0;
- if (++firstCandidate >= Config.cacheSwap.n_configured)
- firstCandidate = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- const int dirn = (firstCandidate + i) % Config.cacheSwap.n_configured;
- const SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(dirn));
-
- int load = 0;
- if (!sd->canStore(*e, objsize, load))
- continue;
-
- if (load < 0 || load > 1000) {
- continue;
- }
-
- return dirn;
- }
-
- return -1;
-}
-
-/*
- * Spread load across all of the store directories
- *
- * Note: We should modify this later on to prefer sticking objects
- * in the *tightest fit* swapdir to conserve space, along with the
- * actual swapdir usage. But for now, this hack will do while
- * testing, so you should order your swapdirs in the config file
- * from smallest max-size= to largest max-size=.
- *
- * We also have to choose nleast == nconf since we need to consider
- * ALL swapdirs, regardless of state. Again, this is a hack while
- * we sort out the real usefulness of this algorithm.
- */
-static int
-storeDirSelectSwapDirLeastLoad(const StoreEntry * e)
-{
- int64_t most_free = 0;
- ssize_t least_objsize = -1;
- int least_load = INT_MAX;
- int load;
- int dirn = -1;
- int i;
- RefCount<SwapDir> SD;
-
- // e->objectLen() is negative at this point when we are still STORE_PENDING
- ssize_t objsize = e->mem_obj->expectedReplySize();
-
- if (objsize != -1)
- objsize += e->mem_obj->swap_hdr_sz;
-
- for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
- SD = dynamic_cast<SwapDir *>(INDEXSD(i));
- SD->flags.selected = false;
-
- if (!SD->canStore(*e, objsize, load))
- continue;
-
- if (load < 0 || load > 1000)
- continue;
-
- if (load > least_load)
- continue;
-
- const int64_t cur_free = SD->maxSize() - SD->currentSize();
-
- /* If the load is equal, then look in more details */
- if (load == least_load) {
- /* closest max-size fit */
-
- if (least_objsize != -1)
- if (SD->maxObjectSize() > least_objsize)
- continue;
-
- /* most free */
- if (cur_free < most_free)
- continue;
- }
-
- least_load = load;
- least_objsize = SD->maxObjectSize();
- most_free = cur_free;
- dirn = i;
- }
-
- if (dirn >= 0)
- dynamic_cast<SwapDir *>(INDEXSD(dirn))->flags.selected = true;
-
- return dirn;
-}
-
-/*
- * An entry written to the swap log MUST have the following
- * properties.
- * 1. It MUST be a public key. It does no good to log
- * a public ADD, change the key, then log a private
- * DEL. So we need to log a DEL before we change a
- * key from public to private.
- * 2. It MUST have a valid (> -1) swap_filen.
- */
-void
-storeDirSwapLog(const StoreEntry * e, int op)
-{
- assert (e);
- assert(!EBIT_TEST(e->flags, KEY_PRIVATE));
- assert(e->swap_filen >= 0);
- /*
- * icons and such; don't write them to the swap log
- */
-
- if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
- return;
-
- assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX);
-
- debugs(20, 3, "storeDirSwapLog: " <<
- swap_log_op_str[op] << " " <<
- e->getMD5Text() << " " <<
- e->swap_dirn << " " <<
- std::hex << std::uppercase << std::setfill('0') << std::setw(8) << e->swap_filen);
-
- dynamic_cast<SwapDir *>(INDEXSD(e->swap_dirn))->logEntry(*e, op);
-}
-
-void
-StoreController::getStats(StoreInfoStats &stats) const
-{
- if (memStore)
- memStore->getStats(stats);
- else {
- // move this code to a non-shared memory cache class when we have it
- stats.mem.shared = false;
- stats.mem.capacity = Config.memMaxSize;
- stats.mem.size = mem_node::StoreMemSize();
- stats.mem.count = hot_obj_count;
- }
-
- swapDir->getStats(stats);
-
- // low-level info not specific to memory or disk cache
- stats.store_entry_count = StoreEntry::inUseCount();
- stats.mem_object_count = MemObject::inUseCount();
-}
-
-void
-StoreController::stat(StoreEntry &output) const
-{
- storeAppendPrintf(&output, "Store Directory Statistics:\n");
- storeAppendPrintf(&output, "Store Entries : %lu\n",
- (unsigned long int)StoreEntry::inUseCount());
- storeAppendPrintf(&output, "Maximum Swap Size : %" PRIu64 " KB\n",
- maxSize() >> 10);
- storeAppendPrintf(&output, "Current Store Swap Size: %.2f KB\n",
- currentSize() / 1024.0);
- storeAppendPrintf(&output, "Current Capacity : %.2f%% used, %.2f%% free\n",
- Math::doublePercent(currentSize(), maxSize()),
- Math::doublePercent((maxSize() - currentSize()), maxSize()));
-
- if (memStore)
- memStore->stat(output);
-
- /* now the swapDir */
- swapDir->stat(output);
-}
-
-/* if needed, this could be taught to cache the result */
-uint64_t
-StoreController::maxSize() const
-{
- /* TODO: include memory cache ? */
- return swapDir->maxSize();
-}
-
-uint64_t
-StoreController::minSize() const
-{
- /* TODO: include memory cache ? */
- return swapDir->minSize();
-}
-
-uint64_t
-StoreController::currentSize() const
-{
- return swapDir->currentSize();
-}
-
-uint64_t
-StoreController::currentCount() const
-{
- return swapDir->currentCount();
-}
-
-int64_t
-StoreController::maxObjectSize() const
-{
- return swapDir->maxObjectSize();
-}
-
-void
-SwapDir::diskFull()
-{
- if (currentSize() >= maxSize())
- return;
-
- max_size = currentSize();
-
- debugs(20, DBG_IMPORTANT, "WARNING: Shrinking cache_dir #" << index << " to " << currentSize() / 1024.0 << " KB");
-}
-
-void
-storeDirOpenSwapLogs(void)
-{
- for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
- dynamic_cast<SwapDir *>(INDEXSD(dirn))->openLog();
-}
-
-void
-storeDirCloseSwapLogs(void)
-{
- for (int dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
- dynamic_cast<SwapDir *>(INDEXSD(dirn))->closeLog();
-}
-
-/*
- * storeDirWriteCleanLogs
- *
- * Writes a "clean" swap log file from in-memory metadata.
- * This is a rewrite of the original function to troll each
- * StoreDir and write the logs, and flush at the end of
- * the run. Thanks goes to Eric Stern, since this solution
- * came out of his COSS code.
- */
-int
-storeDirWriteCleanLogs(int reopen)
-{
- const StoreEntry *e = NULL;
- int n = 0;
-
- struct timeval start;
- double dt;
- RefCount<SwapDir> sd;
- int dirn;
- int notdone = 1;
-
- if (StoreController::store_dirs_rebuilding) {
- debugs(20, DBG_IMPORTANT, "Not currently OK to rewrite swap log.");
- debugs(20, DBG_IMPORTANT, "storeDirWriteCleanLogs: Operation aborted.");
- return 0;
- }
-
- debugs(20, DBG_IMPORTANT, "storeDirWriteCleanLogs: Starting...");
- getCurrentTime();
- start = current_time;
-
- for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) {
- sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
-
- if (sd->writeCleanStart() < 0) {
- debugs(20, DBG_IMPORTANT, "log.clean.start() failed for dir #" << sd->index);
- continue;
- }
- }
-
- /*
- * This may look inefficient as CPU wise it is more efficient to do this
- * sequentially, but I/O wise the parallellism helps as it allows more
- * hdd spindles to be active.
- */
- while (notdone) {
- notdone = 0;
-
- for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn) {
- sd = dynamic_cast<SwapDir *>(INDEXSD(dirn));
-
- if (NULL == sd->cleanLog)
- continue;
-
- e = sd->cleanLog->nextEntry();
-
- if (!e)
- continue;
-
- notdone = 1;
-
- if (!sd->canLog(*e))
- continue;
-
- sd->cleanLog->write(*e);
-
- if ((++n & 0xFFFF) == 0) {
- getCurrentTime();
- debugs(20, DBG_IMPORTANT, " " << std::setw(7) << n <<
- " entries written so far.");
- }
- }
- }
-
- /* Flush */
- for (dirn = 0; dirn < Config.cacheSwap.n_configured; ++dirn)
- dynamic_cast<SwapDir *>(INDEXSD(dirn))->writeCleanDone();
-
- if (reopen)
- storeDirOpenSwapLogs();
-
- getCurrentTime();
-
- dt = tvSubDsec(start, current_time);
-
- debugs(20, DBG_IMPORTANT, " Finished. Wrote " << n << " entries.");
- debugs(20, DBG_IMPORTANT, " Took "<< std::setw(3)<< std::setprecision(2) << dt <<
- " seconds ("<< std::setw(6) << ((double) n / (dt > 0.0 ? dt : 1.0)) << " entries/sec).");
-
- return n;
-}
-
-StoreSearch *
-StoreController::search(String const url, HttpRequest *request)
-{
- /* cheat, for now you can't search the memory hot cache */
- return swapDir->search(url, request);
-}
-
-StorePointer
-StoreHashIndex::store(int const x) const
-{
- return INDEXSD(x);
-}
-
-SwapDir &
-StoreHashIndex::dir(const int i) const
-{
- SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(i));
- assert(sd);
- return *sd;
-}
-
-void
-StoreController::sync(void)
-{
- if (memStore)
- memStore->sync();
- swapDir->sync();
-}
-
-/*
- * handle callbacks all avaliable fs'es
- */
-int
-StoreController::callback()
-{
- /* This will likely double count. Thats ok. */
- PROF_start(storeDirCallback);
-
- /* mem cache callbacks ? */
- int result = swapDir->callback();
-
- PROF_stop(storeDirCallback);
-
- return result;
-}
-
-int
-storeDirGetBlkSize(const char *path, int *blksize)
-{
- struct statvfs sfs;
-
- if (xstatvfs(path, &sfs)) {
- debugs(50, DBG_IMPORTANT, "" << path << ": " << xstrerror());
- *blksize = 2048;
- return 1;
- }
-
- *blksize = (int) sfs.f_frsize;
-
- // Sanity check; make sure we have a meaningful value.
- if (*blksize < 512)
- *blksize = 2048;
-
- return 0;
-}
-
-#define fsbtoblk(num, fsbs, bs) \
- (((fsbs) != 0 && (fsbs) < (bs)) ? \
- (num) / ((bs) / (fsbs)) : (num) * ((fsbs) / (bs)))
-int
-storeDirGetUFSStats(const char *path, int *totl_kb, int *free_kb, int *totl_in, int *free_in)
-{
- struct statvfs sfs;
-
- if (xstatvfs(path, &sfs)) {
- debugs(50, DBG_IMPORTANT, "" << path << ": " << xstrerror());
- return 1;
- }
-
- *totl_kb = (int) fsbtoblk(sfs.f_blocks, sfs.f_frsize, 1024);
- *free_kb = (int) fsbtoblk(sfs.f_bfree, sfs.f_frsize, 1024);
- *totl_in = (int) sfs.f_files;
- *free_in = (int) sfs.f_ffree;
- return 0;
-}
-
-void
-allocate_new_swapdir(SquidConfig::_cacheSwap * swap)
-{
- if (swap->swapDirs == NULL) {
- swap->n_allocated = 4;
- swap->swapDirs = static_cast<SwapDir::Pointer *>(xcalloc(swap->n_allocated, sizeof(SwapDir::Pointer)));
- }
-
- if (swap->n_allocated == swap->n_configured) {
- swap->n_allocated <<= 1;
- SwapDir::Pointer *const tmp = static_cast<SwapDir::Pointer *>(xcalloc(swap->n_allocated, sizeof(SwapDir::Pointer)));
- memcpy(tmp, swap->swapDirs, swap->n_configured * sizeof(SwapDir *));
- xfree(swap->swapDirs);
- swap->swapDirs = tmp;
- }
-}
-
-void
-free_cachedir(SquidConfig::_cacheSwap * swap)
-{
- int i;
- /* DON'T FREE THESE FOR RECONFIGURE */
-
- if (reconfiguring)
- return;
-
- for (i = 0; i < swap->n_configured; ++i) {
- /* TODO XXX this lets the swapdir free resources asynchronously
- * swap->swapDirs[i]->deactivate();
- * but there may be such a means already.
- * RBC 20041225
- */
- swap->swapDirs[i] = NULL;
- }
-
- safe_free(swap->swapDirs);
- swap->swapDirs = NULL;
- swap->n_allocated = 0;
- swap->n_configured = 0;
-}
-
-/* this should be a virtual method on StoreEntry,
- * i.e. e->referenced()
- * so that the entry can notify the creating Store
- */
-void
-StoreController::reference(StoreEntry &e)
-{
- // special entries do not belong to any specific Store, but are IN_MEMORY
- if (EBIT_TEST(e.flags, ENTRY_SPECIAL))
- return;
-
- /* Notify the fs that we're referencing this object again */
-
- if (e.swap_dirn > -1)
- swapDir->reference(e);
-
- // Notify the memory cache that we're referencing this object again
- if (memStore && e.mem_status == IN_MEMORY)
- memStore->reference(e);
-
- // TODO: move this code to a non-shared memory cache class when we have it
- if (e.mem_obj) {
- if (mem_policy->Referenced)
- mem_policy->Referenced(mem_policy, &e, &e.mem_obj->repl);
- }
-}
-
-bool
-StoreController::dereference(StoreEntry &e, bool wantsLocalMemory)
-{
- // special entries do not belong to any specific Store, but are IN_MEMORY
- if (EBIT_TEST(e.flags, ENTRY_SPECIAL))
- return true;
-
- bool keepInStoreTable = false; // keep only if somebody needs it there
-
- /* Notify the fs that we're not referencing this object any more */
-
- if (e.swap_filen > -1)
- keepInStoreTable = swapDir->dereference(e, wantsLocalMemory) || keepInStoreTable;
-
- // Notify the memory cache that we're not referencing this object any more
- if (memStore && e.mem_status == IN_MEMORY)
- keepInStoreTable = memStore->dereference(e, wantsLocalMemory) || keepInStoreTable;
-
- // TODO: move this code to a non-shared memory cache class when we have it
- if (e.mem_obj) {
- if (mem_policy->Dereferenced)
- mem_policy->Dereferenced(mem_policy, &e, &e.mem_obj->repl);
- // non-shared memory cache relies on store_table
- if (!memStore)
- keepInStoreTable = wantsLocalMemory || keepInStoreTable;
- }
-
- return keepInStoreTable;
-}
-
-StoreEntry *
-StoreController::get(const cache_key *key)
-{
- if (StoreEntry *e = find(key)) {
- // this is not very precise: some get()s are not initiated by clients
- e->touch();
- return e;
- }
- return NULL;
-}
-
-/// Internal method to implements the guts of the Store::get() API:
-/// returns an in-transit or cached object with a given key, if any.
-StoreEntry *
-StoreController::find(const cache_key *key)
-{
- if (StoreEntry *e = swapDir->get(key)) {
- // TODO: ignore and maybe handleIdleEntry() unlocked intransit entries
- // because their backing store slot may be gone already.
- debugs(20, 3, HERE << "got in-transit entry: " << *e);
- return e;
- }
-
- // Must search transients before caches because we must sync those we find.
- if (transients) {
- if (StoreEntry *e = transients->get(key)) {
- debugs(20, 3, "got shared in-transit entry: " << *e);
- bool inSync = false;
- const bool found = anchorCollapsed(*e, inSync);
- if (!found || inSync)
- return e;
- assert(!e->locked()); // ensure release will destroyStoreEntry()
- e->release(); // do not let others into the same trap
- return NULL;
- }
- }
-
- if (memStore) {
- if (StoreEntry *e = memStore->get(key)) {
- debugs(20, 3, HERE << "got mem-cached entry: " << *e);
- return e;
- }
- }
-
- // TODO: this disk iteration is misplaced; move to StoreHashIndex when
- // the global store_table is no longer used for in-transit objects.
- if (const int cacheDirs = Config.cacheSwap.n_configured) {
- // ask each cache_dir until the entry is found; use static starting
- // point to avoid asking the same subset of disks more often
- // TODO: coordinate with put() to be able to guess the right disk often
- static int idx = 0;
- for (int n = 0; n < cacheDirs; ++n) {
- idx = (idx + 1) % cacheDirs;
- SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(idx));
- if (!sd->active())
- continue;
-
- if (StoreEntry *e = sd->get(key)) {
- debugs(20, 3, HERE << "cache_dir " << idx <<
- " got cached entry: " << *e);
- return e;
- }
- }
- }
-
- debugs(20, 4, HERE << "none of " << Config.cacheSwap.n_configured <<
- " cache_dirs have " << storeKeyText(key));
- return NULL;
-}
-
-void
-StoreController::get(String const, STOREGETCLIENT, void *)
-{
- fatal("not implemented");
-}
-
-/// updates the collapsed entry with the corresponding on-disk entry, if any
-/// In other words, the SwapDir::anchorCollapsed() API applied to all disks.
-bool
-StoreController::anchorCollapsedOnDisk(StoreEntry &collapsed, bool &inSync)
-{
- // TODO: move this loop to StoreHashIndex, just like the one in get().
- if (const int cacheDirs = Config.cacheSwap.n_configured) {
- // ask each cache_dir until the entry is found; use static starting
- // point to avoid asking the same subset of disks more often
- // TODO: coordinate with put() to be able to guess the right disk often
- static int idx = 0;
- for (int n = 0; n < cacheDirs; ++n) {
- idx = (idx + 1) % cacheDirs;
- SwapDir *sd = dynamic_cast<SwapDir*>(INDEXSD(idx));
- if (!sd->active())
- continue;
-
- if (sd->anchorCollapsed(collapsed, inSync)) {
- debugs(20, 3, "cache_dir " << idx << " anchors " << collapsed);
- return true;
- }
- }
- }
-
- debugs(20, 4, "none of " << Config.cacheSwap.n_configured <<
- " cache_dirs have " << collapsed);
- return false;
-}
-
-void StoreController::markForUnlink(StoreEntry &e)
-{
- if (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0)
- transients->markForUnlink(e);
- if (memStore && e.mem_obj && e.mem_obj->memCache.index >= 0)
- memStore->markForUnlink(e);
- if (e.swap_filen >= 0)
- e.store()->markForUnlink(e);
-}
-
-// move this into [non-shared] memory cache class when we have one
-/// whether e should be kept in local RAM for possible future caching
-bool
-StoreController::keepForLocalMemoryCache(StoreEntry &e) const
-{
- if (!e.memoryCachable())
- return false;
-
- // does the current and expected size obey memory caching limits?
- assert(e.mem_obj);
- const int64_t loadedSize = e.mem_obj->endOffset();
- const int64_t expectedSize = e.mem_obj->expectedReplySize(); // may be < 0
- const int64_t ramSize = max(loadedSize, expectedSize);
- const int64_t ramLimit = min(
- static_cast<int64_t>(Config.memMaxSize),
- static_cast<int64_t>(Config.Store.maxInMemObjSize));
- return ramSize <= ramLimit;
-}
-
-void
-StoreController::memoryOut(StoreEntry &e, const bool preserveSwappable)
-{
- bool keepInLocalMemory = false;
- if (memStore)
- memStore->write(e); // leave keepInLocalMemory false
- else
- keepInLocalMemory = keepForLocalMemoryCache(e);
-
- debugs(20, 7, HERE << "keepInLocalMemory: " << keepInLocalMemory);
-
- if (!keepInLocalMemory)
- e.trimMemory(preserveSwappable);
-}
-
-void
-StoreController::memoryUnlink(StoreEntry &e)
-{
- if (memStore)
- memStore->unlink(e);
- else // TODO: move into [non-shared] memory cache class when we have one
- e.destroyMemObject();
-}
-
-void
-StoreController::memoryDisconnect(StoreEntry &e)
-{
- if (memStore)
- memStore->disconnect(e);
- // else nothing to do for non-shared memory cache
-}
-
-void
-StoreController::transientsAbandon(StoreEntry &e)
-{
- if (transients) {
- assert(e.mem_obj);
- if (e.mem_obj->xitTable.index >= 0)
- transients->abandon(e);
- }
-}
-
-void
-StoreController::transientsCompleteWriting(StoreEntry &e)
-{
- if (transients) {
- assert(e.mem_obj);
- if (e.mem_obj->xitTable.index >= 0)
- transients->completeWriting(e);
- }
-}
-
-int
-StoreController::transientReaders(const StoreEntry &e) const
-{
- return (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0) ?
- transients->readers(e) : 0;
-}
-
-void
-StoreController::transientsDisconnect(MemObject &mem_obj)
-{
- if (transients)
- transients->disconnect(mem_obj);
-}
-
-void
-StoreController::handleIdleEntry(StoreEntry &e)
-{
- bool keepInLocalMemory = false;
-
- if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) {
- // Icons (and cache digests?) should stay in store_table until we
- // have a dedicated storage for them (that would not purge them).
- // They are not managed [well] by any specific Store handled below.
- keepInLocalMemory = true;
- } else if (memStore) {
- // leave keepInLocalMemory false; memStore maintains its own cache
- } else {
- keepInLocalMemory = keepForLocalMemoryCache(e) && // in good shape and
- // the local memory cache is not overflowing
- (mem_node::InUseCount() <= store_pages_max);
- }
-
- // An idle, unlocked entry that only belongs to a SwapDir which controls
- // its own index, should not stay in the global store_table.
- if (!dereference(e, keepInLocalMemory)) {
- debugs(20, 5, HERE << "destroying unlocked entry: " << &e << ' ' << e);
- destroyStoreEntry(static_cast<hash_link*>(&e));
- return;
- }
-
- debugs(20, 5, HERE << "keepInLocalMemory: " << keepInLocalMemory);
-
- // TODO: move this into [non-shared] memory cache class when we have one
- if (keepInLocalMemory) {
- e.setMemStatus(IN_MEMORY);
- e.mem_obj->unlinkRequest();
- } else {
- e.purgeMem(); // may free e
- }
-}
-
-void
-StoreController::allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags,
- const HttpRequestMethod &reqMethod)
-{
- e->makePublic(); // this is needed for both local and SMP collapsing
- if (transients)
- transients->startWriting(e, reqFlags, reqMethod);
- debugs(20, 3, "may " << (transients && e->mem_obj->xitTable.index >= 0 ?
- "SMP-" : "locally-") << "collapse " << *e);
-}
-
-void
-StoreController::syncCollapsed(const sfileno xitIndex)
-{
- assert(transients);
-
- StoreEntry *collapsed = transients->findCollapsed(xitIndex);
- if (!collapsed) { // the entry is no longer locally active, ignore update
- debugs(20, 7, "not SMP-syncing not-transient " << xitIndex);
- return;
- }
- assert(collapsed->mem_obj);
- assert(collapsed->mem_obj->smpCollapsed);
-
- debugs(20, 7, "syncing " << *collapsed);
-
- bool abandoned = transients->abandoned(*collapsed);
- bool found = false;
- bool inSync = false;
- if (memStore && collapsed->mem_obj->memCache.io == MemObject::ioDone) {
- found = true;
- inSync = true;
- debugs(20, 7, "fully mem-loaded " << *collapsed);
- } else if (memStore && collapsed->mem_obj->memCache.index >= 0) {
- found = true;
- inSync = memStore->updateCollapsed(*collapsed);
- } else if (collapsed->swap_filen >= 0) {
- found = true;
- inSync = collapsed->store()->updateCollapsed(*collapsed);
- } else {
- found = anchorCollapsed(*collapsed, inSync);
- }
-
- if (abandoned && collapsed->store_status == STORE_PENDING) {
- debugs(20, 3, "aborting abandoned but STORE_PENDING " << *collapsed);
- collapsed->abort();
- return;
- }
-
- if (inSync) {
- debugs(20, 5, "synced " << *collapsed);
- collapsed->invokeHandlers();
- } else if (found) { // unrecoverable problem syncing this entry
- debugs(20, 3, "aborting unsyncable " << *collapsed);
- collapsed->abort();
- } else { // the entry is still not in one of the caches
- debugs(20, 7, "waiting " << *collapsed);
- }
-}
-
-/// Called for in-transit entries that are not yet anchored to a cache.
-/// For cached entries, return true after synchronizing them with their cache
-/// (making inSync true on success). For not-yet-cached entries, return false.
-bool
-StoreController::anchorCollapsed(StoreEntry &collapsed, bool &inSync)
-{
- // this method is designed to work with collapsed transients only
- assert(collapsed.mem_obj);
- assert(collapsed.mem_obj->xitTable.index >= 0);
- assert(collapsed.mem_obj->smpCollapsed);
-
- debugs(20, 7, "anchoring " << collapsed);
-
- bool found = false;
- if (memStore)
- found = memStore->anchorCollapsed(collapsed, inSync);
- if (!found && Config.cacheSwap.n_configured)
- found = anchorCollapsedOnDisk(collapsed, inSync);
-
- if (found) {
- if (inSync)
- debugs(20, 7, "anchored " << collapsed);
- else
- debugs(20, 5, "failed to anchor " << collapsed);
- } else {
- debugs(20, 7, "skipping not yet cached " << collapsed);
- }
-
- return found;
-}
-
-StoreHashIndex::StoreHashIndex()
-{
- if (store_table)
- abort();
- assert (store_table == NULL);
-}
-
-StoreHashIndex::~StoreHashIndex()
-{
- if (store_table) {
- hashFreeItems(store_table, destroyStoreEntry);
- hashFreeMemory(store_table);
- store_table = NULL;
- }
-}
-
-int
-StoreHashIndex::callback()
-{
- int result = 0;
- int j;
- static int ndir = 0;
-
- do {
- j = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (ndir >= Config.cacheSwap.n_configured)
- ndir = ndir % Config.cacheSwap.n_configured;
-
- int temp_result = store(ndir)->callback();
-
- ++ndir;
-
- j += temp_result;
-
- result += temp_result;
-
- if (j > 100)
- fatal ("too much io\n");
- }
- } while (j > 0);
-
- ++ndir;
-
- return result;
-}
-
-void
-StoreHashIndex::create()
-{
- if (Config.cacheSwap.n_configured == 0) {
- debugs(0, DBG_PARSE_NOTE(DBG_CRITICAL), "No cache_dir stores are configured.");
- }
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).active())
- store(i)->create();
- }
-}
-
-/* Lookup an object in the cache.
- * return just a reference to object, don't start swapping in yet. */
-StoreEntry *
-StoreHashIndex::get(const cache_key *key)
-{
- PROF_start(storeGet);
- debugs(20, 3, "storeGet: looking up " << storeKeyText(key));
- StoreEntry *p = static_cast<StoreEntry *>(hash_lookup(store_table, key));
- PROF_stop(storeGet);
- return p;
-}
-
-void
-StoreHashIndex::get(String const, STOREGETCLIENT, void *)
-{
- fatal("not implemented");
-}
-
-void
-StoreHashIndex::init()
-{
- if (Config.Store.objectsPerBucket <= 0)
- fatal("'store_objects_per_bucket' should be larger than 0.");
-
- if (Config.Store.avgObjectSize <= 0)
- fatal("'store_avg_object_size' should be larger than 0.");
-
- /* Calculate size of hash table (maximum currently 64k buckets). */
- /* this is very bogus, its specific to the any Store maintaining an
- * in-core index, not global */
- size_t buckets = (Store::Root().maxSize() + Config.memMaxSize) / Config.Store.avgObjectSize;
- debugs(20, DBG_IMPORTANT, "Swap maxSize " << (Store::Root().maxSize() >> 10) <<
- " + " << ( Config.memMaxSize >> 10) << " KB, estimated " << buckets << " objects");
- buckets /= Config.Store.objectsPerBucket;
- debugs(20, DBG_IMPORTANT, "Target number of buckets: " << buckets);
- /* ideally the full scan period should be configurable, for the
- * moment it remains at approximately 24 hours. */
- store_hash_buckets = storeKeyHashBuckets(buckets);
- debugs(20, DBG_IMPORTANT, "Using " << store_hash_buckets << " Store buckets");
- debugs(20, DBG_IMPORTANT, "Max Mem size: " << ( Config.memMaxSize >> 10) << " KB" <<
- (Config.memShared ? " [shared]" : ""));
- debugs(20, DBG_IMPORTANT, "Max Swap size: " << (Store::Root().maxSize() >> 10) << " KB");
-
- store_table = hash_create(storeKeyHashCmp,
- store_hash_buckets, storeKeyHashHash);
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- /* this starts a search of the store dirs, loading their
- * index. under the new Store api this should be
- * driven by the StoreHashIndex, not by each store.
- *
- * That is, the HashIndex should perform a search of each dir it is
- * indexing to do the hash insertions. The search is then able to
- * decide 'from-memory', or 'from-clean-log' or 'from-dirty-log' or
- * 'from-no-log'.
- *
- * Step 1: make the store rebuilds use a search internally
- * Step 2: change the search logic to use the four modes described
- * above
- * Step 3: have the hash index walk the searches itself.
- */
- if (dir(i).active())
- store(i)->init();
- }
-}
-
-uint64_t
-StoreHashIndex::maxSize() const
-{
- uint64_t result = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).doReportStat())
- result += store(i)->maxSize();
- }
-
- return result;
-}
-
-uint64_t
-StoreHashIndex::minSize() const
-{
- uint64_t result = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).doReportStat())
- result += store(i)->minSize();
- }
-
- return result;
-}
-
-uint64_t
-StoreHashIndex::currentSize() const
-{
- uint64_t result = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).doReportStat())
- result += store(i)->currentSize();
- }
-
- return result;
-}
-
-uint64_t
-StoreHashIndex::currentCount() const
-{
- uint64_t result = 0;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).doReportStat())
- result += store(i)->currentCount();
- }
-
- return result;
-}
-
-int64_t
-StoreHashIndex::maxObjectSize() const
-{
- int64_t result = -1;
-
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- if (dir(i).active() && store(i)->maxObjectSize() > result)
- result = store(i)->maxObjectSize();
- }
-
- return result;
-}
-
-void
-StoreHashIndex::getStats(StoreInfoStats &stats) const
-{
- // accumulate per-disk cache stats
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i) {
- StoreInfoStats dirStats;
- store(i)->getStats(dirStats);
- stats += dirStats;
- }
-
- // common to all disks
- stats.swap.open_disk_fd = store_open_disk_fd;
-
- // memory cache stats are collected in StoreController::getStats(), for now
-}
-
-void
-StoreHashIndex::stat(StoreEntry & output) const
-{
- int i;
-
- /* Now go through each store, calling its stat routine */
-
- for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
- storeAppendPrintf(&output, "\n");
- store(i)->stat(output);
- }
-}
-
-void
-StoreHashIndex::reference(StoreEntry &e)
-{
- e.store()->reference(e);
-}
-
-bool
-StoreHashIndex::dereference(StoreEntry &e, bool wantsLocalMemory)
-{
- return e.store()->dereference(e, wantsLocalMemory);
-}
-
-void
-StoreHashIndex::maintain()
-{
- int i;
- /* walk each fs */
-
- for (i = 0; i < Config.cacheSwap.n_configured; ++i) {
- /* XXX FixMe: This should be done "in parallell" on the different
- * cache_dirs, not one at a time.
- */
- /* call the maintain function .. */
- store(i)->maintain();
- }
-}
-
-void
-StoreHashIndex::sync()
-{
- for (int i = 0; i < Config.cacheSwap.n_configured; ++i)
- store(i)->sync();
-}
-
-StoreSearch *
-StoreHashIndex::search(String const url, HttpRequest *)
-{
- if (url.size())
- fatal ("Cannot search by url yet\n");
-
- return new StoreSearchHashIndex (this);
-}
-
-CBDATA_CLASS_INIT(StoreSearchHashIndex);
-
-StoreSearchHashIndex::StoreSearchHashIndex(RefCount<StoreHashIndex> aSwapDir) :
- sd(aSwapDir),
- callback(NULL),
- cbdata(NULL),
- _done(false),
- bucket(0)
-{}
-
-/* do not link
-StoreSearchHashIndex::StoreSearchHashIndex(StoreSearchHashIndex const &);
-*/
-
-StoreSearchHashIndex::~StoreSearchHashIndex()
-{}
-
-void
-StoreSearchHashIndex::next(void (aCallback)(void *), void *aCallbackData)
-{
- next();
- aCallback (aCallbackData);
-}
-
-bool
-StoreSearchHashIndex::next()
-{
- if (!entries.empty())
- entries.pop_back();
-
- while (!isDone() && !entries.size())
- copyBucket();
-
- return currentItem() != NULL;
-}
-
-bool
-StoreSearchHashIndex::error() const
-{
- return false;
-}
-
-bool
-StoreSearchHashIndex::isDone() const
-{
- return bucket >= store_hash_buckets || _done;
-}
-
-StoreEntry *
-StoreSearchHashIndex::currentItem()
-{
- if (!entries.size())
- return NULL;
-
- return entries.back();
-}
-
-void
-StoreSearchHashIndex::copyBucket()
-{
- /* probably need to lock the store entries...
- * we copy them all to prevent races on the links. */
- debugs(47, 3, "StoreSearchHashIndex::copyBucket #" << bucket);
- assert (!entries.size());
- hash_link *link_ptr = NULL;
- hash_link *link_next = NULL;
- link_next = hash_get_bucket(store_table, bucket);
-
- while (NULL != (link_ptr = link_next)) {
- link_next = link_ptr->next;
- StoreEntry *e = (StoreEntry *) link_ptr;
-
- entries.push_back(e);
- }
-
- ++bucket;
- debugs(47,3, "got entries: " << entries.size());
-}
-
#include "MemObject.h"
#include "SquidConfig.h"
#include "Store.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
StoreIoStats store_io_stats;
storeOpen(StoreEntry * e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * callback,
void *callback_data)
{
- return dynamic_cast<SwapDir *>(e->store().getRaw())->openStoreIO(*e, file_callback, callback, callback_data);
+ return e->disk().openStoreIO(*e, file_callback, callback, callback_data);
}
void
#define SQUID_STORE_KEY_MD5_H_
#include "hash.h"
+#include "store/forward.h"
class HttpRequestMethod;
class HttpRequest;
-/* MD5 cache keys */
-typedef unsigned char cache_key;
-
cache_key *storeKeyDup(const cache_key *);
cache_key *storeKeyCopy(cache_key *, const cache_key *);
void storeKeyFree(const cache_key *);
#include "store_key_md5.h"
#include "store_rebuild.h"
#include "StoreSearch.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
// for tvSubDsec() which should be in SquidTime.h
#include "util.h"
static int seen = 0;
if (currentSearch == NULL || currentSearch->isDone())
- currentSearch = Store::Root().search(NULL, NULL);
+ currentSearch = Store::Root().search();
size_t statCount = 500;
#include "StatCounters.h"
#include "store_log.h"
#include "swap_log_op.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
static void storeSwapOutStart(StoreEntry * e);
static StoreIOState::STIOCB storeSwapOutFileClosed;
/* FIXME: this should be handle by the link from store IO to
* Store, rather than being a top level API call.
*/
- e->store()->diskFull();
+ e->disk().diskFull();
storeConfigure();
}
if (e->swap_filen >= 0)
- e->unlink();
+ e->disk().unlink(*e);
assert(e->swap_status == SWAPOUT_NONE);
e->swap_file_sz = e->objectLen() + mem->swap_hdr_sz;
e->swap_status = SWAPOUT_DONE;
- e->store()->swappedOut(*e);
+ e->disk().swappedOut(*e);
// XXX: For some Stores, it is pointless to re-check cachability here
// and it leads to double counts in store_check_cachable_hist. We need
void
TestSwapDir::parse(int, char*)
{}
-
-StoreSearch *
-TestSwapDir::search(String, HttpRequest *)
-{
- return NULL;
-}
-
#ifndef TEST_TESTSWAPDIR
#define TEST_TESTSWAPDIR
-#include "SwapDir.h"
+#include "store/Disk.h"
class TestSwapDir : public SwapDir
{
virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *);
virtual void parse(int, char*);
- virtual StoreSearch *search(String, HttpRequest *);
+
+ virtual void markForUnlink(StoreEntry &) override {}
+ virtual void unlink(StoreEntry &) override {}
+ virtual bool updateCollapsed(StoreEntry &) override { return false; }
+ virtual bool anchorCollapsed(StoreEntry &, bool &) override { return false; }
};
typedef RefCount<TestSwapDir> TestSwapDirPointer;
void MemStore::reference(StoreEntry &) STUB
void MemStore::maintain() STUB
void MemStore::noteFreeMapSlice(const Ipc::StoreMapSliceId) STUB
-void MemStore::get(String const, STOREGETCLIENT, void *) STUB
void MemStore::init() STUB
void MemStore::getStats(StoreInfoStats&) const STUB
void MemStore::stat(StoreEntry &) const STUB
-int MemStore::callback() STUB_RETVAL(0)
StoreEntry *MemStore::get(const cache_key *) STUB_RETVAL(NULL)
uint64_t MemStore::maxSize() const STUB_RETVAL(0)
uint64_t MemStore::minSize() const STUB_RETVAL(0)
uint64_t MemStore::currentSize() const STUB_RETVAL(0)
uint64_t MemStore::currentCount() const STUB_RETVAL(0)
int64_t MemStore::maxObjectSize() const STUB_RETVAL(0)
-StoreSearch *MemStore::search(String const, HttpRequest *) STUB_RETVAL(NULL)
-bool MemStore::dereference(StoreEntry &, bool) STUB_RETVAL(false)
+bool MemStore::dereference(StoreEntry &) STUB_RETVAL(false)
void MemStore::markForUnlink(StoreEntry&) STUB
bool MemStore::anchorCollapsed(StoreEntry&, bool&) STUB_RETVAL(false)
bool MemStore::updateCollapsed(StoreEntry&) STUB_RETVAL(false)
*/
#include "squid.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
#define STUB_API "SwapDir.cc"
#include "tests/STUB.h"
int64_t SwapDir::maxObjectSize() const STUB_RETVAL(0)
void SwapDir::maxObjectSize(int64_t) STUB
void SwapDir::reference(StoreEntry &) STUB
-bool SwapDir::dereference(StoreEntry &, bool) STUB_RETVAL(false)
-int SwapDir::callback() STUB_RETVAL(0)
+bool SwapDir::dereference(StoreEntry &) STUB_RETVAL(false)
bool SwapDir::canStore(const StoreEntry &, int64_t, int &) const STUB_RETVAL(false)
bool SwapDir::canLog(StoreEntry const &)const STUB_RETVAL(false)
-void SwapDir::sync() STUB
void SwapDir::openLog() STUB
void SwapDir::closeLog() STUB
int SwapDir::writeCleanStart() STUB_RETVAL(0)
bool SwapDir::optionObjectSizeParse(char const *, const char *, int) STUB_RETVAL(false)
void SwapDir::optionObjectSizeDump(StoreEntry *) const STUB
StoreEntry * SwapDir::get(const cache_key *) STUB_RETVAL(NULL)
-void SwapDir::get(String const, STOREGETCLIENT , void *) STUB
const char *memStatusStr[] = { };
const char *swapStatusStr[] = { };
-/* and code defined in the wrong .cc file */
-#include "SwapDir.h"
-void StoreController::maintain() STUB
#include "RemovalPolicy.h"
RemovalPolicy * createRemovalPolicy(RemovalPolicySettings * settings) STUB_RETVAL(NULL)
#include "Store.h"
-StorePointer Store::CurrentRoot = NULL;
StoreIoStats store_io_stats;
bool StoreEntry::checkDeferRead(int fd) const STUB_RETVAL(false)
const char *StoreEntry::getMD5Text() const STUB_RETVAL(NULL)
bool StoreEntry::mayStartSwapOut() STUB_RETVAL(false)
void StoreEntry::trimMemory(const bool preserveSwappable) STUB
void StoreEntry::abort() STUB
-void StoreEntry::unlink() STUB
void StoreEntry::makePublic() STUB
void StoreEntry::makePrivate() STUB
void StoreEntry::setPublicKey() STUB
bool StoreEntry::modifiedSince(HttpRequest * request) const STUB_RETVAL(false)
bool StoreEntry::hasIfMatchEtag(const HttpRequest &request) const STUB_RETVAL(false)
bool StoreEntry::hasIfNoneMatchEtag(const HttpRequest &request) const STUB_RETVAL(false)
-RefCount<SwapDir> StoreEntry::store() const STUB_RETVAL(NULL)
+Store::Disk &StoreEntry::disk() const STUB_RETREF(Store::Disk)
size_t StoreEntry::inUseCount() STUB_RETVAL(0)
void StoreEntry::getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method) STUB
void StoreEntry::getPublicByRequest(StoreClient * aClient, HttpRequest * request) STUB
// private virtual. Why is this linked from outside?
const char *NullStoreEntry::getSerialisedMetaData() STUB_RETVAL(NULL)
-void Store::Root(Store *) STUB
-void Store::Root(RefCount<Store>) STUB
+Store::Controller &Store::Root() STUB_RETREF(Store::Controller)
+void Store::Init(Store::Controller *root) STUB
+void Store::FreeMemory() STUB
void Store::Stats(StoreEntry * output) STUB
void Store::Maintain(void *unused) STUB
-void Store::create() STUB
-void Store::diskFull() STUB
-void Store::sync() STUB
-void Store::unlink(StoreEntry &) STUB
+int Store::Controller::store_dirs_rebuilding = 0;
+StoreSearch *Store::Controller::search() STUB_RETVAL(NULL)
+void Store::Controller::maintain() STUB
std::ostream &operator <<(std::ostream &os, const StoreEntry &)
{
StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) STUB_RETVAL(NULL)
StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL)
StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL)
-void storeInit(void) STUB
void storeConfigure(void) STUB
-void storeFreeMemory(void) STUB
int expiresMoreThan(time_t, time_t) STUB_RETVAL(0)
void storeAppendPrintf(StoreEntry *, const char *,...) STUB
void storeAppendVPrintf(StoreEntry *, const char *, va_list ap) STUB
#include "squid.h"
#include "MemBuf.h"
#include "store_rebuild.h"
-#include "SwapDir.h"
+#include "store/Controller.h"
#include <cstring>
* Please see the COPYING and CONTRIBUTORS files for details.
*/
-#ifndef SQUID_FORWARD_H_
-#define SQUID_FORWARD_H_
+#include "squid.h"
-typedef int32_t sfileno;
-typedef signed int sdirno;
+#define STUB_API "StoreSearch.cc"
+#include "tests/STUB.h"
-#endif /* SQUID_FORWARD_H_ */
+#include "StoreSearch.h"
+#include "store/LocalSearch.h"
+StoreSearch *Store::NewLocalSearch() STUB_RETVAL(NULL)
#include "MemObject.h"
#include "Store.h"
#include "StoreFileSystem.h"
-#include "SwapDir.h"
#include "testDiskIO.h"
#include "testStoreSupport.h"
#include "unitTestMain.h"
testPackableStream::testGetStream()
{
/* Setup a store root so we can create a StoreEntry */
- StorePointer aStore (new TestStore);
- Store::Root(aStore);
+ Store::Init();
CapturingStoreEntry * anEntry = new CapturingStoreEntry();
{
CPPUNIT_ASSERT_EQUAL(String("12345677.7 some text !."), anEntry->_appended_text);
}
delete anEntry; // does the unlock()
- Store::Root(NULL);
+ Store::FreeMemory();
}
#include "RequestFlags.h"
#include "SquidConfig.h"
#include "Store.h"
+#include "store/Disk.h"
+#include "store/Disks.h"
#include "StoreFileSystem.h"
#include "StoreSearch.h"
-#include "SwapDir.h"
#include "testRock.h"
#include "testStoreSupport.h"
#include "unitTestMain.h"
if (Ipc::Mem::Segment::BasePath == NULL)
Ipc::Mem::Segment::BasePath = ".";
- Store::Root(new StoreController);
+ Store::Init();
store = new Rock::SwapDir();
{
CPPUNIT_NS::TestFixture::tearDown();
- Store::Root(NULL);
+ Store::FreeMemory();
store = NULL;
}
StoreSearch *
-TestStore::search(String const url, HttpRequest *)
+TestStore::search()
{
return NULL;
}
void
testStore::testSetRoot()
{
- StorePointer aStore(new TestStore);
- Store::Root(aStore);
+ Store::Controller *aStore(new TestStore);
+ Store::Init(aStore);
- CPPUNIT_ASSERT_EQUAL(&Store::Root(),aStore.getRaw());
- Store::Root(NULL);
+ CPPUNIT_ASSERT_EQUAL(&Store::Root(), aStore);
+ Store::FreeMemory();
}
void
testStore::testUnsetRoot()
{
- StorePointer aStore(new TestStore);
- StorePointer aStore2(new TestStore);
- Store::Root(aStore);
- Store::Root(aStore2);
- CPPUNIT_ASSERT_EQUAL(&Store::Root(),aStore2.getRaw());
- Store::Root(NULL);
+ Store::Controller *aStore(new TestStore);
+ Store::Controller *aStore2(new TestStore);
+ Store::Init(aStore);
+ Store::FreeMemory();
+ Store::Init(aStore2);
+ CPPUNIT_ASSERT_EQUAL(&Store::Root(),aStore2);
+ Store::FreeMemory();
}
void
testStore::testStats()
{
- TestStorePointer aStore(new TestStore);
- Store::Root(aStore.getRaw());
+ TestStore *aStore(new TestStore);
+ Store::Init(aStore);
CPPUNIT_ASSERT_EQUAL(false, aStore->statsCalled);
Store::Stats(NullStoreEntry::getInstance());
CPPUNIT_ASSERT_EQUAL(true, aStore->statsCalled);
- Store::Root(NULL);
+ Store::FreeMemory();
}
void
testStore::testMaxSize()
{
- StorePointer aStore(new TestStore);
- Store::Root(aStore.getRaw());
+ Store::Controller *aStore(new TestStore);
+ Store::Init(aStore);
CPPUNIT_ASSERT_EQUAL(static_cast<uint64_t>(3), aStore->maxSize());
- Store::Root(NULL);
+ Store::FreeMemory();
}
#define SQUID_SRC_TEST_STORE_H
#include "Store.h"
+#include "store/Controlled.h"
#include <cppunit/extensions/HelperMacros.h>
void testMaxSize();
};
-/* subclass of Store to allow testing of methods without having all the
- * other components live
- */
-
-class TestStore : public Store
+/// allows testing of methods without having all the other components live
+class TestStore : public Store::Controller
{
public:
virtual void reference(StoreEntry &) {} /* Reference this object */
- virtual bool dereference(StoreEntry &, bool) { return true; }
+ virtual bool dereference(StoreEntry &) { return true; }
- virtual StoreSearch *search(String const url, HttpRequest *);
+ virtual StoreSearch *search();
};
typedef RefCount<TestStore> TestStorePointer;
#include "SquidConfig.h"
#include "SquidTime.h"
#include "Store.h"
+#include "store/Disks.h"
#include "StoreSearch.h"
-#include "SwapDir.h"
#include "testStoreController.h"
#include "TestSwapDir.h"
void
testStoreController::testStats()
{
+ Store::Init();
StoreEntry *logEntry = new StoreEntry;
logEntry->makeMemObject();
logEntry->mem_obj->setUris("dummy_storeId", NULL, HttpRequestMethod());
logEntry->store_status = STORE_PENDING;
- StorePointer aRoot (new StoreController);
- Store::Root(aRoot);
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
free_cachedir(&Config.cacheSwap);
CPPUNIT_ASSERT_EQUAL(true, aStore->statsCalled);
CPPUNIT_ASSERT_EQUAL(true, aStore2->statsCalled);
- Store::Root(NULL);
+ Store::FreeMemory();
}
static void
logEntry->makeMemObject();
logEntry->mem_obj->setUris("dummy_storeId", NULL, HttpRequestMethod());
logEntry->store_status = STORE_PENDING;
- StorePointer aRoot (new StoreController);
- Store::Root(aRoot);
+ Store::Init();
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
addSwapDir(aStore2);
CPPUNIT_ASSERT_EQUAL(static_cast<uint64_t>(6), Store::Root().maxSize());
free_cachedir(&Config.cacheSwap);
- Store::Root(NULL);
+ Store::FreeMemory();
}
static StoreEntry *
-addedEntry(StorePointer hashStore,
- StorePointer aStore,
+addedEntry(Store::Disk *aStore,
String name,
String varySpec,
String varyKey
e->swap_dirn = -1;
for (int i=0; i < Config.cacheSwap.n_configured; ++i) {
- if (INDEXSD (i) == aStore.getRaw())
+ if (INDEXSD(i) == aStore)
e->swap_dirn = i;
}
testStoreController::testSearch()
{
commonInit();
- StorePointer aRoot (new StoreController());
- Store::Root(aRoot);
+ Store::Init();
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
addSwapDir(aStore2);
Store::Root().init();
- StoreEntry * entry1 = addedEntry (&Store::Root(), aStore.getRaw(), "name", NULL, NULL);
- StoreEntry * entry2 = addedEntry (&Store::Root(), aStore2.getRaw(), "name2", NULL, NULL);
- StoreSearchPointer search = aRoot->search (NULL, NULL); /* search for everything in the store */
+ StoreEntry * entry1 = addedEntry(aStore.getRaw(), "name", NULL, NULL);
+ StoreEntry * entry2 = addedEntry(aStore2.getRaw(), "name2", NULL, NULL);
+ StoreSearchPointer search = Store::Root().search(); /* search for everything in the store */
/* nothing should be immediately available */
CPPUNIT_ASSERT_EQUAL(false, search->error());
CPPUNIT_ASSERT_EQUAL(static_cast<StoreEntry *>(NULL), search->currentItem());
//CPPUNIT_ASSERT_EQUAL(false, search->next());
- Store::Root(NULL);
+ Store::FreeMemory();
}
#include "SquidConfig.h"
#include "SquidTime.h"
#include "Store.h"
-#include "StoreHashIndex.h"
+#include "store/Disks.h"
#include "StoreSearch.h"
-#include "SwapDir.h"
#include "testStoreHashIndex.h"
#include "TestSwapDir.h"
logEntry->makeMemObject();
logEntry->mem_obj->setUris("dummy_storeId", NULL, HttpRequestMethod());
logEntry->store_status = STORE_PENDING;
- StorePointer aRoot (new StoreHashIndex());
- Store::Root(aRoot);
+ Store::Init();
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
free_cachedir(&Config.cacheSwap);
CPPUNIT_ASSERT_EQUAL(true, aStore->statsCalled);
CPPUNIT_ASSERT_EQUAL(true, aStore2->statsCalled);
- Store::Root(NULL);
+ Store::FreeMemory();
}
void
logEntry->makeMemObject();
logEntry->mem_obj->setUris("dummy_storeId", NULL, HttpRequestMethod());
logEntry->store_status = STORE_PENDING;
- StorePointer aRoot (new StoreHashIndex());
- Store::Root(aRoot);
+ Store::Init();
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
addSwapDir(aStore2);
CPPUNIT_ASSERT_EQUAL(static_cast<uint64_t>(6), Store::Root().maxSize());
free_cachedir(&Config.cacheSwap);
- Store::Root(NULL);
+ Store::FreeMemory();
}
StoreEntry *
-addedEntry(StorePointer hashStore,
- StorePointer aStore,
+addedEntry(Store::Disk *aStore,
String name,
String varySpec,
String varyKey
e->swap_dirn = -1;
for (int i=0; i < Config.cacheSwap.n_configured; ++i) {
- if (INDEXSD (i) == aStore.getRaw())
+ if (INDEXSD(i) == aStore)
e->swap_dirn = i;
}
testStoreHashIndex::testSearch()
{
commonInit();
- StorePointer aRoot (new StoreHashIndex());
- Store::Root(aRoot);
+ Store::Init();
TestSwapDirPointer aStore (new TestSwapDir);
TestSwapDirPointer aStore2 (new TestSwapDir);
addSwapDir(aStore);
addSwapDir(aStore2);
Store::Root().init();
- StoreEntry * entry1 = addedEntry (&Store::Root(), aStore.getRaw(), "name", NULL, NULL);
- StoreEntry * entry2 = addedEntry (&Store::Root(), aStore2.getRaw(), "name2", NULL, NULL);
- StoreSearchPointer search = aRoot->search (NULL, NULL); /* search for everything in the store */
+ StoreEntry * entry1 = addedEntry(aStore.getRaw(), "name", NULL, NULL);
+ StoreEntry * entry2 = addedEntry(aStore2.getRaw(), "name2", NULL, NULL);
+ StoreSearchPointer search = Store::Root().search(); /* search for everything in the store */
/* nothing should be immediately available */
CPPUNIT_ASSERT_EQUAL(false, search->error());
CPPUNIT_ASSERT_EQUAL(static_cast<StoreEntry *>(NULL), search->currentItem());
//CPPUNIT_ASSERT_EQUAL(false, search->next());
- Store::Root(NULL);
+ Store::FreeMemory();
}
#include "RequestFlags.h"
#include "SquidConfig.h"
#include "Store.h"
-#include "SwapDir.h"
+#include "store/Disks.h"
#include "testStoreSupport.h"
#include "testUfs.h"
#include "unitTestMain.h"
CPPUNIT_TEST_SUITE_REGISTRATION( testUfs );
-typedef RefCount<Fs::Ufs::UFSSwapDir> SwapDirPointer;
+typedef RefCount<Fs::Ufs::UFSSwapDir> MySwapDirPointer;
extern REMOVALPOLICYCREATE createRemovalPolicy_lru; /* XXX fails with --enable-removal-policies=heap */
static void
-addSwapDir(SwapDirPointer aStore)
+addSwapDir(MySwapDirPointer aStore)
{
allocate_new_swapdir(&Config.cacheSwap);
Config.cacheSwap.swapDirs[Config.cacheSwap.n_configured] = aStore.getRaw();
if (0 > system ("rm -rf " TESTDIR))
throw std::runtime_error("Failed to clean test work directory");
- Store::Root(new StoreController);
+ Store::Init();
- SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking"));
+ MySwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking"));
aStore->IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find("Blocking")->createStrategy());
/* here we cheat: we know that UFSSwapDirs search off disk. If we did an init call to a new
* swapdir instance, we'd not be testing a clean build.
*/
- StoreSearchPointer search = aStore->search (NULL, NULL); /* search for everything in the store */
+ StoreSearchPointer search = Store::Root().search(); /* search for everything in the store */
/* nothing should be immediately available */
#if 0
CPPUNIT_ASSERT_EQUAL(true, search->isDone());
CPPUNIT_ASSERT_EQUAL(static_cast<StoreEntry *>(NULL), search->currentItem());
- Store::Root(NULL);
+ Store::FreeMemory();
free_cachedir(&Config.cacheSwap);
// objects such as "StorePointer aRoot" from being called.
CPPUNIT_ASSERT(!store_table); // or StoreHashIndex ctor will abort below
- Store::Root(new StoreController);
- SwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking"));
+ Store::Init();
+ MySwapDirPointer aStore (new Fs::Ufs::UFSSwapDir("ufs", "Blocking"));
addSwapDir(aStore);
commonInit();
Config.replPolicy = new RemovalPolicySettings;
safe_free(config_line);
CPPUNIT_ASSERT(aStore->IO->io != NULL);
- Store::Root(NULL);
+ Store::FreeMemory();
free_cachedir(&Config.cacheSwap);
safe_free(Config.replPolicy->type);
delete Config.replPolicy;
#include "anyp/PortCfg.h"
#include "base/Subscription.h"
#include "client_side.h"
-#include "disk.h"
+#include "fatal.h"
+#include "fs_io.h"
#include "fde.h"
#include "fqdncache.h"
#include "htcp.h"
#include "SquidConfig.h"
#include "SquidMath.h"
#include "SquidTime.h"
-#include "SwapDir.h"
+#include "store/Disks.h"
#include "tools.h"
#include "wordlist.h"
#include "squid.h"
#if USE_UNLINKD
-#include "disk.h"
+#include "fs_io.h"
#include "fd.h"
#include "fde.h"
#include "globals.h"
+#include "SquidConfig.h"
#include "SquidIpc.h"
#include "SquidTime.h"
#include "StatCounters.h"
-#include "SwapDir.h"
+#include "store/Disk.h"
#include "tools.h"
#include "xusleep.h"
#include "ip/Address.h"
#include "md5.h"
#include "Parsing.h"
+#include "SquidConfig.h"
#include "Store.h"
-#include "SwapDir.h"
#if HAVE_NETDB_H
#include <netdb.h>
projects / thirdparty / squid.git / commitdiff
