/*
- * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
+ * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
*
* Squid software is distributed under GPLv2+ license and includes
* contributions from numerous individuals and organizations.
#define SQUID_STORE_H
#include "base/Packable.h"
+#include "base/Range.h"
#include "base/RefCount.h"
#include "comm/forward.h"
#include "CommRead.h"
#include "http/RequestMethod.h"
#include "HttpReply.h"
#include "MemObject.h"
-#include "Range.h"
#include "RemovalPolicy.h"
+#include "store/Controller.h"
+#include "store/forward.h"
+#include "store_key_md5.h"
#include "StoreIOBuffer.h"
#include "StoreStats.h"
class AsyncCall;
class HttpRequest;
class RequestFlags;
-class StoreClient;
-class StoreSearch;
-class SwapDir;
extern StoreIoStats store_io_stats;
-/// maximum number of entries per cache_dir
-enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
-
class StoreEntry : public hash_link, public Packable
{
static DeferredRead::DeferrableRead DeferReader;
bool checkDeferRead(int fd) const;
- virtual const char *getMD5Text() const;
+ const char *getMD5Text() const;
StoreEntry();
virtual ~StoreEntry();
- virtual HttpReply const *getReply() const;
- virtual void write (StoreIOBuffer);
+ MemObject &mem() { assert(mem_obj); return *mem_obj; }
+ const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
+
+ /// \retval * the address of freshest reply (if mem_obj exists)
+ /// \retval nullptr when mem_obj does not exist
+ /// \see MemObject::freshestReply()
+ const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
- /** Check if the Store entry is emtpty
+ void write(StoreIOBuffer);
+
+ /** Check if the Store entry is empty
* \retval true Store contains 0 bytes of data.
* \retval false Store contains 1 or more bytes of data.
* \retval false Store contains negative content !!!!!!
*/
- virtual bool isEmpty() const {
- assert (mem_obj);
- return mem_obj->endOffset() == 0;
- }
- virtual bool isAccepting() const;
- virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
- virtual void complete();
- virtual store_client_t storeClientType() const;
- virtual char const *getSerialisedMetaData();
+ bool isEmpty() const { return mem().endOffset() == 0; }
+ bool isAccepting() const;
+ size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
+ /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
+ void lengthWentBad(const char *reason);
+ void complete();
+ store_client_t storeClientType() const;
+ /// \returns a malloc()ed buffer containing a length-long packed swap header
+ const char *getSerialisedMetaData(size_t &length) const;
/// Store a prepared error response. MemObject locks the reply object.
void storeErrorResponse(HttpReply *reply);
- void replaceHttpReply(HttpReply *, bool andStartWriting = true);
+ void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
void startWriting(); ///< pack and write reply headers and, maybe, body
/// whether we may start writing to disk (now or in the future)
- virtual bool mayStartSwapOut();
- virtual void trimMemory(const bool preserveSwappable);
+ bool mayStartSwapOut();
+ void trimMemory(const bool preserveSwappable);
// called when a decision to cache in memory has been made
void memOutDecision(const bool willCacheInRam);
void swapOutDecision(const MemObject::SwapOut::Decision &decision);
void abort();
- void unlink();
- void makePublic();
- void makePrivate();
- void setPublicKey();
- void setPrivateKey();
+ bool makePublic(const KeyScope keyScope = ksDefault);
+ void makePrivate(const bool shareable);
+ /// A low-level method just resetting "private key" flags.
+ /// To avoid key inconsistency please use forcePublicKey()
+ /// or similar instead.
+ void clearPrivate();
+ bool setPublicKey(const KeyScope keyScope = ksDefault);
+ /// Resets existing public key to a public key with default scope,
+ /// releasing the old default-scope entry (if any).
+ /// Does nothing if the existing public key already has default scope.
+ void clearPublicKeyScope();
+
+ /// \returns public key (if the entry has it) or nil (otherwise)
+ const cache_key *publicKey() const {
+ return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
+ reinterpret_cast<const cache_key*>(key): // may be nil
+ nullptr;
+ }
+
+ /// Either fills this entry with private key or changes the existing key
+ /// from public to private.
+ /// \param permanent whether this entry should be private forever.
+ void setPrivateKey(const bool shareable, const bool permanent);
+
void expireNow();
- void releaseRequest();
+ /// Makes the StoreEntry private and marks the corresponding entry
+ /// for eventual removal from the Store.
+ void releaseRequest(const bool shareable = false);
void negativeCache();
- void cacheNegatively(); /** \todo argh, why both? */
+ bool cacheNegatively(); /** \todo argh, why both? */
void invokeHandlers();
- void purgeMem();
void cacheInMemory(); ///< start or continue storing in memory cache
void swapOut();
/// whether we are in the process of writing this entry to disk
bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
+ /// whether the entire entry is now on disk (possibly marked for deletion)
+ bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
+ /// whether we failed to write this entry to disk
+ bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
void swapOutFileClose(int how);
const char *url() const;
/// Satisfies cachability requirements shared among disk and RAM caches.
/// TODO: Rename and make private so only those two methods can call this.
bool checkCachable();
int checkNegativeHit() const;
- int locked() const;
+ int locked() const { return lock_count; }
int validToSend() const;
bool memoryCachable(); ///< checkCachable() and can be cached in memory
- /// if needed, initialize mem_obj member w/o URI-related information
- MemObject *makeMemObject();
+ /// initialize mem_obj; assert if mem_obj already exists
+ /// avoid this method in favor of createMemObject(trio)!
+ void createMemObject();
- /// initialize mem_obj member (if needed) and supply URI-related info
+ /// initialize mem_obj with URIs/method; assert if mem_obj already exists
void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
+ /// initialize mem_obj (if needed) and set URIs/method (if missing)
+ void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
+
void dump(int debug_lvl) const;
void hashDelete();
void hashInsert(const cache_key *);
void registerAbort(STABH * cb, void *);
void reset();
void setMemStatus(mem_status_t);
- void timestampsSet();
+ bool timestampsSet();
void unregisterAbort();
void destroyMemObject();
int checkTooSmall();
void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
void setNoDelay (bool const);
- bool modifiedSince(HttpRequest * request) const;
+ void lastModified(const time_t when) { lastModified_ = when; }
+ /// \returns entry's 'effective' modification time
+ time_t lastModified() const {
+ // may still return -1 if timestamp is not set
+ return lastModified_ < 0 ? timestamp : lastModified_;
+ }
+ /// \returns a formatted string with entry's timestamps
+ const char *describeTimestamps() const;
+ // TODO: consider removing currently unsupported imslen parameter
+ bool modifiedSince(const time_t ims, const int imslen = -1) const;
/// has ETag matching at least one of the If-Match etags
bool hasIfMatchEtag(const HttpRequest &request) const;
/// has ETag matching at least one of the If-None-Match etags
/// whether this entry has an ETag; if yes, puts ETag value into parameter
bool hasEtag(ETag &etag) const;
- /** What store does this entry belong too ? */
- virtual RefCount<SwapDir> store() const;
+ /// Updates easily-accessible non-Store-specific parts of the entry.
+ /// Use Controller::updateOnNotModified() instead of this helper.
+ /// \returns whether anything was actually updated
+ bool updateOnNotModified(const StoreEntry &e304);
+
+ /// the disk this entry is [being] cached on; asserts for entries w/o a disk
+ Store::Disk &disk() const;
+ /// whether one of this StoreEntry owners has locked the corresponding
+ /// disk entry (at the specified disk entry coordinates, if any)
+ bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
+ /// Makes hasDisk(dirn, filn) true. The caller should have locked
+ /// the corresponding disk store entry for reading or writing.
+ void attachToDisk(const sdirno, const sfileno, const swap_status_t);
+ /// Makes hasDisk() false. The caller should have unlocked
+ /// the corresponding disk store entry.
+ void detachFromDisk();
+
+ /// whether there is a corresponding locked transients table entry
+ bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
+ /// whether there is a corresponding locked shared memory table entry
+ bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
+
+ /// whether this entry can feed collapsed requests and only them
+ bool hittingRequiresCollapsing() const { return EBIT_TEST(flags, ENTRY_REQUIRES_COLLAPSING); }
+
+ /// allow or forbid collapsed requests feeding
+ void setCollapsingRequirement(const bool required);
MemObject *mem_obj;
RemovalPolicyNode repl;
time_t timestamp;
time_t lastref;
time_t expires;
- time_t lastmod;
+private:
+ time_t lastModified_; ///< received Last-Modified value or -1; use lastModified()
+public:
uint64_t swap_file_sz;
uint16_t refcount;
uint16_t flags;
static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
- virtual bool isNull() {
- return false;
- };
-
void *operator new(size_t byteCount);
void operator delete(void *address);
- void setReleaseFlag();
#if USE_SQUID_ESI
ESIElement::Pointer cachedESITree;
#endif
- /** disable sending content to the clients */
- virtual void buffer();
- /** flush any buffered content */
- virtual void flush();
- virtual int64_t objectLen() const;
- virtual int64_t contentLen() const;
+ int64_t objectLen() const { return mem().object_sz; }
+ int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
/// claim shared ownership of this entry (for use in a given context)
/// matching lock() and unlock() contexts eases leak triage but is optional
/// update last reference timestamp and related Store metadata
void touch();
- virtual void release();
+ /// One of the three methods to get rid of an unlocked StoreEntry object.
+ /// Removes all unlocked (and marks for eventual removal all locked) Store
+ /// entries, including attached and unattached entries that have our key.
+ /// Also destroys us if we are unlocked or makes us private otherwise.
+ void release(const bool shareable = false);
+
+ /// One of the three methods to get rid of an unlocked StoreEntry object.
+ /// May destroy this object if it is unlocked; does nothing otherwise.
+ /// Unlike release(), may not trigger eviction of underlying store entries,
+ /// but, unlike destroyStoreEntry(), does honor an earlier release request.
+ void abandon(const char *context) { if (!locked()) doAbandon(context); }
+
+ /// May the caller commit to treating this [previously locked]
+ /// entry as a cache hit?
+ bool mayStartHitting() const {
+ return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
+ }
#if USE_ADAPTATION
/// call back producer when more buffer space is available
/* Packable API */
virtual void append(char const *, int);
virtual void vappendf(const char *, va_list);
+ virtual void buffer();
+ virtual void flush();
protected:
+ typedef Store::EntryGuard EntryGuard;
+
void transientsAbandonmentCheck();
+ /// does nothing except throwing if disk-associated data members are inconsistent
+ void checkDisk() const;
private:
+ void doAbandon(const char *context);
bool checkTooBig() const;
+ void forcePublicKey(const cache_key *newkey);
+ StoreEntry *adjustVary();
+ const cache_key *calcPublicKey(const KeyScope keyScope);
static MemAllocator *pool;
unsigned short lock_count; /* Assume < 65536! */
+ /// Nobody can find/lock KEY_PRIVATE entries, but some transactions
+ /// (e.g., collapsed requests) find/lock a public entry before it becomes
+ /// private. May such transactions start using the now-private entry
+ /// they previously locked? This member should not affect transactions
+ /// that already started reading from the entry.
+ bool shareableWhenPrivate;
+
#if USE_ADAPTATION
/// producer callback registered with deferProducer
AsyncCall::Pointer deferredProducer;
bool validLength() const;
bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
+
+ friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
};
std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
-/// \ingroup StoreAPI
-class NullStoreEntry:public StoreEntry
-{
-
-public:
- static NullStoreEntry *getInstance();
- bool isNull() {
- return true;
- }
-
- const char *getMD5Text() const;
- HttpReply const *getReply() const { return NULL; }
- void write (StoreIOBuffer) {}
-
- bool isEmpty () const {return true;}
-
- virtual size_t bytesWanted(Range<size_t> const aRange, bool) const { return aRange.end; }
-
- void operator delete(void *address);
- void complete() {}
-
-private:
- store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
-
- char const *getSerialisedMetaData();
- virtual bool mayStartSwapOut() { return false; }
-
- void trimMemory(const bool) {}
-
- static NullStoreEntry _instance;
-};
-
/// \ingroup StoreAPI
typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
-/**
- \ingroup StoreAPI
- * Abstract base class that will replace the whole store and swapdir interface.
- */
-class Store : public RefCountable
-{
+namespace Store {
+/// a smart pointer similar to std::unique_ptr<> that automatically
+/// release()s and unlock()s the guarded Entry on stack-unwinding failures
+class EntryGuard {
public:
- /** The root store */
- static Store &Root() {
- if (CurrentRoot == NULL)
- fatal("No Store Root has been set");
- return *CurrentRoot;
+ /// \param entry either nil or a locked Entry to manage
+ /// \param context default unlock() message
+ EntryGuard(Entry *entry, const char *context):
+ entry_(entry), context_(context) {
+ assert(!entry_ || entry_->locked());
}
- static void Root(Store *);
- static void Root(RefCount<Store>);
- static void Stats(StoreEntry * output);
- static void Maintain(void *unused);
-
- virtual ~Store() {}
-
- /** Handle pending callbacks - called by the event loop. */
- virtual int callback() = 0;
-
- /** create the resources needed for this store to operate */
- virtual void create();
-
- /**
- * Notify this store that its disk is full.
- \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
- */
- virtual void diskFull();
-
- /** Retrieve a store entry from the store */
- virtual StoreEntry * get(const cache_key *) = 0;
-
- /** \todo imeplement the async version */
- virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
-
- /* prepare the store for use. The store need not be usable immediately,
- * it should respond to readable() and writable() with true as soon
- * as it can provide those services
- */
- virtual void init() = 0;
-
- /**
- * The maximum size the store will support in normal use. Inaccuracy is permitted,
- * but may throw estimates for memory etc out of whack.
- */
- virtual uint64_t maxSize() const = 0;
-
- /** The minimum size the store will shrink to via normal housekeeping */
- virtual uint64_t minSize() const = 0;
-
- /** current store size */
- virtual uint64_t currentSize() const = 0;
-
- /** the total number of objects stored */
- virtual uint64_t currentCount() const = 0;
-
- /** the maximum object size that can be stored, -1 if unlimited */
- virtual int64_t maxObjectSize() const = 0;
-
- /// collect cache storage-related statistics
- virtual void getStats(StoreInfoStats &stats) const = 0;
-
- /**
- * Output stats to the provided store entry.
- \todo make these calls asynchronous
- */
- virtual void stat(StoreEntry &) const = 0;
-
- /** Sync the store prior to shutdown */
- virtual void sync();
- /** remove a Store entry from the store */
- virtual void unlink (StoreEntry &);
-
- /* search in the store */
- virtual StoreSearch *search(String const url, HttpRequest *) = 0;
-
- /* pulled up from SwapDir for migration.... probably do not belong here */
- virtual void reference(StoreEntry &) = 0; /* Reference this object */
-
- /// Undo reference(), returning false iff idle e should be destroyed
- virtual bool dereference(StoreEntry &, bool wantsLocalMemory) = 0;
-
- virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// informs stores that this entry will be eventually unlinked
- virtual void markForUnlink(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // because test cases use non-StoreController derivatives as Root
- /// called when the entry is no longer needed by any transaction
- virtual void handleIdleEntry(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // because test cases use non-StoreController derivatives as Root
- /// called to get rid of no longer needed entry data in RAM, if any
- virtual void memoryOut(StoreEntry &, const bool /*preserveSwappable*/) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// makes the entry available for collapsing future requests
- virtual void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// marks the entry completed for collapsed requests
- virtual void transientsCompleteWriting(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// Update local intransit entry after changes made by appending worker.
- virtual void syncCollapsed(const sfileno) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// calls Root().transients->abandon() if transients are tracked
- virtual void transientsAbandon(StoreEntry &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// number of the transient entry readers some time ago
- virtual int transientReaders(const StoreEntry &) const { return 0; }
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// disassociates the entry from the intransit table
- virtual void transientsDisconnect(MemObject &) {}
-
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// removes the entry from the memory cache
- virtual void memoryUnlink(StoreEntry &) {}
+ ~EntryGuard() {
+ if (entry_) {
+ // something went wrong -- the caller did not unlockAndReset() us
+ onException();
+ }
+ }
- // XXX: This method belongs to Store::Root/StoreController, but it is here
- // to avoid casting Root() to StoreController until Root() API is fixed.
- /// disassociates the entry from the memory cache, preserving cached data
- virtual void memoryDisconnect(StoreEntry &) {}
+ EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
- /// If the entry is not found, return false. Otherwise, return true after
- /// tying the entry to this cache and setting inSync to updateCollapsed().
- virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
+ /// like std::unique_ptr::get()
+ /// \returns nil or the guarded (locked) entry
+ Entry *get() {
+ return entry_;
+ }
- /// update a local collapsed entry with fresh info from this cache (if any)
- virtual bool updateCollapsed(StoreEntry &) { return false; }
+ /// like std::unique_ptr::reset()
+ /// stops guarding the entry
+ /// unlocks the entry (which may destroy it)
+ void unlockAndReset(const char *resetContext = nullptr) {
+ if (entry_) {
+ entry_->unlock(resetContext ? resetContext : context_);
+ entry_ = nullptr;
+ }
+ }
private:
- static RefCount<Store> CurrentRoot;
+ void onException() noexcept;
+
+ Entry *entry_; ///< the guarded Entry or nil
+ const char *context_; ///< default unlock() message
};
-/// \ingroup StoreAPI
-typedef RefCount<Store> StorePointer;
+void Stats(StoreEntry *output);
+void Maintain(void *unused);
+}; // namespace Store
/// \ingroup StoreAPI
size_t storeEntryInUse();
StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
/// \ingroup StoreAPI
-StoreEntry *storeGetPublicByRequest(HttpRequest * request);
+StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
/// \ingroup StoreAPI
-StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
+StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
/// \ingroup StoreAPI
/// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
/// \ingroup StoreAPI
/// Creates a new StoreEntry with mem_obj and sets initial flags/states.
-StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
+StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
/// \ingroup StoreAPI
void storeInit(void);
/// \ingroup StoreAPI
int storeTooManyDiskFilesOpen(void);
-class SwapDir;
/// \ingroup StoreAPI
void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
/// \ingroup StoreAPI
void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
-/// \ingroup StoreAPI
+/// One of the three methods to get rid of an unlocked StoreEntry object.
+/// This low-level method ignores lock()ing and release() promises. It never
+/// leaves the entry in the local store_table.
+/// TODO: Hide by moving its functionality into the StoreEntry destructor.
extern FREE destroyStoreEntry;
/// \ingroup StoreAPI