src/Store.h

   1 /*
   2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 #include "base/Packable.h"
  13 #include "base/Range.h"
  14 #include "base/RefCount.h"
  15 #include "comm/forward.h"
  16 #include "CommRead.h"
  17 #include "hash.h"
  18 #include "http/forward.h"
  19 #include "http/RequestMethod.h"
  20 #include "HttpReply.h"
  21 #include "MemObject.h"
  22 #include "RemovalPolicy.h"
  23 #include "store/Controller.h"
  24 #include "store/forward.h"
  25 #include "store_key_md5.h"
  26 #include "StoreIOBuffer.h"
  27 #include "StoreStats.h"
  28
  29 #if USE_SQUID_ESI
  30 #include "esi/Element.h"
  31 #endif
  32
  33 #include <ostream>
  34
  35 class AsyncCall;
  36 class HttpRequest;
  37 class RequestFlags;
  38
  39 extern StoreIoStats store_io_stats;
  40
  41 class StoreEntry : public hash_link, public Packable
  42 {
  43
  44 public:
  45     static DeferredRead::DeferrableRead DeferReader;
  46     bool checkDeferRead(int fd) const;
  47
  48     const char *getMD5Text() const;
  49     StoreEntry();
  50     virtual ~StoreEntry();
  51
  52     MemObject &mem() { assert(mem_obj); return *mem_obj; }
  53     const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
  54
  55     /// \retval * the address of freshest reply (if mem_obj exists)
  56     /// \retval nullptr when mem_obj does not exist
  57     /// \see MemObject::freshestReply()
  58     const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
  59
  60     void write(StoreIOBuffer);
  61
  62     /** Check if the Store entry is empty
  63      * \retval true   Store contains 0 bytes of data.
  64      * \retval false  Store contains 1 or more bytes of data.
  65      * \retval false  Store contains negative content !!!!!!
  66      */
  67     bool isEmpty() const { return mem().endOffset() == 0; }
  68     bool isAccepting() const;
  69     size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  70     /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
  71     void lengthWentBad(const char *reason);
  72     void complete();
  73     store_client_t storeClientType() const;
  74     /// \returns a malloc()ed buffer containing a length-long packed swap header
  75     const char *getSerialisedMetaData(size_t &length) const;
  76     /// Store a prepared error response. MemObject locks the reply object.
  77     void storeErrorResponse(HttpReply *reply);
  78     void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
  79     void startWriting(); ///< pack and write reply headers and, maybe, body
  80     /// whether we may start writing to disk (now or in the future)
  81     bool mayStartSwapOut();
  82     void trimMemory(const bool preserveSwappable);
  83
  84     // called when a decision to cache in memory has been made
  85     void memOutDecision(const bool willCacheInRam);
  86     // called when a decision to cache on disk has been made
  87     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  88
  89     void abort();
  90     bool makePublic(const KeyScope keyScope = ksDefault);
  91     void makePrivate(const bool shareable);
  92     /// A low-level method just resetting "private key" flags.
  93     /// To avoid key inconsistency please use forcePublicKey()
  94     /// or similar instead.
  95     void clearPrivate();
  96     bool setPublicKey(const KeyScope keyScope = ksDefault);
  97     /// Resets existing public key to a public key with default scope,
  98     /// releasing the old default-scope entry (if any).
  99     /// Does nothing if the existing public key already has default scope.
 100     void clearPublicKeyScope();
 101
 102     /// \returns public key (if the entry has it) or nil (otherwise)
 103     const cache_key *publicKey() const {
 104         return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
 105                reinterpret_cast<const cache_key*>(key): // may be nil
 106                nullptr;
 107     }
 108
 109     /// Either fills this entry with private key or changes the existing key
 110     /// from public to private.
 111     /// \param permanent whether this entry should be private forever.
 112     void setPrivateKey(const bool shareable, const bool permanent);
 113
 114     void expireNow();
 115     /// Makes the StoreEntry private and marks the corresponding entry
 116     /// for eventual removal from the Store.
 117     void releaseRequest(const bool shareable = false);
 118     void negativeCache();
 119     bool cacheNegatively();     // TODO: why both negativeCache() and cacheNegatively() ?
 120     void invokeHandlers();
 121     void cacheInMemory(); ///< start or continue storing in memory cache
 122     void swapOut();
 123     /// whether we are in the process of writing this entry to disk
 124     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 125     /// whether the entire entry is now on disk (possibly marked for deletion)
 126     bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
 127     /// whether we failed to write this entry to disk
 128     bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
 129     void swapOutFileClose(int how);
 130     const char *url() const;
 131     /// Satisfies cachability requirements shared among disk and RAM caches.
 132     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 133     /// TODO: Rename and make private so only those two methods can call this.
 134     bool checkCachable();
 135     int checkNegativeHit() const;
 136     int locked() const { return lock_count; }
 137     int validToSend() const;
 138     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 139
 140     /// initialize mem_obj; assert if mem_obj already exists
 141     /// avoid this method in favor of createMemObject(trio)!
 142     void createMemObject();
 143
 144     /// initialize mem_obj with URIs/method; assert if mem_obj already exists
 145     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 146
 147     /// initialize mem_obj (if needed) and set URIs/method (if missing)
 148     void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 149
 150     void dump(int debug_lvl) const;
 151     void hashDelete();
 152     void hashInsert(const cache_key *);
 153     /// notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
 154     void registerAbortCallback(const AsyncCall::Pointer &);
 155     void reset();
 156     void setMemStatus(mem_status_t);
 157     bool timestampsSet();
 158     /// Avoid notifying anybody about a 3rd-party initiated StoreEntry abort.
 159     /// Calling this method does not cancel the already queued notification.
 160     /// TODO: Refactor to represent the end of (shared) ownership by our writer.
 161     void unregisterAbortCallback(const char *reason);
 162     void destroyMemObject();
 163     int checkTooSmall();
 164
 165     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 166
 167     void setNoDelay (bool const);
 168     void lastModified(const time_t when) { lastModified_ = when; }
 169     /// \returns entry's 'effective' modification time
 170     time_t lastModified() const {
 171         // may still return -1 if timestamp is not set
 172         return lastModified_ < 0 ? timestamp : lastModified_;
 173     }
 174     /// \returns a formatted string with entry's timestamps
 175     const char *describeTimestamps() const;
 176     // TODO: consider removing currently unsupported imslen parameter
 177     bool modifiedSince(const time_t ims, const int imslen = -1) const;
 178     /// has ETag matching at least one of the If-Match etags
 179     bool hasIfMatchEtag(const HttpRequest &request) const;
 180     /// has ETag matching at least one of the If-None-Match etags
 181     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 182     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 183     bool hasEtag(ETag &etag) const;
 184
 185     /// Updates easily-accessible non-Store-specific parts of the entry.
 186     /// Use Controller::updateOnNotModified() instead of this helper.
 187     /// \returns whether anything was actually updated
 188     bool updateOnNotModified(const StoreEntry &e304);
 189
 190     /// the disk this entry is [being] cached on; asserts for entries w/o a disk
 191     Store::Disk &disk() const;
 192     /// whether one of this StoreEntry owners has locked the corresponding
 193     /// disk entry (at the specified disk entry coordinates, if any)
 194     bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
 195     /// Makes hasDisk(dirn, filn) true. The caller should have locked
 196     /// the corresponding disk store entry for reading or writing.
 197     void attachToDisk(const sdirno, const sfileno, const swap_status_t);
 198     /// Makes hasDisk() false. The caller should have unlocked
 199     /// the corresponding disk store entry.
 200     void detachFromDisk();
 201
 202     /// whether there is a corresponding locked transients table entry
 203     bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
 204     /// whether there is a corresponding locked shared memory table entry
 205     bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
 206
 207     /// whether this entry can feed collapsed requests and only them
 208     bool hittingRequiresCollapsing() const { return EBIT_TEST(flags, ENTRY_REQUIRES_COLLAPSING); }
 209
 210     /// allow or forbid collapsed requests feeding
 211     void setCollapsingRequirement(const bool required);
 212
 213     MemObject *mem_obj;
 214     RemovalPolicyNode repl;
 215     /* START OF ON-DISK STORE_META_STD TLV field */
 216     time_t timestamp;
 217     time_t lastref;
 218     time_t expires;
 219 private:
 220     time_t lastModified_; ///< received Last-Modified value or -1; use lastModified()
 221 public:
 222     uint64_t swap_file_sz;
 223     uint16_t refcount;
 224     uint16_t flags;
 225     /* END OF ON-DISK STORE_META_STD */
 226
 227     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 228     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 229
 230     sdirno swap_dirn:7;
 231
 232     mem_status_t mem_status:3;
 233
 234     ping_status_t ping_status:3;
 235
 236     store_status_t store_status:3;
 237
 238     swap_status_t swap_status:3;
 239
 240 public:
 241     static size_t inUseCount();
 242
 243     void *operator new(size_t byteCount);
 244     void operator delete(void *address);
 245 #if USE_SQUID_ESI
 246
 247     ESIElement::Pointer cachedESITree;
 248 #endif
 249     int64_t objectLen() const { return mem().object_sz; }
 250     int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
 251
 252     /// claim shared ownership of this entry (for use in a given context)
 253     /// matching lock() and unlock() contexts eases leak triage but is optional
 254     void lock(const char *context);
 255
 256     /// disclaim shared ownership; may remove entry from store and delete it
 257     /// returns remaining lock level (zero for unlocked and possibly gone entry)
 258     int unlock(const char *context);
 259
 260     /// returns a local concurrent use counter, for debugging
 261     int locks() const { return static_cast<int>(lock_count); }
 262
 263     /// update last reference timestamp and related Store metadata
 264     void touch();
 265
 266     /// One of the three methods to get rid of an unlocked StoreEntry object.
 267     /// Removes all unlocked (and marks for eventual removal all locked) Store
 268     /// entries, including attached and unattached entries that have our key.
 269     /// Also destroys us if we are unlocked or makes us private otherwise.
 270     void release(const bool shareable = false);
 271
 272     /// One of the three methods to get rid of an unlocked StoreEntry object.
 273     /// May destroy this object if it is unlocked; does nothing otherwise.
 274     /// Unlike release(), may not trigger eviction of underlying store entries,
 275     /// but, unlike destroyStoreEntry(), does honor an earlier release request.
 276     void abandon(const char *context) { if (!locked()) doAbandon(context); }
 277
 278     /// May the caller commit to treating this [previously locked]
 279     /// entry as a cache hit?
 280     bool mayStartHitting() const {
 281         return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
 282     }
 283
 284 #if USE_ADAPTATION
 285     /// call back producer when more buffer space is available
 286     void deferProducer(const AsyncCall::Pointer &producer);
 287     /// calls back producer registered with deferProducer
 288     void kickProducer();
 289 #endif
 290
 291     /* Packable API */
 292     virtual void append(char const *, int);
 293     virtual void vappendf(const char *, va_list);
 294     virtual void buffer();
 295     virtual void flush();
 296
 297 protected:
 298     typedef Store::EntryGuard EntryGuard;
 299
 300     void transientsAbandonmentCheck();
 301     /// does nothing except throwing if disk-associated data members are inconsistent
 302     void checkDisk() const;
 303
 304 private:
 305     void doAbandon(const char *context);
 306     bool checkTooBig() const;
 307     void forcePublicKey(const cache_key *newkey);
 308     StoreEntry *adjustVary();
 309     const cache_key *calcPublicKey(const KeyScope keyScope);
 310
 311     static MemAllocator *pool;
 312
 313     unsigned short lock_count;      /* Assume < 65536! */
 314
 315     /// Nobody can find/lock KEY_PRIVATE entries, but some transactions
 316     /// (e.g., collapsed requests) find/lock a public entry before it becomes
 317     /// private. May such transactions start using the now-private entry
 318     /// they previously locked? This member should not affect transactions
 319     /// that already started reading from the entry.
 320     bool shareableWhenPrivate;
 321
 322 #if USE_ADAPTATION
 323     /// producer callback registered with deferProducer
 324     AsyncCall::Pointer deferredProducer;
 325 #endif
 326
 327     bool validLength() const;
 328     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 329
 330     friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 331 };
 332
 333 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 334
 335 /// \ingroup StoreAPI
 336 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 337
 338 namespace Store {
 339
 340 /// a smart pointer similar to std::unique_ptr<> that automatically
 341 /// release()s and unlock()s the guarded Entry on stack-unwinding failures
 342 class EntryGuard {
 343 public:
 344     /// \param entry either nil or a locked Entry to manage
 345     /// \param context default unlock() message
 346     EntryGuard(Entry *entry, const char *context):
 347         entry_(entry), context_(context) {
 348         assert(!entry_ || entry_->locked());
 349     }
 350
 351     ~EntryGuard() {
 352         if (entry_) {
 353             // something went wrong -- the caller did not unlockAndReset() us
 354             onException();
 355         }
 356     }
 357
 358     EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
 359
 360     /// like std::unique_ptr::get()
 361     /// \returns nil or the guarded (locked) entry
 362     Entry *get() {
 363         return entry_;
 364     }
 365
 366     /// like std::unique_ptr::reset()
 367     /// stops guarding the entry
 368     /// unlocks the entry (which may destroy it)
 369     void unlockAndReset(const char *resetContext = nullptr) {
 370         if (entry_) {
 371             entry_->unlock(resetContext ? resetContext : context_);
 372             entry_ = nullptr;
 373         }
 374     }
 375
 376 private:
 377     void onException() noexcept;
 378
 379     Entry *entry_; ///< the guarded Entry or nil
 380     const char *context_; ///< default unlock() message
 381 };
 382
 383 void Stats(StoreEntry *output);
 384 void Maintain(void *unused);
 385 }; // namespace Store
 386
 387 /// \ingroup StoreAPI
 388 size_t storeEntryInUse();
 389
 390 /// \ingroup StoreAPI
 391 const char *storeEntryFlags(const StoreEntry *);
 392
 393 /// \ingroup StoreAPI
 394 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 395
 396 /// \ingroup StoreAPI
 397 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 398
 399 /// \ingroup StoreAPI
 400 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
 401
 402 /// \ingroup StoreAPI
 403 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
 404
 405 /// \ingroup StoreAPI
 406 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 407 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 408
 409 /// \ingroup StoreAPI
 410 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 411 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
 412
 413 /// \ingroup StoreAPI
 414 void storeInit(void);
 415
 416 /// \ingroup StoreAPI
 417 void storeConfigure(void);
 418
 419 /// \ingroup StoreAPI
 420 void storeFreeMemory(void);
 421
 422 /// \ingroup StoreAPI
 423 int expiresMoreThan(time_t, time_t);
 424
 425 /// \ingroup StoreAPI
 426 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 427
 428 /// \ingroup StoreAPI
 429 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 430
 431 /// \ingroup StoreAPI
 432 int storeTooManyDiskFilesOpen(void);
 433
 434 /// \ingroup StoreAPI
 435 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 436
 437 /// \ingroup StoreAPI
 438 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 439
 440 /// \ingroup StoreAPI
 441 void storeFsInit(void);
 442
 443 /// \ingroup StoreAPI
 444 void storeFsDone(void);
 445
 446 /// \ingroup StoreAPI
 447 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 448
 449 /// One of the three methods to get rid of an unlocked StoreEntry object.
 450 /// This low-level method ignores lock()ing and release() promises. It never
 451 /// leaves the entry in the local store_table.
 452 /// TODO: Hide by moving its functionality into the StoreEntry destructor.
 453 extern FREE destroyStoreEntry;
 454
 455 /// \ingroup StoreAPI
 456 void storeGetMemSpace(int size);
 457
 458 #endif /* SQUID_STORE_H */
 459