src/Store.h

   1 /*
   2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 #include "base/Packable.h"
  13 #include "base/Range.h"
  14 #include "base/RefCount.h"
  15 #include "comm/forward.h"
  16 #include "CommRead.h"
  17 #include "hash.h"
  18 #include "http/forward.h"
  19 #include "http/RequestMethod.h"
  20 #include "HttpReply.h"
  21 #include "MemObject.h"
  22 #include "RemovalPolicy.h"
  23 #include "store/Controller.h"
  24 #include "store/forward.h"
  25 #include "store_key_md5.h"
  26 #include "StoreIOBuffer.h"
  27 #include "StoreStats.h"
  28
  29 #if USE_SQUID_ESI
  30 #include "esi/Element.h"
  31 #endif
  32
  33 #include <ostream>
  34
  35 class AsyncCall;
  36 class HttpRequest;
  37 class RequestFlags;
  38
  39 extern StoreIoStats store_io_stats;
  40
  41 class StoreEntry : public hash_link, public Packable
  42 {
  43
  44 public:
  45     static DeferredRead::DeferrableRead DeferReader;
  46     bool checkDeferRead(int fd) const;
  47
  48     const char *getMD5Text() const;
  49     StoreEntry();
  50     virtual ~StoreEntry();
  51
  52     MemObject &mem() { assert(mem_obj); return *mem_obj; }
  53     const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
  54
  55     /// \retval * the address of freshest reply (if mem_obj exists)
  56     /// \retval nullptr when mem_obj does not exist
  57     /// \see MemObject::freshestReply()
  58     const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
  59
  60     void write(StoreIOBuffer);
  61
  62     /** Check if the Store entry is empty
  63      * \retval true   Store contains 0 bytes of data.
  64      * \retval false  Store contains 1 or more bytes of data.
  65      * \retval false  Store contains negative content !!!!!!
  66      */
  67     bool isEmpty() const { return mem().endOffset() == 0; }
  68     bool isAccepting() const;
  69     size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  70
  71     /// Signals that the entire response has been stored and no more append()
  72     /// calls should be expected; cf. completeTruncated().
  73     void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply);
  74
  75     /// Signals that a partial response (if any) has been stored but no more
  76     /// append() calls should be expected; cf. completeSuccessfully().
  77     void completeTruncated(const char *whyWeConsiderTheReplyTruncated);
  78
  79     /// \deprecated use either completeSuccessfully() or completeTruncated() instead
  80     void complete();
  81
  82     store_client_t storeClientType() const;
  83     /// \returns a malloc()ed buffer containing a length-long packed swap header
  84     const char *getSerialisedMetaData(size_t &length) const;
  85     /// Store a prepared error response. MemObject locks the reply object.
  86     void storeErrorResponse(HttpReply *reply);
  87     void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
  88     void startWriting(); ///< pack and write reply headers and, maybe, body
  89     /// whether we may start writing to disk (now or in the future)
  90     bool mayStartSwapOut();
  91     void trimMemory(const bool preserveSwappable);
  92
  93     // called when a decision to cache in memory has been made
  94     void memOutDecision(const bool willCacheInRam);
  95     // called when a decision to cache on disk has been made
  96     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  97
  98     void abort();
  99     bool makePublic(const KeyScope keyScope = ksDefault);
 100     void makePrivate(const bool shareable);
 101     /// A low-level method just resetting "private key" flags.
 102     /// To avoid key inconsistency please use forcePublicKey()
 103     /// or similar instead.
 104     void clearPrivate();
 105     bool setPublicKey(const KeyScope keyScope = ksDefault);
 106     /// Resets existing public key to a public key with default scope,
 107     /// releasing the old default-scope entry (if any).
 108     /// Does nothing if the existing public key already has default scope.
 109     void clearPublicKeyScope();
 110
 111     /// \returns public key (if the entry has it) or nil (otherwise)
 112     const cache_key *publicKey() const {
 113         return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
 114                reinterpret_cast<const cache_key*>(key): // may be nil
 115                nullptr;
 116     }
 117
 118     /// Either fills this entry with private key or changes the existing key
 119     /// from public to private.
 120     /// \param permanent whether this entry should be private forever.
 121     void setPrivateKey(const bool shareable, const bool permanent);
 122
 123     void expireNow();
 124     /// Makes the StoreEntry private and marks the corresponding entry
 125     /// for eventual removal from the Store.
 126     void releaseRequest(const bool shareable = false);
 127     void negativeCache();
 128     bool cacheNegatively();     // TODO: why both negativeCache() and cacheNegatively() ?
 129     void invokeHandlers();
 130     void cacheInMemory(); ///< start or continue storing in memory cache
 131     void swapOut();
 132     /// whether we are in the process of writing this entry to disk
 133     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 134     /// whether the entire entry is now on disk (possibly marked for deletion)
 135     bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
 136     /// whether we failed to write this entry to disk
 137     bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
 138     void swapOutFileClose(int how);
 139     const char *url() const;
 140     /// Satisfies cachability requirements shared among disk and RAM caches.
 141     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 142     /// TODO: Rename and make private so only those two methods can call this.
 143     bool checkCachable();
 144     int checkNegativeHit() const;
 145     int locked() const { return lock_count; }
 146     int validToSend() const;
 147     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 148
 149     /// initialize mem_obj; assert if mem_obj already exists
 150     /// avoid this method in favor of createMemObject(trio)!
 151     void createMemObject();
 152
 153     /// initialize mem_obj with URIs/method; assert if mem_obj already exists
 154     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 155
 156     /// initialize mem_obj (if needed) and set URIs/method (if missing)
 157     void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 158
 159     void dump(int debug_lvl) const;
 160     void hashDelete();
 161     void hashInsert(const cache_key *);
 162     /// notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
 163     void registerAbortCallback(const AsyncCall::Pointer &);
 164     void reset();
 165     void setMemStatus(mem_status_t);
 166     bool timestampsSet();
 167     /// Avoid notifying anybody about a 3rd-party initiated StoreEntry abort.
 168     /// Calling this method does not cancel the already queued notification.
 169     /// TODO: Refactor to represent the end of (shared) ownership by our writer.
 170     void unregisterAbortCallback(const char *reason);
 171     void destroyMemObject();
 172     int checkTooSmall();
 173
 174     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 175
 176     void setNoDelay (bool const);
 177     void lastModified(const time_t when) { lastModified_ = when; }
 178     /// \returns entry's 'effective' modification time
 179     time_t lastModified() const {
 180         // may still return -1 if timestamp is not set
 181         return lastModified_ < 0 ? timestamp : lastModified_;
 182     }
 183     /// \returns a formatted string with entry's timestamps
 184     const char *describeTimestamps() const;
 185     // TODO: consider removing currently unsupported imslen parameter
 186     bool modifiedSince(const time_t ims, const int imslen = -1) const;
 187     /// has ETag matching at least one of the If-Match etags
 188     bool hasIfMatchEtag(const HttpRequest &request) const;
 189     /// has ETag matching at least one of the If-None-Match etags
 190     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 191     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 192     bool hasEtag(ETag &etag) const;
 193
 194     /// Updates easily-accessible non-Store-specific parts of the entry.
 195     /// Use Controller::updateOnNotModified() instead of this helper.
 196     /// \returns whether anything was actually updated
 197     bool updateOnNotModified(const StoreEntry &e304);
 198
 199     /// the disk this entry is [being] cached on; asserts for entries w/o a disk
 200     Store::Disk &disk() const;
 201     /// whether one of this StoreEntry owners has locked the corresponding
 202     /// disk entry (at the specified disk entry coordinates, if any)
 203     bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
 204     /// Makes hasDisk(dirn, filn) true. The caller should have locked
 205     /// the corresponding disk store entry for reading or writing.
 206     void attachToDisk(const sdirno, const sfileno, const swap_status_t);
 207     /// Makes hasDisk() false. The caller should have unlocked
 208     /// the corresponding disk store entry.
 209     void detachFromDisk();
 210
 211     /// whether there is a corresponding locked transients table entry
 212     bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
 213     /// whether there is a corresponding locked shared memory table entry
 214     bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
 215
 216     /// whether this entry can feed collapsed requests and only them
 217     bool hittingRequiresCollapsing() const { return EBIT_TEST(flags, ENTRY_REQUIRES_COLLAPSING); }
 218
 219     /// allow or forbid collapsed requests feeding
 220     void setCollapsingRequirement(const bool required);
 221
 222     MemObject *mem_obj;
 223     RemovalPolicyNode repl;
 224     /* START OF ON-DISK STORE_META_STD TLV field */
 225     time_t timestamp;
 226     time_t lastref;
 227     time_t expires;
 228 private:
 229     time_t lastModified_; ///< received Last-Modified value or -1; use lastModified()
 230 public:
 231     uint64_t swap_file_sz;
 232     uint16_t refcount;
 233     uint16_t flags;
 234     /* END OF ON-DISK STORE_META_STD */
 235
 236     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 237     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 238
 239     sdirno swap_dirn:7;
 240
 241     mem_status_t mem_status:3;
 242
 243     ping_status_t ping_status:3;
 244
 245     store_status_t store_status:3;
 246
 247     swap_status_t swap_status:3;
 248
 249 public:
 250     static size_t inUseCount();
 251
 252     void *operator new(size_t byteCount);
 253     void operator delete(void *address);
 254 #if USE_SQUID_ESI
 255
 256     ESIElement::Pointer cachedESITree;
 257 #endif
 258     int64_t objectLen() const { return mem().object_sz; }
 259     int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
 260
 261     /// claim shared ownership of this entry (for use in a given context)
 262     /// matching lock() and unlock() contexts eases leak triage but is optional
 263     void lock(const char *context);
 264
 265     /// disclaim shared ownership; may remove entry from store and delete it
 266     /// returns remaining lock level (zero for unlocked and possibly gone entry)
 267     int unlock(const char *context);
 268
 269     /// returns a local concurrent use counter, for debugging
 270     int locks() const { return static_cast<int>(lock_count); }
 271
 272     /// update last reference timestamp and related Store metadata
 273     void touch();
 274
 275     /// One of the three methods to get rid of an unlocked StoreEntry object.
 276     /// Removes all unlocked (and marks for eventual removal all locked) Store
 277     /// entries, including attached and unattached entries that have our key.
 278     /// Also destroys us if we are unlocked or makes us private otherwise.
 279     void release(const bool shareable = false);
 280
 281     /// One of the three methods to get rid of an unlocked StoreEntry object.
 282     /// May destroy this object if it is unlocked; does nothing otherwise.
 283     /// Unlike release(), may not trigger eviction of underlying store entries,
 284     /// but, unlike destroyStoreEntry(), does honor an earlier release request.
 285     void abandon(const char *context) { if (!locked()) doAbandon(context); }
 286
 287     /// May the caller commit to treating this [previously locked]
 288     /// entry as a cache hit?
 289     bool mayStartHitting() const {
 290         return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
 291     }
 292
 293 #if USE_ADAPTATION
 294     /// call back producer when more buffer space is available
 295     void deferProducer(const AsyncCall::Pointer &producer);
 296     /// calls back producer registered with deferProducer
 297     void kickProducer();
 298 #endif
 299
 300     /* Packable API */
 301     virtual void append(char const *, int);
 302     virtual void vappendf(const char *, va_list);
 303     virtual void buffer();
 304     virtual void flush();
 305
 306 protected:
 307     typedef Store::EntryGuard EntryGuard;
 308
 309     void transientsAbandonmentCheck();
 310     /// does nothing except throwing if disk-associated data members are inconsistent
 311     void checkDisk() const;
 312
 313 private:
 314     void doAbandon(const char *context);
 315     bool checkTooBig() const;
 316     void forcePublicKey(const cache_key *newkey);
 317     StoreEntry *adjustVary();
 318     const cache_key *calcPublicKey(const KeyScope keyScope);
 319
 320     /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
 321     void lengthWentBad(const char *reason);
 322
 323     static MemAllocator *pool;
 324
 325     unsigned short lock_count;      /* Assume < 65536! */
 326
 327     /// Nobody can find/lock KEY_PRIVATE entries, but some transactions
 328     /// (e.g., collapsed requests) find/lock a public entry before it becomes
 329     /// private. May such transactions start using the now-private entry
 330     /// they previously locked? This member should not affect transactions
 331     /// that already started reading from the entry.
 332     bool shareableWhenPrivate;
 333
 334 #if USE_ADAPTATION
 335     /// producer callback registered with deferProducer
 336     AsyncCall::Pointer deferredProducer;
 337 #endif
 338
 339     bool validLength() const;
 340     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 341
 342     friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 343 };
 344
 345 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 346
 347 /// \ingroup StoreAPI
 348 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 349
 350 namespace Store {
 351
 352 /// a smart pointer similar to std::unique_ptr<> that automatically
 353 /// release()s and unlock()s the guarded Entry on stack-unwinding failures
 354 class EntryGuard {
 355 public:
 356     /// \param entry either nil or a locked Entry to manage
 357     /// \param context default unlock() message
 358     EntryGuard(Entry *entry, const char *context):
 359         entry_(entry), context_(context) {
 360         assert(!entry_ || entry_->locked());
 361     }
 362
 363     ~EntryGuard() {
 364         if (entry_) {
 365             // something went wrong -- the caller did not unlockAndReset() us
 366             onException();
 367         }
 368     }
 369
 370     EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
 371
 372     /// like std::unique_ptr::get()
 373     /// \returns nil or the guarded (locked) entry
 374     Entry *get() {
 375         return entry_;
 376     }
 377
 378     /// like std::unique_ptr::reset()
 379     /// stops guarding the entry
 380     /// unlocks the entry (which may destroy it)
 381     void unlockAndReset(const char *resetContext = nullptr) {
 382         if (entry_) {
 383             entry_->unlock(resetContext ? resetContext : context_);
 384             entry_ = nullptr;
 385         }
 386     }
 387
 388 private:
 389     void onException() noexcept;
 390
 391     Entry *entry_; ///< the guarded Entry or nil
 392     const char *context_; ///< default unlock() message
 393 };
 394
 395 void Stats(StoreEntry *output);
 396 void Maintain(void *unused);
 397 }; // namespace Store
 398
 399 /// \ingroup StoreAPI
 400 size_t storeEntryInUse();
 401
 402 /// \ingroup StoreAPI
 403 const char *storeEntryFlags(const StoreEntry *);
 404
 405 /// \ingroup StoreAPI
 406 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 407
 408 /// \ingroup StoreAPI
 409 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 410
 411 /// \ingroup StoreAPI
 412 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
 413
 414 /// \ingroup StoreAPI
 415 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
 416
 417 /// \ingroup StoreAPI
 418 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 419 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 420
 421 /// \ingroup StoreAPI
 422 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 423 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
 424
 425 /// \ingroup StoreAPI
 426 void storeInit(void);
 427
 428 /// \ingroup StoreAPI
 429 void storeConfigure(void);
 430
 431 /// \ingroup StoreAPI
 432 void storeFreeMemory(void);
 433
 434 /// \ingroup StoreAPI
 435 int expiresMoreThan(time_t, time_t);
 436
 437 /// \ingroup StoreAPI
 438 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 439
 440 /// \ingroup StoreAPI
 441 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 442
 443 /// \ingroup StoreAPI
 444 int storeTooManyDiskFilesOpen(void);
 445
 446 /// \ingroup StoreAPI
 447 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 448
 449 /// \ingroup StoreAPI
 450 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 451
 452 /// \ingroup StoreAPI
 453 void storeFsInit(void);
 454
 455 /// \ingroup StoreAPI
 456 void storeFsDone(void);
 457
 458 /// \ingroup StoreAPI
 459 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 460
 461 /// One of the three methods to get rid of an unlocked StoreEntry object.
 462 /// This low-level method ignores lock()ing and release() promises. It never
 463 /// leaves the entry in the local store_table.
 464 /// TODO: Hide by moving its functionality into the StoreEntry destructor.
 465 extern FREE destroyStoreEntry;
 466
 467 /// \ingroup StoreAPI
 468 void storeGetMemSpace(int size);
 469
 470 #endif /* SQUID_STORE_H */
 471