src/Store.h

   1 /*
   2  * Copyright (C) 1996-2025 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_SRC_STORE_H
  10 #define SQUID_SRC_STORE_H
  11
  12 #include "base/DelayedAsyncCalls.h"
  13 #include "base/Packable.h"
  14 #include "base/Range.h"
  15 #include "base/RefCount.h"
  16 #include "comm/forward.h"
  17 #include "hash.h"
  18 #include "http/forward.h"
  19 #include "http/RequestMethod.h"
  20 #include "HttpReply.h"
  21 #include "MemObject.h"
  22 #include "RemovalPolicy.h"
  23 #include "store/Controller.h"
  24 #include "store/forward.h"
  25 #include "store_key_md5.h"
  26 #include "StoreIOBuffer.h"
  27 #include "StoreStats.h"
  28
  29 #include <ostream>
  30
  31 class AsyncCall;
  32 class HttpRequest;
  33 class RequestFlags;
  34
  35 extern StoreIoStats store_io_stats;
  36
  37 class StoreEntry : public hash_link, public Packable
  38 {
  39
  40 public:
  41     bool checkDeferRead(int fd) const;
  42
  43     const char *getMD5Text() const;
  44     StoreEntry();
  45     ~StoreEntry() override;
  46
  47     MemObject &mem() { assert(mem_obj); return *mem_obj; }
  48     const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
  49
  50     /// \retval * the address of freshest reply (if mem_obj exists)
  51     /// \retval nullptr when mem_obj does not exist
  52     /// \see MemObject::freshestReply()
  53     const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
  54
  55     /// whether this entry has access to [deserialized] [HTTP] response headers
  56     bool hasParsedReplyHeader() const;
  57
  58     void write(StoreIOBuffer);
  59
  60     /** Check if the Store entry is empty
  61      * \retval true   Store contains 0 bytes of data.
  62      * \retval false  Store contains 1 or more bytes of data.
  63      * \retval false  Store contains negative content !!!!!!
  64      */
  65     bool isEmpty() const { return mem().endOffset() == 0; }
  66     bool isAccepting() const;
  67     size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  68
  69     /// Signals that the entire response has been stored and no more append()
  70     /// calls should be expected; cf. completeTruncated().
  71     void completeSuccessfully(const char *whyWeAreSureWeStoredTheWholeReply);
  72
  73     /// Signals that a partial response (if any) has been stored but no more
  74     /// append() calls should be expected; cf. completeSuccessfully().
  75     void completeTruncated(const char *whyWeConsiderTheReplyTruncated);
  76
  77     /// \deprecated use either completeSuccessfully() or completeTruncated() instead
  78     void complete();
  79
  80     store_client_t storeClientType() const;
  81     /// \returns a malloc()ed buffer containing a length-long packed swap header
  82     const char *getSerialisedMetaData(size_t &length) const;
  83     /// Store a prepared error response. MemObject locks the reply object.
  84     void storeErrorResponse(HttpReply *reply);
  85     void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
  86     void startWriting(); ///< pack and write reply headers and, maybe, body
  87     /// whether we may start writing to disk (now or in the future)
  88     bool mayStartSwapOut();
  89     void trimMemory(const bool preserveSwappable);
  90
  91     // called when a decision to cache in memory has been made
  92     void memOutDecision(const bool willCacheInRam);
  93     // called when a decision to cache on disk has been made
  94     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  95     /// called when a store writer ends its work (successfully or not)
  96     void storeWriterDone();
  97
  98     void abort();
  99     bool makePublic(const KeyScope keyScope = ksDefault);
 100     void makePrivate(const bool shareable);
 101     /// A low-level method just resetting "private key" flags.
 102     /// To avoid key inconsistency please use forcePublicKey()
 103     /// or similar instead.
 104     void clearPrivate();
 105     bool setPublicKey(const KeyScope keyScope = ksDefault);
 106     /// Resets existing public key to a public key with default scope,
 107     /// releasing the old default-scope entry (if any).
 108     /// Does nothing if the existing public key already has default scope.
 109     void clearPublicKeyScope();
 110
 111     /// \returns public key (if the entry has it) or nil (otherwise)
 112     const cache_key *publicKey() const {
 113         return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
 114                reinterpret_cast<const cache_key*>(key): // may be nil
 115                nullptr;
 116     }
 117
 118     /// Either fills this entry with private key or changes the existing key
 119     /// from public to private.
 120     /// \param permanent whether this entry should be private forever.
 121     void setPrivateKey(const bool shareable, const bool permanent);
 122
 123     void expireNow();
 124     /// Makes the StoreEntry private and marks the corresponding entry
 125     /// for eventual removal from the Store.
 126     void releaseRequest(const bool shareable = false);
 127     void negativeCache();
 128     bool cacheNegatively();     // TODO: why both negativeCache() and cacheNegatively() ?
 129     void invokeHandlers();
 130     void cacheInMemory(); ///< start or continue storing in memory cache
 131     void swapOut();
 132     /// whether we are in the process of writing this entry to disk
 133     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 134     /// whether the entire entry is now on disk (possibly marked for deletion)
 135     bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
 136     /// whether we failed to write this entry to disk
 137     bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
 138     void swapOutFileClose(int how);
 139     const char *url() const;
 140     /// Satisfies cachability requirements shared among disk and RAM caches.
 141     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 142     /// TODO: Rename and make private so only those two methods can call this.
 143     bool checkCachable();
 144     int checkNegativeHit() const;
 145     int locked() const { return lock_count; }
 146     int validToSend() const;
 147     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 148
 149     /// initialize mem_obj; assert if mem_obj already exists
 150     /// avoid this method in favor of createMemObject(trio)!
 151     void createMemObject();
 152
 153     /// initialize mem_obj with URIs/method; assert if mem_obj already exists
 154     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 155
 156     /// initialize mem_obj (if needed) and set URIs/method (if missing)
 157     void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 158
 159     void dump(int debug_lvl) const;
 160     void hashDelete();
 161     void hashInsert(const cache_key *);
 162     /// notify the StoreEntry writer of a 3rd-party initiated StoreEntry abort
 163     void registerAbortCallback(const AsyncCall::Pointer &);
 164     void reset();
 165     void setMemStatus(mem_status_t);
 166     bool timestampsSet();
 167     /// Avoid notifying anybody about a 3rd-party initiated StoreEntry abort.
 168     /// Calling this method does not cancel the already queued notification.
 169     /// TODO: Refactor to represent the end of (shared) ownership by our writer.
 170     void unregisterAbortCallback(const char *reason);
 171     void destroyMemObject();
 172     int checkTooSmall();
 173
 174     void setNoDelay (bool const);
 175     void lastModified(const time_t when) { lastModified_ = when; }
 176     /// \returns entry's 'effective' modification time
 177     time_t lastModified() const {
 178         // may still return -1 if timestamp is not set
 179         return lastModified_ < 0 ? timestamp : lastModified_;
 180     }
 181     /// \returns a formatted string with entry's timestamps
 182     const char *describeTimestamps() const;
 183     // TODO: consider removing currently unsupported imslen parameter
 184     bool modifiedSince(const time_t ims, const int imslen = -1) const;
 185     /// has ETag matching at least one of the If-Match etags
 186     bool hasIfMatchEtag(const HttpRequest &request) const;
 187     /// has ETag matching at least one of the If-None-Match etags
 188     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 189     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 190     bool hasEtag(ETag &etag) const;
 191
 192     /// Updates easily-accessible non-Store-specific parts of the entry.
 193     /// Use Controller::updateOnNotModified() instead of this helper.
 194     /// \returns whether anything was actually updated
 195     bool updateOnNotModified(const StoreEntry &e304);
 196
 197     /// the disk this entry is [being] cached on; asserts for entries w/o a disk
 198     Store::Disk &disk() const;
 199     /// whether one of this StoreEntry owners has locked the corresponding
 200     /// disk entry (at the specified disk entry coordinates, if any)
 201     bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
 202     /// Makes hasDisk(dirn, filn) true. The caller should have locked
 203     /// the corresponding disk store entry for reading or writing.
 204     void attachToDisk(const sdirno, const sfileno, const swap_status_t);
 205     /// Makes hasDisk() false. The caller should have unlocked
 206     /// the corresponding disk store entry.
 207     void detachFromDisk();
 208
 209     /// whether there is a corresponding locked transients table entry
 210     bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
 211     /// whether there is a corresponding locked shared memory table entry
 212     bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
 213
 214     /// whether this entry can feed collapsed requests and only them
 215     bool hittingRequiresCollapsing() const { return EBIT_TEST(flags, ENTRY_REQUIRES_COLLAPSING); }
 216
 217     /// allow or forbid collapsed requests feeding
 218     void setCollapsingRequirement(const bool required);
 219
 220     MemObject *mem_obj;
 221     RemovalPolicyNode repl;
 222     /* START OF ON-DISK STORE_META_STD TLV field */
 223     time_t timestamp;
 224     time_t lastref;
 225     time_t expires;
 226 private:
 227     time_t lastModified_; ///< received Last-Modified value or -1; use lastModified()
 228 public:
 229     uint64_t swap_file_sz;
 230     uint16_t refcount;
 231     uint16_t flags;
 232     /* END OF ON-DISK STORE_META_STD */
 233
 234     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 235     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 236
 237     sdirno swap_dirn:7;
 238
 239     mem_status_t mem_status:3;
 240
 241     ping_status_t ping_status:3;
 242
 243     store_status_t store_status:3;
 244
 245     swap_status_t swap_status:3;
 246
 247 public:
 248     static size_t inUseCount();
 249
 250     void *operator new(size_t byteCount);
 251     void operator delete(void *address);
 252
 253     int64_t objectLen() const { return mem().object_sz; }
 254     int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
 255
 256     /// claim shared ownership of this entry (for use in a given context)
 257     /// matching lock() and unlock() contexts eases leak triage but is optional
 258     void lock(const char *context);
 259
 260     /// disclaim shared ownership; may remove entry from store and delete it
 261     /// returns remaining lock level (zero for unlocked and possibly gone entry)
 262     int unlock(const char *context);
 263
 264     /// returns a local concurrent use counter, for debugging
 265     int locks() const { return static_cast<int>(lock_count); }
 266
 267     /// update last reference timestamp and related Store metadata
 268     void touch();
 269
 270     /// One of the three methods to get rid of an unlocked StoreEntry object.
 271     /// Removes all unlocked (and marks for eventual removal all locked) Store
 272     /// entries, including attached and unattached entries that have our key.
 273     /// Also destroys us if we are unlocked or makes us private otherwise.
 274     void release(const bool shareable = false);
 275
 276     /// One of the three methods to get rid of an unlocked StoreEntry object.
 277     /// May destroy this object if it is unlocked; does nothing otherwise.
 278     /// Unlike release(), may not trigger eviction of underlying store entries,
 279     /// but, unlike destroyStoreEntry(), does honor an earlier release request.
 280     void abandon(const char *context) { if (!locked()) doAbandon(context); }
 281
 282     /// May the caller commit to treating this [previously locked]
 283     /// entry as a cache hit?
 284     bool mayStartHitting() const {
 285         return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
 286     }
 287
 288 #if USE_ADAPTATION
 289     /// call back producer when more buffer space is available
 290     void deferProducer(const AsyncCall::Pointer &producer);
 291     /// calls back producer registered with deferProducer
 292     void kickProducer();
 293 #endif
 294
 295     /* Packable API */
 296     void append(char const *, int) override;
 297     void vappendf(const char *, va_list) override;
 298     void buffer() override;
 299     void flush() override;
 300
 301 protected:
 302     typedef Store::EntryGuard EntryGuard;
 303
 304     void storeWritingCheckpoint();
 305     /// does nothing except throwing if disk-associated data members are inconsistent
 306     void checkDisk() const;
 307
 308 private:
 309     void doAbandon(const char *context);
 310     bool checkTooBig() const;
 311     void forcePublicKey(const cache_key *newkey);
 312     StoreEntry *adjustVary();
 313     const cache_key *calcPublicKey(const KeyScope keyScope);
 314
 315     /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
 316     void lengthWentBad(const char *reason);
 317
 318     static Mem::Allocator *pool;
 319
 320     unsigned short lock_count;      /* Assume < 65536! */
 321
 322     /// Nobody can find/lock KEY_PRIVATE entries, but some transactions
 323     /// (e.g., collapsed requests) find/lock a public entry before it becomes
 324     /// private. May such transactions start using the now-private entry
 325     /// they previously locked? This member should not affect transactions
 326     /// that already started reading from the entry.
 327     bool shareableWhenPrivate;
 328
 329 #if USE_ADAPTATION
 330     /// producer callback registered with deferProducer
 331     AsyncCall::Pointer deferredProducer;
 332 #endif
 333
 334     bool validLength() const;
 335     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 336
 337     friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 338 };
 339
 340 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 341
 342 /// \ingroup StoreAPI
 343 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 344
 345 namespace Store {
 346
 347 /// a smart pointer similar to std::unique_ptr<> that automatically
 348 /// release()s and unlock()s the guarded Entry on stack-unwinding failures
 349 class EntryGuard {
 350 public:
 351     /// \param entry either nil or a locked Entry to manage
 352     /// \param context default unlock() message
 353     EntryGuard(Entry *entry, const char *context):
 354         entry_(entry), context_(context) {
 355         assert(!entry_ || entry_->locked());
 356     }
 357
 358     ~EntryGuard() {
 359         if (entry_) {
 360             // something went wrong -- the caller did not unlockAndReset() us
 361             onException();
 362         }
 363     }
 364
 365     EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
 366
 367     /// like std::unique_ptr::get()
 368     /// \returns nil or the guarded (locked) entry
 369     Entry *get() {
 370         return entry_;
 371     }
 372
 373     /// like std::unique_ptr::reset()
 374     /// stops guarding the entry
 375     /// unlocks the entry (which may destroy it)
 376     void unlockAndReset(const char *resetContext = nullptr) {
 377         if (entry_) {
 378             entry_->unlock(resetContext ? resetContext : context_);
 379             entry_ = nullptr;
 380         }
 381     }
 382
 383 private:
 384     void onException() noexcept;
 385
 386     Entry *entry_; ///< the guarded Entry or nil
 387     const char *context_; ///< default unlock() message
 388 };
 389
 390 void Stats(StoreEntry *output);
 391 void Maintain(void *unused);
 392 }; // namespace Store
 393
 394 /// \ingroup StoreAPI
 395 size_t storeEntryInUse();
 396
 397 /// \ingroup StoreAPI
 398 const char *storeEntryFlags(const StoreEntry *);
 399
 400 /// \ingroup StoreAPI
 401 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 402
 403 /// \ingroup StoreAPI
 404 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 405
 406 /// \ingroup StoreAPI
 407 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
 408
 409 /// \ingroup StoreAPI
 410 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
 411
 412 /// \ingroup StoreAPI
 413 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 414 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 415
 416 /// \ingroup StoreAPI
 417 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 418 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
 419
 420 /// \ingroup StoreAPI
 421 void storeInit(void);
 422
 423 /// \ingroup StoreAPI
 424 void storeConfigure(void);
 425
 426 /// \ingroup StoreAPI
 427 int expiresMoreThan(time_t, time_t);
 428
 429 /// \ingroup StoreAPI
 430 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 431
 432 /// \ingroup StoreAPI
 433 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 434
 435 /// \ingroup StoreAPI
 436 int storeTooManyDiskFilesOpen(void);
 437
 438 /// \ingroup StoreAPI
 439 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 440
 441 /// \ingroup StoreAPI
 442 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 443
 444 /// \ingroup StoreAPI
 445 void storeFsInit(void);
 446
 447 /// \ingroup StoreAPI
 448 void storeFsDone(void);
 449
 450 /// \ingroup StoreAPI
 451 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 452
 453 /// One of the three methods to get rid of an unlocked StoreEntry object.
 454 /// This low-level method ignores lock()ing and release() promises. It never
 455 /// leaves the entry in the local store_table.
 456 /// TODO: Hide by moving its functionality into the StoreEntry destructor.
 457 extern FREE destroyStoreEntry;
 458
 459 /// \ingroup StoreAPI
 460 void storeGetMemSpace(int size);
 461
 462 #endif /* SQUID_SRC_STORE_H */
 463