src/Store.h

   1 /*
   2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 #include "base/Packable.h"
  13 #include "base/Range.h"
  14 #include "base/RefCount.h"
  15 #include "comm/forward.h"
  16 #include "CommRead.h"
  17 #include "hash.h"
  18 #include "http/forward.h"
  19 #include "http/RequestMethod.h"
  20 #include "HttpReply.h"
  21 #include "MemObject.h"
  22 #include "RemovalPolicy.h"
  23 #include "store/Controller.h"
  24 #include "store/forward.h"
  25 #include "store_key_md5.h"
  26 #include "StoreIOBuffer.h"
  27 #include "StoreStats.h"
  28
  29 #if USE_SQUID_ESI
  30 #include "esi/Element.h"
  31 #endif
  32
  33 #include <ostream>
  34
  35 class AsyncCall;
  36 class HttpRequest;
  37 class RequestFlags;
  38
  39 extern StoreIoStats store_io_stats;
  40
  41 class StoreEntry : public hash_link, public Packable
  42 {
  43
  44 public:
  45     static DeferredRead::DeferrableRead DeferReader;
  46     bool checkDeferRead(int fd) const;
  47
  48     const char *getMD5Text() const;
  49     StoreEntry();
  50     virtual ~StoreEntry();
  51
  52     MemObject &mem() { assert(mem_obj); return *mem_obj; }
  53     const MemObject &mem() const { assert(mem_obj); return *mem_obj; }
  54
  55     /// \retval * the address of freshest reply (if mem_obj exists)
  56     /// \retval nullptr when mem_obj does not exist
  57     /// \see MemObject::freshestReply()
  58     const HttpReply *hasFreshestReply() const { return mem_obj ? &mem_obj->freshestReply() : nullptr; }
  59
  60     void write(StoreIOBuffer);
  61
  62     /** Check if the Store entry is empty
  63      * \retval true   Store contains 0 bytes of data.
  64      * \retval false  Store contains 1 or more bytes of data.
  65      * \retval false  Store contains negative content !!!!!!
  66      */
  67     bool isEmpty() const { return mem().endOffset() == 0; }
  68     bool isAccepting() const;
  69     size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  70     /// flags [truncated or too big] entry with ENTRY_BAD_LENGTH and releases it
  71     void lengthWentBad(const char *reason);
  72     void complete();
  73     store_client_t storeClientType() const;
  74     /// \returns a malloc()ed buffer containing a length-long packed swap header
  75     const char *getSerialisedMetaData(size_t &length) const;
  76     /// Store a prepared error response. MemObject locks the reply object.
  77     void storeErrorResponse(HttpReply *reply);
  78     void replaceHttpReply(const HttpReplyPointer &, const bool andStartWriting = true);
  79     void startWriting(); ///< pack and write reply headers and, maybe, body
  80     /// whether we may start writing to disk (now or in the future)
  81     bool mayStartSwapOut();
  82     void trimMemory(const bool preserveSwappable);
  83
  84     // called when a decision to cache in memory has been made
  85     void memOutDecision(const bool willCacheInRam);
  86     // called when a decision to cache on disk has been made
  87     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  88
  89     void abort();
  90     bool makePublic(const KeyScope keyScope = ksDefault);
  91     void makePrivate(const bool shareable);
  92     /// A low-level method just resetting "private key" flags.
  93     /// To avoid key inconsistency please use forcePublicKey()
  94     /// or similar instead.
  95     void clearPrivate();
  96     bool setPublicKey(const KeyScope keyScope = ksDefault);
  97     /// Resets existing public key to a public key with default scope,
  98     /// releasing the old default-scope entry (if any).
  99     /// Does nothing if the existing public key already has default scope.
 100     void clearPublicKeyScope();
 101
 102     /// \returns public key (if the entry has it) or nil (otherwise)
 103     const cache_key *publicKey() const {
 104         return (!EBIT_TEST(flags, KEY_PRIVATE)) ?
 105                reinterpret_cast<const cache_key*>(key): // may be nil
 106                nullptr;
 107     }
 108
 109     /// Either fills this entry with private key or changes the existing key
 110     /// from public to private.
 111     /// \param permanent whether this entry should be private forever.
 112     void setPrivateKey(const bool shareable, const bool permanent);
 113
 114     void expireNow();
 115     /// Makes the StoreEntry private and marks the corresponding entry
 116     /// for eventual removal from the Store.
 117     void releaseRequest(const bool shareable = false);
 118     void negativeCache();
 119     bool cacheNegatively();     /** \todo argh, why both? */
 120     void invokeHandlers();
 121     void cacheInMemory(); ///< start or continue storing in memory cache
 122     void swapOut();
 123     /// whether we are in the process of writing this entry to disk
 124     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 125     /// whether the entire entry is now on disk (possibly marked for deletion)
 126     bool swappedOut() const { return swap_status == SWAPOUT_DONE; }
 127     /// whether we failed to write this entry to disk
 128     bool swapoutFailed() const { return swap_status == SWAPOUT_FAILED; }
 129     void swapOutFileClose(int how);
 130     const char *url() const;
 131     /// Satisfies cachability requirements shared among disk and RAM caches.
 132     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 133     /// TODO: Rename and make private so only those two methods can call this.
 134     bool checkCachable();
 135     int checkNegativeHit() const;
 136     int locked() const { return lock_count; }
 137     int validToSend() const;
 138     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 139
 140     /// initialize mem_obj; assert if mem_obj already exists
 141     /// avoid this method in favor of createMemObject(trio)!
 142     void createMemObject();
 143
 144     /// initialize mem_obj with URIs/method; assert if mem_obj already exists
 145     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 146
 147     /// initialize mem_obj (if needed) and set URIs/method (if missing)
 148     void ensureMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 149
 150     void dump(int debug_lvl) const;
 151     void hashDelete();
 152     void hashInsert(const cache_key *);
 153     void registerAbort(STABH * cb, void *);
 154     void reset();
 155     void setMemStatus(mem_status_t);
 156     bool timestampsSet();
 157     void unregisterAbort();
 158     void destroyMemObject();
 159     int checkTooSmall();
 160
 161     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 162
 163     void setNoDelay (bool const);
 164     void lastModified(const time_t when) { lastModified_ = when; }
 165     /// \returns entry's 'effective' modification time
 166     time_t lastModified() const {
 167         // may still return -1 if timestamp is not set
 168         return lastModified_ < 0 ? timestamp : lastModified_;
 169     }
 170     /// \returns a formatted string with entry's timestamps
 171     const char *describeTimestamps() const;
 172     // TODO: consider removing currently unsupported imslen parameter
 173     bool modifiedSince(const time_t ims, const int imslen = -1) const;
 174     /// has ETag matching at least one of the If-Match etags
 175     bool hasIfMatchEtag(const HttpRequest &request) const;
 176     /// has ETag matching at least one of the If-None-Match etags
 177     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 178     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 179     bool hasEtag(ETag &etag) const;
 180
 181     /// Updates easily-accessible non-Store-specific parts of the entry.
 182     /// Use Controller::updateOnNotModified() instead of this helper.
 183     /// \returns whether anything was actually updated
 184     bool updateOnNotModified(const StoreEntry &e304);
 185
 186     /// the disk this entry is [being] cached on; asserts for entries w/o a disk
 187     Store::Disk &disk() const;
 188     /// whether one of this StoreEntry owners has locked the corresponding
 189     /// disk entry (at the specified disk entry coordinates, if any)
 190     bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const;
 191     /// Makes hasDisk(dirn, filn) true. The caller should have locked
 192     /// the corresponding disk store entry for reading or writing.
 193     void attachToDisk(const sdirno, const sfileno, const swap_status_t);
 194     /// Makes hasDisk() false. The caller should have unlocked
 195     /// the corresponding disk store entry.
 196     void detachFromDisk();
 197
 198     /// whether there is a corresponding locked transients table entry
 199     bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; }
 200     /// whether there is a corresponding locked shared memory table entry
 201     bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; }
 202
 203     /// whether this entry can feed collapsed requests and only them
 204     bool hittingRequiresCollapsing() const { return EBIT_TEST(flags, ENTRY_REQUIRES_COLLAPSING); }
 205
 206     /// allow or forbid collapsed requests feeding
 207     void setCollapsingRequirement(const bool required);
 208
 209     MemObject *mem_obj;
 210     RemovalPolicyNode repl;
 211     /* START OF ON-DISK STORE_META_STD TLV field */
 212     time_t timestamp;
 213     time_t lastref;
 214     time_t expires;
 215 private:
 216     time_t lastModified_; ///< received Last-Modified value or -1; use lastModified()
 217 public:
 218     uint64_t swap_file_sz;
 219     uint16_t refcount;
 220     uint16_t flags;
 221     /* END OF ON-DISK STORE_META_STD */
 222
 223     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 224     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 225
 226     sdirno swap_dirn:7;
 227
 228     mem_status_t mem_status:3;
 229
 230     ping_status_t ping_status:3;
 231
 232     store_status_t store_status:3;
 233
 234     swap_status_t swap_status:3;
 235
 236 public:
 237     static size_t inUseCount();
 238     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 239     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 240     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 241
 242     void *operator new(size_t byteCount);
 243     void operator delete(void *address);
 244 #if USE_SQUID_ESI
 245
 246     ESIElement::Pointer cachedESITree;
 247 #endif
 248     int64_t objectLen() const { return mem().object_sz; }
 249     int64_t contentLen() const { return objectLen() - mem().baseReply().hdr_sz; }
 250
 251     /// claim shared ownership of this entry (for use in a given context)
 252     /// matching lock() and unlock() contexts eases leak triage but is optional
 253     void lock(const char *context);
 254
 255     /// disclaim shared ownership; may remove entry from store and delete it
 256     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 257     int unlock(const char *context);
 258
 259     /// returns a local concurrent use counter, for debugging
 260     int locks() const { return static_cast<int>(lock_count); }
 261
 262     /// update last reference timestamp and related Store metadata
 263     void touch();
 264
 265     /// One of the three methods to get rid of an unlocked StoreEntry object.
 266     /// Removes all unlocked (and marks for eventual removal all locked) Store
 267     /// entries, including attached and unattached entries that have our key.
 268     /// Also destroys us if we are unlocked or makes us private otherwise.
 269     void release(const bool shareable = false);
 270
 271     /// One of the three methods to get rid of an unlocked StoreEntry object.
 272     /// May destroy this object if it is unlocked; does nothing otherwise.
 273     /// Unlike release(), may not trigger eviction of underlying store entries,
 274     /// but, unlike destroyStoreEntry(), does honor an earlier release request.
 275     void abandon(const char *context) { if (!locked()) doAbandon(context); }
 276
 277     /// May the caller commit to treating this [previously locked]
 278     /// entry as a cache hit?
 279     bool mayStartHitting() const {
 280         return !EBIT_TEST(flags, KEY_PRIVATE) || shareableWhenPrivate;
 281     }
 282
 283 #if USE_ADAPTATION
 284     /// call back producer when more buffer space is available
 285     void deferProducer(const AsyncCall::Pointer &producer);
 286     /// calls back producer registered with deferProducer
 287     void kickProducer();
 288 #endif
 289
 290     /* Packable API */
 291     virtual void append(char const *, int);
 292     virtual void vappendf(const char *, va_list);
 293     virtual void buffer();
 294     virtual void flush();
 295
 296 protected:
 297     typedef Store::EntryGuard EntryGuard;
 298
 299     void transientsAbandonmentCheck();
 300     /// does nothing except throwing if disk-associated data members are inconsistent
 301     void checkDisk() const;
 302
 303 private:
 304     void doAbandon(const char *context);
 305     bool checkTooBig() const;
 306     void forcePublicKey(const cache_key *newkey);
 307     StoreEntry *adjustVary();
 308     const cache_key *calcPublicKey(const KeyScope keyScope);
 309
 310     static MemAllocator *pool;
 311
 312     unsigned short lock_count;      /* Assume < 65536! */
 313
 314     /// Nobody can find/lock KEY_PRIVATE entries, but some transactions
 315     /// (e.g., collapsed requests) find/lock a public entry before it becomes
 316     /// private. May such transactions start using the now-private entry
 317     /// they previously locked? This member should not affect transactions
 318     /// that already started reading from the entry.
 319     bool shareableWhenPrivate;
 320
 321 #if USE_ADAPTATION
 322     /// producer callback registered with deferProducer
 323     AsyncCall::Pointer deferredProducer;
 324 #endif
 325
 326     bool validLength() const;
 327     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 328
 329     friend std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 330 };
 331
 332 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 333
 334 /// \ingroup StoreAPI
 335 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 336
 337 namespace Store {
 338
 339 /// a smart pointer similar to std::unique_ptr<> that automatically
 340 /// release()s and unlock()s the guarded Entry on stack-unwinding failures
 341 class EntryGuard {
 342 public:
 343     /// \param entry either nil or a locked Entry to manage
 344     /// \param context default unlock() message
 345     EntryGuard(Entry *entry, const char *context):
 346         entry_(entry), context_(context) {
 347         assert(!entry_ || entry_->locked());
 348     }
 349
 350     ~EntryGuard() {
 351         if (entry_) {
 352             // something went wrong -- the caller did not unlockAndReset() us
 353             onException();
 354         }
 355     }
 356
 357     EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now)
 358
 359     /// like std::unique_ptr::get()
 360     /// \returns nil or the guarded (locked) entry
 361     Entry *get() {
 362         return entry_;
 363     }
 364
 365     /// like std::unique_ptr::reset()
 366     /// stops guarding the entry
 367     /// unlocks the entry (which may destroy it)
 368     void unlockAndReset(const char *resetContext = nullptr) {
 369         if (entry_) {
 370             entry_->unlock(resetContext ? resetContext : context_);
 371             entry_ = nullptr;
 372         }
 373     }
 374
 375 private:
 376     void onException() noexcept;
 377
 378     Entry *entry_; ///< the guarded Entry or nil
 379     const char *context_; ///< default unlock() message
 380 };
 381
 382 void Stats(StoreEntry *output);
 383 void Maintain(void *unused);
 384 }; // namespace Store
 385
 386 /// \ingroup StoreAPI
 387 size_t storeEntryInUse();
 388
 389 /// \ingroup StoreAPI
 390 const char *storeEntryFlags(const StoreEntry *);
 391
 392 /// \ingroup StoreAPI
 393 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 394
 395 /// \ingroup StoreAPI
 396 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 397
 398 /// \ingroup StoreAPI
 399 StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope keyScope = ksDefault);
 400
 401 /// \ingroup StoreAPI
 402 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope keyScope = ksDefault);
 403
 404 /// \ingroup StoreAPI
 405 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 406 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 407
 408 /// \ingroup StoreAPI
 409 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 410 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&);
 411
 412 /// \ingroup StoreAPI
 413 void storeInit(void);
 414
 415 /// \ingroup StoreAPI
 416 void storeConfigure(void);
 417
 418 /// \ingroup StoreAPI
 419 void storeFreeMemory(void);
 420
 421 /// \ingroup StoreAPI
 422 int expiresMoreThan(time_t, time_t);
 423
 424 /// \ingroup StoreAPI
 425 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 426
 427 /// \ingroup StoreAPI
 428 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 429
 430 /// \ingroup StoreAPI
 431 int storeTooManyDiskFilesOpen(void);
 432
 433 /// \ingroup StoreAPI
 434 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 435
 436 /// \ingroup StoreAPI
 437 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 438
 439 /// \ingroup StoreAPI
 440 void storeFsInit(void);
 441
 442 /// \ingroup StoreAPI
 443 void storeFsDone(void);
 444
 445 /// \ingroup StoreAPI
 446 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 447
 448 /// One of the three methods to get rid of an unlocked StoreEntry object.
 449 /// This low-level method ignores lock()ing and release() promises. It never
 450 /// leaves the entry in the local store_table.
 451 /// TODO: Hide by moving its functionality into the StoreEntry destructor.
 452 extern FREE destroyStoreEntry;
 453
 454 /// \ingroup StoreAPI
 455 void storeGetMemSpace(int size);
 456
 457 #endif /* SQUID_STORE_H */
 458