src/Store.h

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 #include "base/Packable.h"
  13 #include "base/RefCount.h"
  14 #include "comm/forward.h"
  15 #include "CommRead.h"
  16 #include "hash.h"
  17 #include "http/forward.h"
  18 #include "http/RequestMethod.h"
  19 #include "HttpReply.h"
  20 #include "MemObject.h"
  21 #include "Range.h"
  22 #include "RemovalPolicy.h"
  23 #include "StoreIOBuffer.h"
  24 #include "StoreStats.h"
  25
  26 #if USE_SQUID_ESI
  27 #include "esi/Element.h"
  28 #endif
  29
  30 #include <ostream>
  31
  32 class AsyncCall;
  33 class HttpRequest;
  34 class RequestFlags;
  35 class StoreClient;
  36 class StoreSearch;
  37 class SwapDir;
  38
  39 extern StoreIoStats store_io_stats;
  40
  41 /// maximum number of entries per cache_dir
  42 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  43
  44 class StoreEntry : public hash_link, public Packable
  45 {
  46
  47 public:
  48     static DeferredRead::DeferrableRead DeferReader;
  49     bool checkDeferRead(int fd) const;
  50
  51     virtual const char *getMD5Text() const;
  52     StoreEntry();
  53     virtual ~StoreEntry();
  54
  55     virtual HttpReply const *getReply() const;
  56     virtual void write (StoreIOBuffer);
  57
  58     /** Check if the Store entry is emtpty
  59      * \retval true   Store contains 0 bytes of data.
  60      * \retval false  Store contains 1 or more bytes of data.
  61      * \retval false  Store contains negative content !!!!!!
  62      */
  63     virtual bool isEmpty() const {
  64         assert (mem_obj);
  65         return mem_obj->endOffset() == 0;
  66     }
  67     virtual bool isAccepting() const;
  68     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  69     virtual void complete();
  70     virtual store_client_t storeClientType() const;
  71     virtual char const *getSerialisedMetaData();
  72     /// Store a prepared error response. MemObject locks the reply object.
  73     void storeErrorResponse(HttpReply *reply);
  74     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  75     void startWriting(); ///< pack and write reply headers and, maybe, body
  76     /// whether we may start writing to disk (now or in the future)
  77     virtual bool mayStartSwapOut();
  78     virtual void trimMemory(const bool preserveSwappable);
  79
  80     // called when a decision to cache in memory has been made
  81     void memOutDecision(const bool willCacheInRam);
  82     // called when a decision to cache on disk has been made
  83     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  84
  85     void abort();
  86     void unlink();
  87     void makePublic();
  88     void makePrivate();
  89     void setPublicKey();
  90     void setPrivateKey();
  91     void expireNow();
  92     void releaseRequest();
  93     void negativeCache();
  94     void cacheNegatively();     /** \todo argh, why both? */
  95     void invokeHandlers();
  96     void purgeMem();
  97     void cacheInMemory(); ///< start or continue storing in memory cache
  98     void swapOut();
  99     /// whether we are in the process of writing this entry to disk
 100     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 101     void swapOutFileClose(int how);
 102     const char *url() const;
 103     /// Satisfies cachability requirements shared among disk and RAM caches.
 104     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 105     /// TODO: Rename and make private so only those two methods can call this.
 106     bool checkCachable();
 107     int checkNegativeHit() const;
 108     int locked() const;
 109     int validToSend() const;
 110     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 111
 112     /// if needed, initialize mem_obj member w/o URI-related information
 113     MemObject *makeMemObject();
 114
 115     /// initialize mem_obj member (if needed) and supply URI-related info
 116     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 117
 118     void dump(int debug_lvl) const;
 119     void hashDelete();
 120     void hashInsert(const cache_key *);
 121     void registerAbort(STABH * cb, void *);
 122     void reset();
 123     void setMemStatus(mem_status_t);
 124     void timestampsSet();
 125     void unregisterAbort();
 126     void destroyMemObject();
 127     int checkTooSmall();
 128
 129     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 130
 131     void setNoDelay (bool const);
 132     bool modifiedSince(HttpRequest * request) const;
 133     /// has ETag matching at least one of the If-Match etags
 134     bool hasIfMatchEtag(const HttpRequest &request) const;
 135     /// has ETag matching at least one of the If-None-Match etags
 136     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 137     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 138     bool hasEtag(ETag &etag) const;
 139
 140     /** What store does this entry belong too ? */
 141     virtual RefCount<SwapDir> store() const;
 142
 143     MemObject *mem_obj;
 144     RemovalPolicyNode repl;
 145     /* START OF ON-DISK STORE_META_STD TLV field */
 146     time_t timestamp;
 147     time_t lastref;
 148     time_t expires;
 149     time_t lastmod;
 150     uint64_t swap_file_sz;
 151     uint16_t refcount;
 152     uint16_t flags;
 153     /* END OF ON-DISK STORE_META_STD */
 154
 155     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 156     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 157
 158     sdirno swap_dirn:7;
 159
 160     mem_status_t mem_status:3;
 161
 162     ping_status_t ping_status:3;
 163
 164     store_status_t store_status:3;
 165
 166     swap_status_t swap_status:3;
 167
 168 public:
 169     static size_t inUseCount();
 170     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 171     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 172     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 173
 174     virtual bool isNull() {
 175         return false;
 176     };
 177
 178     void *operator new(size_t byteCount);
 179     void operator delete(void *address);
 180     void setReleaseFlag();
 181 #if USE_SQUID_ESI
 182
 183     ESIElement::Pointer cachedESITree;
 184 #endif
 185     /** disable sending content to the clients */
 186     virtual void buffer();
 187     /** flush any buffered content */
 188     virtual void flush();
 189     virtual int64_t objectLen() const;
 190     virtual int64_t contentLen() const;
 191
 192     /// claim shared ownership of this entry (for use in a given context)
 193     /// matching lock() and unlock() contexts eases leak triage but is optional
 194     void lock(const char *context);
 195
 196     /// disclaim shared ownership; may remove entry from store and delete it
 197     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 198     int unlock(const char *context);
 199
 200     /// returns a local concurrent use counter, for debugging
 201     int locks() const { return static_cast<int>(lock_count); }
 202
 203     /// update last reference timestamp and related Store metadata
 204     void touch();
 205
 206     virtual void release();
 207
 208 #if USE_ADAPTATION
 209     /// call back producer when more buffer space is available
 210     void deferProducer(const AsyncCall::Pointer &producer);
 211     /// calls back producer registered with deferProducer
 212     void kickProducer();
 213 #endif
 214
 215     /* Packable API */
 216     virtual void append(char const *, int);
 217     virtual void vappendf(const char *, va_list);
 218
 219 protected:
 220     void transientsAbandonmentCheck();
 221
 222 private:
 223     bool checkTooBig() const;
 224
 225     static MemAllocator *pool;
 226
 227     unsigned short lock_count;      /* Assume < 65536! */
 228
 229 #if USE_ADAPTATION
 230     /// producer callback registered with deferProducer
 231     AsyncCall::Pointer deferredProducer;
 232 #endif
 233
 234     bool validLength() const;
 235     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 236 };
 237
 238 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 239
 240 /// \ingroup StoreAPI
 241 class NullStoreEntry:public StoreEntry
 242 {
 243
 244 public:
 245     static NullStoreEntry *getInstance();
 246     bool isNull() {
 247         return true;
 248     }
 249
 250     const char *getMD5Text() const;
 251     HttpReply const *getReply() const { return NULL; }
 252     void write (StoreIOBuffer) {}
 253
 254     bool isEmpty () const {return true;}
 255
 256     virtual size_t bytesWanted(Range<size_t> const aRange, bool) const { return aRange.end; }
 257
 258     void operator delete(void *address);
 259     void complete() {}
 260
 261 private:
 262     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 263
 264     char const *getSerialisedMetaData();
 265     virtual bool mayStartSwapOut() { return false; }
 266
 267     void trimMemory(const bool) {}
 268
 269     static NullStoreEntry _instance;
 270 };
 271
 272 /// \ingroup StoreAPI
 273 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 274
 275 /**
 276  \ingroup StoreAPI
 277  * Abstract base class that will replace the whole store and swapdir interface.
 278  */
 279 class Store : public RefCountable
 280 {
 281
 282 public:
 283     /** The root store */
 284     static Store &Root() {
 285         if (CurrentRoot == NULL)
 286             fatal("No Store Root has been set");
 287         return *CurrentRoot;
 288     }
 289     static void Root(Store *);
 290     static void Root(RefCount<Store>);
 291     static void Stats(StoreEntry * output);
 292     static void Maintain(void *unused);
 293
 294     virtual ~Store() {}
 295
 296     /** Handle pending callbacks - called by the event loop. */
 297     virtual int callback() = 0;
 298
 299     /** create the resources needed for this store to operate */
 300     virtual void create();
 301
 302     /**
 303      * Notify this store that its disk is full.
 304      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 305      */
 306     virtual void diskFull();
 307
 308     /** Retrieve a store entry from the store */
 309     virtual StoreEntry * get(const cache_key *) = 0;
 310
 311     /** \todo imeplement the async version */
 312     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 313
 314     /* prepare the store for use. The store need not be usable immediately,
 315      * it should respond to readable() and writable() with true as soon
 316      * as it can provide those services
 317      */
 318     virtual void init() = 0;
 319
 320     /**
 321      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 322      * but may throw estimates for memory etc out of whack.
 323      */
 324     virtual uint64_t maxSize() const = 0;
 325
 326     /** The minimum size the store will shrink to via normal housekeeping */
 327     virtual uint64_t minSize() const = 0;
 328
 329     /** current store size */
 330     virtual uint64_t currentSize() const = 0;
 331
 332     /** the total number of objects stored */
 333     virtual uint64_t currentCount() const = 0;
 334
 335     /** the maximum object size that can be stored, -1 if unlimited */
 336     virtual int64_t maxObjectSize() const = 0;
 337
 338     /// collect cache storage-related statistics
 339     virtual void getStats(StoreInfoStats &stats) const = 0;
 340
 341     /**
 342      * Output stats to the provided store entry.
 343      \todo make these calls asynchronous
 344      */
 345     virtual void stat(StoreEntry &) const = 0;
 346
 347     /** Sync the store prior to shutdown */
 348     virtual void sync();
 349
 350     /** remove a Store entry from the store */
 351     virtual void unlink (StoreEntry &);
 352
 353     /* search in the store */
 354     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 355
 356     /* pulled up from SwapDir for migration.... probably do not belong here */
 357     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 358
 359     /// Undo reference(), returning false iff idle e should be destroyed
 360     virtual bool dereference(StoreEntry &, bool wantsLocalMemory) = 0;
 361
 362     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 363
 364     // XXX: This method belongs to Store::Root/StoreController, but it is here
 365     // to avoid casting Root() to StoreController until Root() API is fixed.
 366     /// informs stores that this entry will be eventually unlinked
 367     virtual void markForUnlink(StoreEntry &) {}
 368
 369     // XXX: This method belongs to Store::Root/StoreController, but it is here
 370     // because test cases use non-StoreController derivatives as Root
 371     /// called when the entry is no longer needed by any transaction
 372     virtual void handleIdleEntry(StoreEntry &) {}
 373
 374     // XXX: This method belongs to Store::Root/StoreController, but it is here
 375     // because test cases use non-StoreController derivatives as Root
 376     /// called to get rid of no longer needed entry data in RAM, if any
 377     virtual void memoryOut(StoreEntry &, const bool /*preserveSwappable*/) {}
 378
 379     // XXX: This method belongs to Store::Root/StoreController, but it is here
 380     // to avoid casting Root() to StoreController until Root() API is fixed.
 381     /// makes the entry available for collapsing future requests
 382     virtual void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &) {}
 383
 384     // XXX: This method belongs to Store::Root/StoreController, but it is here
 385     // to avoid casting Root() to StoreController until Root() API is fixed.
 386     /// marks the entry completed for collapsed requests
 387     virtual void transientsCompleteWriting(StoreEntry &) {}
 388
 389     // XXX: This method belongs to Store::Root/StoreController, but it is here
 390     // to avoid casting Root() to StoreController until Root() API is fixed.
 391     /// Update local intransit entry after changes made by appending worker.
 392     virtual void syncCollapsed(const sfileno) {}
 393
 394     // XXX: This method belongs to Store::Root/StoreController, but it is here
 395     // to avoid casting Root() to StoreController until Root() API is fixed.
 396     /// calls Root().transients->abandon() if transients are tracked
 397     virtual void transientsAbandon(StoreEntry &) {}
 398
 399     // XXX: This method belongs to Store::Root/StoreController, but it is here
 400     // to avoid casting Root() to StoreController until Root() API is fixed.
 401     /// number of the transient entry readers some time ago
 402     virtual int transientReaders(const StoreEntry &) const { return 0; }
 403
 404     // XXX: This method belongs to Store::Root/StoreController, but it is here
 405     // to avoid casting Root() to StoreController until Root() API is fixed.
 406     /// disassociates the entry from the intransit table
 407     virtual void transientsDisconnect(MemObject &) {}
 408
 409     // XXX: This method belongs to Store::Root/StoreController, but it is here
 410     // to avoid casting Root() to StoreController until Root() API is fixed.
 411     /// removes the entry from the memory cache
 412     virtual void memoryUnlink(StoreEntry &) {}
 413
 414     // XXX: This method belongs to Store::Root/StoreController, but it is here
 415     // to avoid casting Root() to StoreController until Root() API is fixed.
 416     /// disassociates the entry from the memory cache, preserving cached data
 417     virtual void memoryDisconnect(StoreEntry &) {}
 418
 419     /// If the entry is not found, return false. Otherwise, return true after
 420     /// tying the entry to this cache and setting inSync to updateCollapsed().
 421     virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
 422
 423     /// update a local collapsed entry with fresh info from this cache (if any)
 424     virtual bool updateCollapsed(StoreEntry &) { return false; }
 425
 426 private:
 427     static RefCount<Store> CurrentRoot;
 428 };
 429
 430 /// \ingroup StoreAPI
 431 typedef RefCount<Store> StorePointer;
 432
 433 /// \ingroup StoreAPI
 434 size_t storeEntryInUse();
 435
 436 /// \ingroup StoreAPI
 437 const char *storeEntryFlags(const StoreEntry *);
 438
 439 /// \ingroup StoreAPI
 440 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 441
 442 /// \ingroup StoreAPI
 443 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 444
 445 /// \ingroup StoreAPI
 446 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 447
 448 /// \ingroup StoreAPI
 449 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 450
 451 /// \ingroup StoreAPI
 452 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 453 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 454
 455 /// \ingroup StoreAPI
 456 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 457 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 458
 459 /// \ingroup StoreAPI
 460 void storeInit(void);
 461
 462 /// \ingroup StoreAPI
 463 void storeConfigure(void);
 464
 465 /// \ingroup StoreAPI
 466 void storeFreeMemory(void);
 467
 468 /// \ingroup StoreAPI
 469 int expiresMoreThan(time_t, time_t);
 470
 471 /// \ingroup StoreAPI
 472 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 473
 474 /// \ingroup StoreAPI
 475 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 476
 477 /// \ingroup StoreAPI
 478 int storeTooManyDiskFilesOpen(void);
 479
 480 class SwapDir;
 481 /// \ingroup StoreAPI
 482 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 483
 484 /// \ingroup StoreAPI
 485 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 486
 487 /// \ingroup StoreAPI
 488 void storeFsInit(void);
 489
 490 /// \ingroup StoreAPI
 491 void storeFsDone(void);
 492
 493 /// \ingroup StoreAPI
 494 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 495
 496 /// \ingroup StoreAPI
 497 extern FREE destroyStoreEntry;
 498
 499 /// \ingroup StoreAPI
 500 void storeGetMemSpace(int size);
 501
 502 #endif /* SQUID_STORE_H */
 503