src/Store.h

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 #include "base/Packable.h"
  13 #include "base/RefCount.h"
  14 #include "comm/forward.h"
  15 #include "CommRead.h"
  16 #include "hash.h"
  17 #include "http/forward.h"
  18 #include "http/RequestMethod.h"
  19 #include "HttpReply.h"
  20 #include "MemObject.h"
  21 #include "Range.h"
  22 #include "RemovalPolicy.h"
  23 #include "StoreIOBuffer.h"
  24 #include "StoreStats.h"
  25
  26 #if USE_SQUID_ESI
  27 #include "esi/Element.h"
  28 #endif
  29
  30 #include <ostream>
  31
  32 class AsyncCall;
  33 class HttpRequest;
  34 class Packer;
  35 class RequestFlags;
  36 class StoreClient;
  37 class StoreSearch;
  38 class SwapDir;
  39
  40 extern StoreIoStats store_io_stats;
  41
  42 /// maximum number of entries per cache_dir
  43 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  44
  45 class StoreEntry : public hash_link, public Packable
  46 {
  47
  48 public:
  49     static DeferredRead::DeferrableRead DeferReader;
  50     bool checkDeferRead(int fd) const;
  51
  52     virtual const char *getMD5Text() const;
  53     StoreEntry();
  54     virtual ~StoreEntry();
  55
  56     virtual HttpReply const *getReply() const;
  57     virtual void write (StoreIOBuffer);
  58
  59     /** Check if the Store entry is emtpty
  60      * \retval true   Store contains 0 bytes of data.
  61      * \retval false  Store contains 1 or more bytes of data.
  62      * \retval false  Store contains negative content !!!!!!
  63      */
  64     virtual bool isEmpty() const {
  65         assert (mem_obj);
  66         return mem_obj->endOffset() == 0;
  67     }
  68     virtual bool isAccepting() const;
  69     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  70     virtual void complete();
  71     virtual store_client_t storeClientType() const;
  72     virtual char const *getSerialisedMetaData();
  73     /// Store a prepared error response. MemObject locks the reply object.
  74     void storeErrorResponse(HttpReply *reply);
  75     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  76     void startWriting(); ///< pack and write reply headers and, maybe, body
  77     /// whether we may start writing to disk (now or in the future)
  78     virtual bool mayStartSwapOut();
  79     virtual void trimMemory(const bool preserveSwappable);
  80
  81     // called when a decision to cache in memory has been made
  82     void memOutDecision(const bool willCacheInRam);
  83     // called when a decision to cache on disk has been made
  84     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  85
  86     void abort();
  87     void unlink();
  88     void makePublic();
  89     void makePrivate();
  90     void setPublicKey();
  91     void setPrivateKey();
  92     void expireNow();
  93     void releaseRequest();
  94     void negativeCache();
  95     void cacheNegatively();     /** \todo argh, why both? */
  96     void invokeHandlers();
  97     void purgeMem();
  98     void cacheInMemory(); ///< start or continue storing in memory cache
  99     void swapOut();
 100     /// whether we are in the process of writing this entry to disk
 101     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 102     void swapOutFileClose(int how);
 103     const char *url() const;
 104     /// Satisfies cachability requirements shared among disk and RAM caches.
 105     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 106     /// TODO: Rename and make private so only those two methods can call this.
 107     bool checkCachable();
 108     int checkNegativeHit() const;
 109     int locked() const;
 110     int validToSend() const;
 111     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 112
 113     /// if needed, initialize mem_obj member w/o URI-related information
 114     MemObject *makeMemObject();
 115
 116     /// initialize mem_obj member (if needed) and supply URI-related info
 117     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 118
 119     void dump(int debug_lvl) const;
 120     void hashDelete();
 121     void hashInsert(const cache_key *);
 122     void registerAbort(STABH * cb, void *);
 123     void reset();
 124     void setMemStatus(mem_status_t);
 125     void timestampsSet();
 126     void unregisterAbort();
 127     void destroyMemObject();
 128     int checkTooSmall();
 129
 130     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 131
 132     void setNoDelay (bool const);
 133     bool modifiedSince(HttpRequest * request) const;
 134     /// has ETag matching at least one of the If-Match etags
 135     bool hasIfMatchEtag(const HttpRequest &request) const;
 136     /// has ETag matching at least one of the If-None-Match etags
 137     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 138     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 139     bool hasEtag(ETag &etag) const;
 140
 141     /** What store does this entry belong too ? */
 142     virtual RefCount<SwapDir> store() const;
 143
 144     MemObject *mem_obj;
 145     RemovalPolicyNode repl;
 146     /* START OF ON-DISK STORE_META_STD TLV field */
 147     time_t timestamp;
 148     time_t lastref;
 149     time_t expires;
 150     time_t lastmod;
 151     uint64_t swap_file_sz;
 152     uint16_t refcount;
 153     uint16_t flags;
 154     /* END OF ON-DISK STORE_META_STD */
 155
 156     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 157     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 158
 159     sdirno swap_dirn:7;
 160
 161     mem_status_t mem_status:3;
 162
 163     ping_status_t ping_status:3;
 164
 165     store_status_t store_status:3;
 166
 167     swap_status_t swap_status:3;
 168
 169 public:
 170     static size_t inUseCount();
 171     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 172     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 173     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 174
 175     virtual bool isNull() {
 176         return false;
 177     };
 178
 179     void *operator new(size_t byteCount);
 180     void operator delete(void *address);
 181     void setReleaseFlag();
 182 #if USE_SQUID_ESI
 183
 184     ESIElement::Pointer cachedESITree;
 185 #endif
 186     /** disable sending content to the clients */
 187     virtual void buffer();
 188     /** flush any buffered content */
 189     virtual void flush();
 190     virtual int64_t objectLen() const;
 191     virtual int64_t contentLen() const;
 192
 193     /// claim shared ownership of this entry (for use in a given context)
 194     /// matching lock() and unlock() contexts eases leak triage but is optional
 195     void lock(const char *context);
 196
 197     /// disclaim shared ownership; may remove entry from store and delete it
 198     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 199     int unlock(const char *context);
 200
 201     /// returns a local concurrent use counter, for debugging
 202     int locks() const { return static_cast<int>(lock_count); }
 203
 204     /// update last reference timestamp and related Store metadata
 205     void touch();
 206
 207     virtual void release();
 208
 209 #if USE_ADAPTATION
 210     /// call back producer when more buffer space is available
 211     void deferProducer(const AsyncCall::Pointer &producer);
 212     /// calls back producer registered with deferProducer
 213     void kickProducer();
 214 #endif
 215
 216     /* Packable API */
 217     virtual void append(char const *, int);
 218     virtual void vappendf(const char *, va_list);
 219
 220 protected:
 221     void transientsAbandonmentCheck();
 222
 223 private:
 224     bool checkTooBig() const;
 225
 226     static MemAllocator *pool;
 227
 228     unsigned short lock_count;      /* Assume < 65536! */
 229
 230 #if USE_ADAPTATION
 231     /// producer callback registered with deferProducer
 232     AsyncCall::Pointer deferredProducer;
 233 #endif
 234
 235     bool validLength() const;
 236     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 237 };
 238
 239 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 240
 241 /// \ingroup StoreAPI
 242 class NullStoreEntry:public StoreEntry
 243 {
 244
 245 public:
 246     static NullStoreEntry *getInstance();
 247     bool isNull() {
 248         return true;
 249     }
 250
 251     const char *getMD5Text() const;
 252     HttpReply const *getReply() const { return NULL; }
 253     void write (StoreIOBuffer) {}
 254
 255     bool isEmpty () const {return true;}
 256
 257     virtual size_t bytesWanted(Range<size_t> const aRange, bool) const { return aRange.end; }
 258
 259     void operator delete(void *address);
 260     void complete() {}
 261
 262 private:
 263     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 264
 265     char const *getSerialisedMetaData();
 266     virtual bool mayStartSwapOut() { return false; }
 267
 268     void trimMemory(const bool) {}
 269
 270     static NullStoreEntry _instance;
 271 };
 272
 273 /// \ingroup StoreAPI
 274 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 275
 276 /**
 277  \ingroup StoreAPI
 278  * Abstract base class that will replace the whole store and swapdir interface.
 279  */
 280 class Store : public RefCountable
 281 {
 282
 283 public:
 284     /** The root store */
 285     static Store &Root() {
 286         if (CurrentRoot == NULL)
 287             fatal("No Store Root has been set");
 288         return *CurrentRoot;
 289     }
 290     static void Root(Store *);
 291     static void Root(RefCount<Store>);
 292     static void Stats(StoreEntry * output);
 293     static void Maintain(void *unused);
 294
 295     virtual ~Store() {}
 296
 297     /** Handle pending callbacks - called by the event loop. */
 298     virtual int callback() = 0;
 299
 300     /** create the resources needed for this store to operate */
 301     virtual void create();
 302
 303     /**
 304      * Notify this store that its disk is full.
 305      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 306      */
 307     virtual void diskFull();
 308
 309     /** Retrieve a store entry from the store */
 310     virtual StoreEntry * get(const cache_key *) = 0;
 311
 312     /** \todo imeplement the async version */
 313     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 314
 315     /* prepare the store for use. The store need not be usable immediately,
 316      * it should respond to readable() and writable() with true as soon
 317      * as it can provide those services
 318      */
 319     virtual void init() = 0;
 320
 321     /**
 322      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 323      * but may throw estimates for memory etc out of whack.
 324      */
 325     virtual uint64_t maxSize() const = 0;
 326
 327     /** The minimum size the store will shrink to via normal housekeeping */
 328     virtual uint64_t minSize() const = 0;
 329
 330     /** current store size */
 331     virtual uint64_t currentSize() const = 0;
 332
 333     /** the total number of objects stored */
 334     virtual uint64_t currentCount() const = 0;
 335
 336     /** the maximum object size that can be stored, -1 if unlimited */
 337     virtual int64_t maxObjectSize() const = 0;
 338
 339     /// collect cache storage-related statistics
 340     virtual void getStats(StoreInfoStats &stats) const = 0;
 341
 342     /**
 343      * Output stats to the provided store entry.
 344      \todo make these calls asynchronous
 345      */
 346     virtual void stat(StoreEntry &) const = 0;
 347
 348     /** Sync the store prior to shutdown */
 349     virtual void sync();
 350
 351     /** remove a Store entry from the store */
 352     virtual void unlink (StoreEntry &);
 353
 354     /* search in the store */
 355     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 356
 357     /* pulled up from SwapDir for migration.... probably do not belong here */
 358     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 359
 360     /// Undo reference(), returning false iff idle e should be destroyed
 361     virtual bool dereference(StoreEntry &, bool wantsLocalMemory) = 0;
 362
 363     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 364
 365     // XXX: This method belongs to Store::Root/StoreController, but it is here
 366     // to avoid casting Root() to StoreController until Root() API is fixed.
 367     /// informs stores that this entry will be eventually unlinked
 368     virtual void markForUnlink(StoreEntry &) {}
 369
 370     // XXX: This method belongs to Store::Root/StoreController, but it is here
 371     // because test cases use non-StoreController derivatives as Root
 372     /// called when the entry is no longer needed by any transaction
 373     virtual void handleIdleEntry(StoreEntry &) {}
 374
 375     // XXX: This method belongs to Store::Root/StoreController, but it is here
 376     // because test cases use non-StoreController derivatives as Root
 377     /// called to get rid of no longer needed entry data in RAM, if any
 378     virtual void memoryOut(StoreEntry &, const bool /*preserveSwappable*/) {}
 379
 380     // XXX: This method belongs to Store::Root/StoreController, but it is here
 381     // to avoid casting Root() to StoreController until Root() API is fixed.
 382     /// makes the entry available for collapsing future requests
 383     virtual void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &) {}
 384
 385     // XXX: This method belongs to Store::Root/StoreController, but it is here
 386     // to avoid casting Root() to StoreController until Root() API is fixed.
 387     /// marks the entry completed for collapsed requests
 388     virtual void transientsCompleteWriting(StoreEntry &) {}
 389
 390     // XXX: This method belongs to Store::Root/StoreController, but it is here
 391     // to avoid casting Root() to StoreController until Root() API is fixed.
 392     /// Update local intransit entry after changes made by appending worker.
 393     virtual void syncCollapsed(const sfileno) {}
 394
 395     // XXX: This method belongs to Store::Root/StoreController, but it is here
 396     // to avoid casting Root() to StoreController until Root() API is fixed.
 397     /// calls Root().transients->abandon() if transients are tracked
 398     virtual void transientsAbandon(StoreEntry &) {}
 399
 400     // XXX: This method belongs to Store::Root/StoreController, but it is here
 401     // to avoid casting Root() to StoreController until Root() API is fixed.
 402     /// number of the transient entry readers some time ago
 403     virtual int transientReaders(const StoreEntry &) const { return 0; }
 404
 405     // XXX: This method belongs to Store::Root/StoreController, but it is here
 406     // to avoid casting Root() to StoreController until Root() API is fixed.
 407     /// disassociates the entry from the intransit table
 408     virtual void transientsDisconnect(MemObject &) {}
 409
 410     // XXX: This method belongs to Store::Root/StoreController, but it is here
 411     // to avoid casting Root() to StoreController until Root() API is fixed.
 412     /// removes the entry from the memory cache
 413     virtual void memoryUnlink(StoreEntry &) {}
 414
 415     // XXX: This method belongs to Store::Root/StoreController, but it is here
 416     // to avoid casting Root() to StoreController until Root() API is fixed.
 417     /// disassociates the entry from the memory cache, preserving cached data
 418     virtual void memoryDisconnect(StoreEntry &) {}
 419
 420     /// If the entry is not found, return false. Otherwise, return true after
 421     /// tying the entry to this cache and setting inSync to updateCollapsed().
 422     virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
 423
 424     /// update a local collapsed entry with fresh info from this cache (if any)
 425     virtual bool updateCollapsed(StoreEntry &) { return false; }
 426
 427 private:
 428     static RefCount<Store> CurrentRoot;
 429 };
 430
 431 /// \ingroup StoreAPI
 432 typedef RefCount<Store> StorePointer;
 433
 434 /// \ingroup StoreAPI
 435 size_t storeEntryInUse();
 436
 437 /// \ingroup StoreAPI
 438 const char *storeEntryFlags(const StoreEntry *);
 439
 440 /// \ingroup StoreAPI
 441 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 442
 443 /// \ingroup StoreAPI
 444 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 445
 446 /// \ingroup StoreAPI
 447 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 448
 449 /// \ingroup StoreAPI
 450 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 451
 452 /// \ingroup StoreAPI
 453 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 454 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 455
 456 /// \ingroup StoreAPI
 457 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 458 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 459
 460 /// \ingroup StoreAPI
 461 void storeInit(void);
 462
 463 /// \ingroup StoreAPI
 464 void storeConfigure(void);
 465
 466 /// \ingroup StoreAPI
 467 void storeFreeMemory(void);
 468
 469 /// \ingroup StoreAPI
 470 int expiresMoreThan(time_t, time_t);
 471
 472 /// \ingroup StoreAPI
 473 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 474
 475 /// \ingroup StoreAPI
 476 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 477
 478 /// \ingroup StoreAPI
 479 int storeTooManyDiskFilesOpen(void);
 480
 481 class SwapDir;
 482 /// \ingroup StoreAPI
 483 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 484
 485 /// \ingroup StoreAPI
 486 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 487
 488 /// \ingroup StoreAPI
 489 void storeFsInit(void);
 490
 491 /// \ingroup StoreAPI
 492 void storeFsDone(void);
 493
 494 /// \ingroup StoreAPI
 495 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 496
 497 /// \ingroup StoreAPI
 498 extern FREE destroyStoreEntry;
 499
 500 /**
 501  \ingroup StoreAPI
 502  \todo should be a subclass of Packer perhaps ?
 503  */
 504 void packerToStoreInit(Packer * p, StoreEntry * e);
 505
 506 /// \ingroup StoreAPI
 507 void storeGetMemSpace(int size);
 508
 509 #endif /* SQUID_STORE_H */
 510