src/Store.h

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 /**
  13  \defgroup StoreAPI  Store API
  14  \ingroup FileSystems
  15  */
  16
  17 #include "base/RefCount.h"
  18 #include "comm/forward.h"
  19 #include "CommRead.h"
  20 #include "hash.h"
  21 #include "HttpReply.h"
  22 #include "HttpRequestMethod.h"
  23 #include "MemObject.h"
  24 #include "Range.h"
  25 #include "RemovalPolicy.h"
  26 #include "StoreIOBuffer.h"
  27 #include "StoreStats.h"
  28
  29 #if USE_SQUID_ESI
  30 #include "esi/Element.h"
  31 #endif
  32
  33 #include <ostream>
  34
  35 class AsyncCall;
  36 class HttpRequest;
  37 class Packer;
  38 class RequestFlags;
  39 class StoreClient;
  40 class StoreSearch;
  41 class SwapDir;
  42
  43 extern StoreIoStats store_io_stats;
  44
  45 /// maximum number of entries per cache_dir
  46 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  47
  48 /**
  49  \ingroup StoreAPI
  50  */
  51 class StoreEntry : public hash_link
  52 {
  53
  54 public:
  55     static DeferredRead::DeferrableRead DeferReader;
  56     bool checkDeferRead(int fd) const;
  57
  58     virtual const char *getMD5Text() const;
  59     StoreEntry();
  60     virtual ~StoreEntry();
  61
  62     virtual HttpReply const *getReply() const;
  63     virtual void write (StoreIOBuffer);
  64
  65     /** Check if the Store entry is emtpty
  66      * \retval true   Store contains 0 bytes of data.
  67      * \retval false  Store contains 1 or more bytes of data.
  68      * \retval false  Store contains negative content !!!!!!
  69      */
  70     virtual bool isEmpty() const {
  71         assert (mem_obj);
  72         return mem_obj->endOffset() == 0;
  73     }
  74     virtual bool isAccepting() const;
  75     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  76     virtual void complete();
  77     virtual store_client_t storeClientType() const;
  78     virtual char const *getSerialisedMetaData();
  79     /// Store a prepared error response. MemObject locks the reply object.
  80     void storeErrorResponse(HttpReply *reply);
  81     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  82     void startWriting(); ///< pack and write reply headers and, maybe, body
  83     /// whether we may start writing to disk (now or in the future)
  84     virtual bool mayStartSwapOut();
  85     virtual void trimMemory(const bool preserveSwappable);
  86
  87     // called when a decision to cache in memory has been made
  88     void memOutDecision(const bool willCacheInRam);
  89     // called when a decision to cache on disk has been made
  90     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  91
  92     void abort();
  93     void unlink();
  94     void makePublic();
  95     void makePrivate();
  96     void setPublicKey();
  97     void setPrivateKey();
  98     void expireNow();
  99     void releaseRequest();
 100     void negativeCache();
 101     void cacheNegatively();             /** \todo argh, why both? */
 102     void invokeHandlers();
 103     void purgeMem();
 104     void cacheInMemory(); ///< start or continue storing in memory cache
 105     void swapOut();
 106     /// whether we are in the process of writing this entry to disk
 107     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 108     void swapOutFileClose(int how);
 109     const char *url() const;
 110     /// Satisfies cachability requirements shared among disk and RAM caches.
 111     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 112     /// TODO: Rename and make private so only those two methods can call this.
 113     bool checkCachable();
 114     int checkNegativeHit() const;
 115     int locked() const;
 116     int validToSend() const;
 117     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 118
 119     /// if needed, initialize mem_obj member w/o URI-related information
 120     MemObject *makeMemObject();
 121
 122     /// initialize mem_obj member (if needed) and supply URI-related info
 123     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 124
 125     void dump(int debug_lvl) const;
 126     void hashDelete();
 127     void hashInsert(const cache_key *);
 128     void registerAbort(STABH * cb, void *);
 129     void reset();
 130     void setMemStatus(mem_status_t);
 131     void timestampsSet();
 132     void unregisterAbort();
 133     void destroyMemObject();
 134     int checkTooSmall();
 135
 136     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 137
 138     void setNoDelay (bool const);
 139     bool modifiedSince(HttpRequest * request) const;
 140     /// has ETag matching at least one of the If-Match etags
 141     bool hasIfMatchEtag(const HttpRequest &request) const;
 142     /// has ETag matching at least one of the If-None-Match etags
 143     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 144     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 145     bool hasEtag(ETag &etag) const;
 146
 147     /** What store does this entry belong too ? */
 148     virtual RefCount<SwapDir> store() const;
 149
 150     MemObject *mem_obj;
 151     RemovalPolicyNode repl;
 152     /* START OF ON-DISK STORE_META_STD TLV field */
 153     time_t timestamp;
 154     time_t lastref;
 155     time_t expires;
 156     time_t lastmod;
 157     uint64_t swap_file_sz;
 158     uint16_t refcount;
 159     uint16_t flags;
 160     /* END OF ON-DISK STORE_META_STD */
 161
 162     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 163     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 164
 165     sdirno swap_dirn:7;
 166
 167     mem_status_t mem_status:3;
 168
 169     ping_status_t ping_status:3;
 170
 171     store_status_t store_status:3;
 172
 173     swap_status_t swap_status:3;
 174
 175 public:
 176     static size_t inUseCount();
 177     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 178     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 179     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 180
 181     virtual bool isNull() {
 182         return false;
 183     };
 184
 185     void *operator new(size_t byteCount);
 186     void operator delete(void *address);
 187     void setReleaseFlag();
 188 #if USE_SQUID_ESI
 189
 190     ESIElement::Pointer cachedESITree;
 191 #endif
 192     /** append bytes to the buffer */
 193     virtual void append(char const *, int len);
 194     /** disable sending content to the clients */
 195     virtual void buffer();
 196     /** flush any buffered content */
 197     virtual void flush();
 198     virtual int64_t objectLen() const;
 199     virtual int64_t contentLen() const;
 200
 201     /// claim shared ownership of this entry (for use in a given context)
 202     /// matching lock() and unlock() contexts eases leak triage but is optional
 203     void lock(const char *context);
 204
 205     /// disclaim shared ownership; may remove entry from store and delete it
 206     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 207     int unlock(const char *context);
 208
 209     /// returns a local concurrent use counter, for debugging
 210     int locks() const { return static_cast<int>(lock_count); }
 211
 212     /// update last reference timestamp and related Store metadata
 213     void touch();
 214
 215     virtual void release();
 216
 217 #if USE_ADAPTATION
 218     /// call back producer when more buffer space is available
 219     void deferProducer(const AsyncCall::Pointer &producer);
 220     /// calls back producer registered with deferProducer
 221     void kickProducer();
 222 #endif
 223
 224 protected:
 225     void transientsAbandonmentCheck();
 226
 227 private:
 228     static MemAllocator *pool;
 229
 230     unsigned short lock_count;          /* Assume < 65536! */
 231
 232 #if USE_ADAPTATION
 233     /// producer callback registered with deferProducer
 234     AsyncCall::Pointer deferredProducer;
 235 #endif
 236
 237     bool validLength() const;
 238     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 239 };
 240
 241 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 242
 243 /// \ingroup StoreAPI
 244 class NullStoreEntry:public StoreEntry
 245 {
 246
 247 public:
 248     static NullStoreEntry *getInstance();
 249     bool isNull() {
 250         return true;
 251     }
 252
 253     const char *getMD5Text() const;
 254     HttpReply const *getReply() const { return NULL; }
 255     void write (StoreIOBuffer) {}
 256
 257     bool isEmpty () const {return true;}
 258
 259     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const { return aRange.end; }
 260
 261     void operator delete(void *address);
 262     void complete() {}
 263
 264 private:
 265     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 266
 267     char const *getSerialisedMetaData();
 268     virtual bool mayStartSwapOut() { return false; }
 269
 270     void trimMemory(const bool preserveSwappable) {}
 271
 272     static NullStoreEntry _instance;
 273 };
 274
 275 /// \ingroup StoreAPI
 276 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 277
 278 /**
 279  \ingroup StoreAPI
 280  * Abstract base class that will replace the whole store and swapdir interface.
 281  */
 282 class Store : public RefCountable
 283 {
 284
 285 public:
 286     /** The root store */
 287     static Store &Root() {
 288         if (CurrentRoot == NULL)
 289             fatal("No Store Root has been set");
 290         return *CurrentRoot;
 291     }
 292     static void Root(Store *);
 293     static void Root(RefCount<Store>);
 294     static void Stats(StoreEntry * output);
 295     static void Maintain(void *unused);
 296
 297     virtual ~Store() {}
 298
 299     /** Handle pending callbacks - called by the event loop. */
 300     virtual int callback() = 0;
 301
 302     /** create the resources needed for this store to operate */
 303     virtual void create();
 304
 305     /**
 306      * Notify this store that its disk is full.
 307      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 308      */
 309     virtual void diskFull();
 310
 311     /** Retrieve a store entry from the store */
 312     virtual StoreEntry * get(const cache_key *) = 0;
 313
 314     /** \todo imeplement the async version */
 315     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 316
 317     /* prepare the store for use. The store need not be usable immediately,
 318      * it should respond to readable() and writable() with true as soon
 319      * as it can provide those services
 320      */
 321     virtual void init() = 0;
 322
 323     /**
 324      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 325      * but may throw estimates for memory etc out of whack.
 326      */
 327     virtual uint64_t maxSize() const = 0;
 328
 329     /** The minimum size the store will shrink to via normal housekeeping */
 330     virtual uint64_t minSize() const = 0;
 331
 332     /** current store size */
 333     virtual uint64_t currentSize() const = 0;
 334
 335     /** the total number of objects stored */
 336     virtual uint64_t currentCount() const = 0;
 337
 338     /** the maximum object size that can be stored, -1 if unlimited */
 339     virtual int64_t maxObjectSize() const = 0;
 340
 341     /// collect cache storage-related statistics
 342     virtual void getStats(StoreInfoStats &stats) const = 0;
 343
 344     /**
 345      * Output stats to the provided store entry.
 346      \todo make these calls asynchronous
 347      */
 348     virtual void stat(StoreEntry &) const = 0;
 349
 350     /** Sync the store prior to shutdown */
 351     virtual void sync();
 352
 353     /** remove a Store entry from the store */
 354     virtual void unlink (StoreEntry &);
 355
 356     /* search in the store */
 357     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 358
 359     /* pulled up from SwapDir for migration.... probably do not belong here */
 360     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 361
 362     /// Undo reference(), returning false iff idle e should be destroyed
 363     virtual bool dereference(StoreEntry &e, bool wantsLocalMemory) = 0;
 364
 365     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 366
 367     // XXX: This method belongs to Store::Root/StoreController, but it is here
 368     // to avoid casting Root() to StoreController until Root() API is fixed.
 369     /// informs stores that this entry will be eventually unlinked
 370     virtual void markForUnlink(StoreEntry &e) {}
 371
 372     // XXX: This method belongs to Store::Root/StoreController, but it is here
 373     // because test cases use non-StoreController derivatives as Root
 374     /// called when the entry is no longer needed by any transaction
 375     virtual void handleIdleEntry(StoreEntry &e) {}
 376
 377     // XXX: This method belongs to Store::Root/StoreController, but it is here
 378     // because test cases use non-StoreController derivatives as Root
 379     /// called to get rid of no longer needed entry data in RAM, if any
 380     virtual void memoryOut(StoreEntry &e, const bool preserveSwappable) {}
 381
 382     // XXX: This method belongs to Store::Root/StoreController, but it is here
 383     // to avoid casting Root() to StoreController until Root() API is fixed.
 384     /// makes the entry available for collapsing future requests
 385     virtual void allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) {}
 386
 387     // XXX: This method belongs to Store::Root/StoreController, but it is here
 388     // to avoid casting Root() to StoreController until Root() API is fixed.
 389     /// marks the entry completed for collapsed requests
 390     virtual void transientsCompleteWriting(StoreEntry &e) {}
 391
 392     // XXX: This method belongs to Store::Root/StoreController, but it is here
 393     // to avoid casting Root() to StoreController until Root() API is fixed.
 394     /// Update local intransit entry after changes made by appending worker.
 395     virtual void syncCollapsed(const sfileno xitIndex) {}
 396
 397     // XXX: This method belongs to Store::Root/StoreController, but it is here
 398     // to avoid casting Root() to StoreController until Root() API is fixed.
 399     /// calls Root().transients->abandon() if transients are tracked
 400     virtual void transientsAbandon(StoreEntry &e) {}
 401
 402     // XXX: This method belongs to Store::Root/StoreController, but it is here
 403     // to avoid casting Root() to StoreController until Root() API is fixed.
 404     /// number of the transient entry readers some time ago
 405     virtual int transientReaders(const StoreEntry &e) const { return 0; }
 406
 407     // XXX: This method belongs to Store::Root/StoreController, but it is here
 408     // to avoid casting Root() to StoreController until Root() API is fixed.
 409     /// disassociates the entry from the intransit table
 410     virtual void transientsDisconnect(MemObject &mem_obj) {}
 411
 412     // XXX: This method belongs to Store::Root/StoreController, but it is here
 413     // to avoid casting Root() to StoreController until Root() API is fixed.
 414     /// removes the entry from the memory cache
 415     virtual void memoryUnlink(StoreEntry &e) {}
 416
 417     // XXX: This method belongs to Store::Root/StoreController, but it is here
 418     // to avoid casting Root() to StoreController until Root() API is fixed.
 419     /// disassociates the entry from the memory cache, preserving cached data
 420     virtual void memoryDisconnect(StoreEntry &e) {}
 421
 422     /// If the entry is not found, return false. Otherwise, return true after
 423     /// tying the entry to this cache and setting inSync to updateCollapsed().
 424     virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync) { return false; }
 425
 426     /// update a local collapsed entry with fresh info from this cache (if any)
 427     virtual bool updateCollapsed(StoreEntry &collapsed) { return false; }
 428
 429 private:
 430     static RefCount<Store> CurrentRoot;
 431 };
 432
 433 /// \ingroup StoreAPI
 434 typedef RefCount<Store> StorePointer;
 435
 436 /// \ingroup StoreAPI
 437 size_t storeEntryInUse();
 438
 439 /// \ingroup StoreAPI
 440 const char *storeEntryFlags(const StoreEntry *);
 441
 442 /// \ingroup StoreAPI
 443 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 444
 445 /// \ingroup StoreAPI
 446 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 447
 448 /// \ingroup StoreAPI
 449 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 450
 451 /// \ingroup StoreAPI
 452 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 453
 454 /// \ingroup StoreAPI
 455 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 456 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 457
 458 /// \ingroup StoreAPI
 459 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 460 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 461
 462 /// \ingroup StoreAPI
 463 void storeInit(void);
 464
 465 /// \ingroup StoreAPI
 466 void storeConfigure(void);
 467
 468 /// \ingroup StoreAPI
 469 void storeFreeMemory(void);
 470
 471 /// \ingroup StoreAPI
 472 int expiresMoreThan(time_t, time_t);
 473
 474 /// \ingroup StoreAPI
 475 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 476
 477 /// \ingroup StoreAPI
 478 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 479
 480 /// \ingroup StoreAPI
 481 int storeTooManyDiskFilesOpen(void);
 482
 483 class SwapDir;
 484 /// \ingroup StoreAPI
 485 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 486
 487 /// \ingroup StoreAPI
 488 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 489
 490 /// \ingroup StoreAPI
 491 void storeFsInit(void);
 492
 493 /// \ingroup StoreAPI
 494 void storeFsDone(void);
 495
 496 /// \ingroup StoreAPI
 497 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 498
 499 /// \ingroup StoreAPI
 500 extern FREE destroyStoreEntry;
 501
 502 /**
 503  \ingroup StoreAPI
 504  \todo should be a subclass of Packer perhaps ?
 505  */
 506 void packerToStoreInit(Packer * p, StoreEntry * e);
 507
 508 /// \ingroup StoreAPI
 509 void storeGetMemSpace(int size);
 510
 511 #endif /* SQUID_STORE_H */