src/Store.h

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 #ifndef SQUID_STORE_H
  10 #define SQUID_STORE_H
  11
  12 /**
  13  \defgroup StoreAPI  Store API
  14  \ingroup FileSystems
  15  */
  16
  17 #include "base/RefCount.h"
  18 #include "comm/forward.h"
  19 #include "CommRead.h"
  20 #include "hash.h"
  21 #include "http/forward.h"
  22 #include "http/RequestMethod.h"
  23 #include "HttpReply.h"
  24 #include "MemObject.h"
  25 #include "Range.h"
  26 #include "RemovalPolicy.h"
  27 #include "StoreIOBuffer.h"
  28 #include "StoreStats.h"
  29
  30 #if USE_SQUID_ESI
  31 #include "esi/Element.h"
  32 #endif
  33
  34 #include <ostream>
  35
  36 class AsyncCall;
  37 class HttpRequest;
  38 class Packer;
  39 class RequestFlags;
  40 class StoreClient;
  41 class StoreSearch;
  42 class SwapDir;
  43
  44 extern StoreIoStats store_io_stats;
  45
  46 /// maximum number of entries per cache_dir
  47 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  48
  49 /**
  50  \ingroup StoreAPI
  51  */
  52 class StoreEntry : public hash_link
  53 {
  54
  55 public:
  56     static DeferredRead::DeferrableRead DeferReader;
  57     bool checkDeferRead(int fd) const;
  58
  59     virtual const char *getMD5Text() const;
  60     StoreEntry();
  61     virtual ~StoreEntry();
  62
  63     virtual HttpReply const *getReply() const;
  64     virtual void write (StoreIOBuffer);
  65
  66     /** Check if the Store entry is emtpty
  67      * \retval true   Store contains 0 bytes of data.
  68      * \retval false  Store contains 1 or more bytes of data.
  69      * \retval false  Store contains negative content !!!!!!
  70      */
  71     virtual bool isEmpty() const {
  72         assert (mem_obj);
  73         return mem_obj->endOffset() == 0;
  74     }
  75     virtual bool isAccepting() const;
  76     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  77     virtual void complete();
  78     virtual store_client_t storeClientType() const;
  79     virtual char const *getSerialisedMetaData();
  80     /// Store a prepared error response. MemObject locks the reply object.
  81     void storeErrorResponse(HttpReply *reply);
  82     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  83     void startWriting(); ///< pack and write reply headers and, maybe, body
  84     /// whether we may start writing to disk (now or in the future)
  85     virtual bool mayStartSwapOut();
  86     virtual void trimMemory(const bool preserveSwappable);
  87
  88     // called when a decision to cache in memory has been made
  89     void memOutDecision(const bool willCacheInRam);
  90     // called when a decision to cache on disk has been made
  91     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
  92
  93     void abort();
  94     void unlink();
  95     void makePublic();
  96     void makePrivate();
  97     void setPublicKey();
  98     void setPrivateKey();
  99     void expireNow();
 100     void releaseRequest();
 101     void negativeCache();
 102     void cacheNegatively();     /** \todo argh, why both? */
 103     void invokeHandlers();
 104     void purgeMem();
 105     void cacheInMemory(); ///< start or continue storing in memory cache
 106     void swapOut();
 107     /// whether we are in the process of writing this entry to disk
 108     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 109     void swapOutFileClose(int how);
 110     const char *url() const;
 111     /// Satisfies cachability requirements shared among disk and RAM caches.
 112     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 113     /// TODO: Rename and make private so only those two methods can call this.
 114     bool checkCachable();
 115     int checkNegativeHit() const;
 116     int locked() const;
 117     int validToSend() const;
 118     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 119
 120     /// if needed, initialize mem_obj member w/o URI-related information
 121     MemObject *makeMemObject();
 122
 123     /// initialize mem_obj member (if needed) and supply URI-related info
 124     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 125
 126     void dump(int debug_lvl) const;
 127     void hashDelete();
 128     void hashInsert(const cache_key *);
 129     void registerAbort(STABH * cb, void *);
 130     void reset();
 131     void setMemStatus(mem_status_t);
 132     void timestampsSet();
 133     void unregisterAbort();
 134     void destroyMemObject();
 135     int checkTooSmall();
 136
 137     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 138
 139     void setNoDelay (bool const);
 140     bool modifiedSince(HttpRequest * request) const;
 141     /// has ETag matching at least one of the If-Match etags
 142     bool hasIfMatchEtag(const HttpRequest &request) const;
 143     /// has ETag matching at least one of the If-None-Match etags
 144     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 145     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 146     bool hasEtag(ETag &etag) const;
 147
 148     /** What store does this entry belong too ? */
 149     virtual RefCount<SwapDir> store() const;
 150
 151     MemObject *mem_obj;
 152     RemovalPolicyNode repl;
 153     /* START OF ON-DISK STORE_META_STD TLV field */
 154     time_t timestamp;
 155     time_t lastref;
 156     time_t expires;
 157     time_t lastmod;
 158     uint64_t swap_file_sz;
 159     uint16_t refcount;
 160     uint16_t flags;
 161     /* END OF ON-DISK STORE_META_STD */
 162
 163     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 164     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 165
 166     sdirno swap_dirn:7;
 167
 168     mem_status_t mem_status:3;
 169
 170     ping_status_t ping_status:3;
 171
 172     store_status_t store_status:3;
 173
 174     swap_status_t swap_status:3;
 175
 176 public:
 177     static size_t inUseCount();
 178     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 179     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 180     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 181
 182     virtual bool isNull() {
 183         return false;
 184     };
 185
 186     void *operator new(size_t byteCount);
 187     void operator delete(void *address);
 188     void setReleaseFlag();
 189 #if USE_SQUID_ESI
 190
 191     ESIElement::Pointer cachedESITree;
 192 #endif
 193     /** append bytes to the buffer */
 194     virtual void append(char const *, int len);
 195     /** disable sending content to the clients */
 196     virtual void buffer();
 197     /** flush any buffered content */
 198     virtual void flush();
 199     virtual int64_t objectLen() const;
 200     virtual int64_t contentLen() const;
 201
 202     /// claim shared ownership of this entry (for use in a given context)
 203     /// matching lock() and unlock() contexts eases leak triage but is optional
 204     void lock(const char *context);
 205
 206     /// disclaim shared ownership; may remove entry from store and delete it
 207     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 208     int unlock(const char *context);
 209
 210     /// returns a local concurrent use counter, for debugging
 211     int locks() const { return static_cast<int>(lock_count); }
 212
 213     /// update last reference timestamp and related Store metadata
 214     void touch();
 215
 216     virtual void release();
 217
 218 #if USE_ADAPTATION
 219     /// call back producer when more buffer space is available
 220     void deferProducer(const AsyncCall::Pointer &producer);
 221     /// calls back producer registered with deferProducer
 222     void kickProducer();
 223 #endif
 224
 225 protected:
 226     void transientsAbandonmentCheck();
 227
 228 private:
 229     bool checkTooBig() const;
 230
 231     static MemAllocator *pool;
 232
 233     unsigned short lock_count;      /* Assume < 65536! */
 234
 235 #if USE_ADAPTATION
 236     /// producer callback registered with deferProducer
 237     AsyncCall::Pointer deferredProducer;
 238 #endif
 239
 240     bool validLength() const;
 241     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 242 };
 243
 244 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 245
 246 /// \ingroup StoreAPI
 247 class NullStoreEntry:public StoreEntry
 248 {
 249
 250 public:
 251     static NullStoreEntry *getInstance();
 252     bool isNull() {
 253         return true;
 254     }
 255
 256     const char *getMD5Text() const;
 257     HttpReply const *getReply() const { return NULL; }
 258     void write (StoreIOBuffer) {}
 259
 260     bool isEmpty () const {return true;}
 261
 262     virtual size_t bytesWanted(Range<size_t> const aRange, bool) const { return aRange.end; }
 263
 264     void operator delete(void *address);
 265     void complete() {}
 266
 267 private:
 268     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 269
 270     char const *getSerialisedMetaData();
 271     virtual bool mayStartSwapOut() { return false; }
 272
 273     void trimMemory(const bool) {}
 274
 275     static NullStoreEntry _instance;
 276 };
 277
 278 /// \ingroup StoreAPI
 279 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 280
 281 /**
 282  \ingroup StoreAPI
 283  * Abstract base class that will replace the whole store and swapdir interface.
 284  */
 285 class Store : public RefCountable
 286 {
 287
 288 public:
 289     /** The root store */
 290     static Store &Root() {
 291         if (CurrentRoot == NULL)
 292             fatal("No Store Root has been set");
 293         return *CurrentRoot;
 294     }
 295     static void Root(Store *);
 296     static void Root(RefCount<Store>);
 297     static void Stats(StoreEntry * output);
 298     static void Maintain(void *unused);
 299
 300     virtual ~Store() {}
 301
 302     /** Handle pending callbacks - called by the event loop. */
 303     virtual int callback() = 0;
 304
 305     /** create the resources needed for this store to operate */
 306     virtual void create();
 307
 308     /**
 309      * Notify this store that its disk is full.
 310      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 311      */
 312     virtual void diskFull();
 313
 314     /** Retrieve a store entry from the store */
 315     virtual StoreEntry * get(const cache_key *) = 0;
 316
 317     /** \todo imeplement the async version */
 318     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 319
 320     /* prepare the store for use. The store need not be usable immediately,
 321      * it should respond to readable() and writable() with true as soon
 322      * as it can provide those services
 323      */
 324     virtual void init() = 0;
 325
 326     /**
 327      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 328      * but may throw estimates for memory etc out of whack.
 329      */
 330     virtual uint64_t maxSize() const = 0;
 331
 332     /** The minimum size the store will shrink to via normal housekeeping */
 333     virtual uint64_t minSize() const = 0;
 334
 335     /** current store size */
 336     virtual uint64_t currentSize() const = 0;
 337
 338     /** the total number of objects stored */
 339     virtual uint64_t currentCount() const = 0;
 340
 341     /** the maximum object size that can be stored, -1 if unlimited */
 342     virtual int64_t maxObjectSize() const = 0;
 343
 344     /// collect cache storage-related statistics
 345     virtual void getStats(StoreInfoStats &stats) const = 0;
 346
 347     /**
 348      * Output stats to the provided store entry.
 349      \todo make these calls asynchronous
 350      */
 351     virtual void stat(StoreEntry &) const = 0;
 352
 353     /** Sync the store prior to shutdown */
 354     virtual void sync();
 355
 356     /** remove a Store entry from the store */
 357     virtual void unlink (StoreEntry &);
 358
 359     /* search in the store */
 360     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 361
 362     /* pulled up from SwapDir for migration.... probably do not belong here */
 363     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 364
 365     /// Undo reference(), returning false iff idle e should be destroyed
 366     virtual bool dereference(StoreEntry &, bool wantsLocalMemory) = 0;
 367
 368     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 369
 370     // XXX: This method belongs to Store::Root/StoreController, but it is here
 371     // to avoid casting Root() to StoreController until Root() API is fixed.
 372     /// informs stores that this entry will be eventually unlinked
 373     virtual void markForUnlink(StoreEntry &) {}
 374
 375     // XXX: This method belongs to Store::Root/StoreController, but it is here
 376     // because test cases use non-StoreController derivatives as Root
 377     /// called when the entry is no longer needed by any transaction
 378     virtual void handleIdleEntry(StoreEntry &) {}
 379
 380     // XXX: This method belongs to Store::Root/StoreController, but it is here
 381     // because test cases use non-StoreController derivatives as Root
 382     /// called to get rid of no longer needed entry data in RAM, if any
 383     virtual void memoryOut(StoreEntry &, const bool /*preserveSwappable*/) {}
 384
 385     // XXX: This method belongs to Store::Root/StoreController, but it is here
 386     // to avoid casting Root() to StoreController until Root() API is fixed.
 387     /// makes the entry available for collapsing future requests
 388     virtual void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &) {}
 389
 390     // XXX: This method belongs to Store::Root/StoreController, but it is here
 391     // to avoid casting Root() to StoreController until Root() API is fixed.
 392     /// marks the entry completed for collapsed requests
 393     virtual void transientsCompleteWriting(StoreEntry &) {}
 394
 395     // XXX: This method belongs to Store::Root/StoreController, but it is here
 396     // to avoid casting Root() to StoreController until Root() API is fixed.
 397     /// Update local intransit entry after changes made by appending worker.
 398     virtual void syncCollapsed(const sfileno) {}
 399
 400     // XXX: This method belongs to Store::Root/StoreController, but it is here
 401     // to avoid casting Root() to StoreController until Root() API is fixed.
 402     /// calls Root().transients->abandon() if transients are tracked
 403     virtual void transientsAbandon(StoreEntry &) {}
 404
 405     // XXX: This method belongs to Store::Root/StoreController, but it is here
 406     // to avoid casting Root() to StoreController until Root() API is fixed.
 407     /// number of the transient entry readers some time ago
 408     virtual int transientReaders(const StoreEntry &) const { return 0; }
 409
 410     // XXX: This method belongs to Store::Root/StoreController, but it is here
 411     // to avoid casting Root() to StoreController until Root() API is fixed.
 412     /// disassociates the entry from the intransit table
 413     virtual void transientsDisconnect(MemObject &) {}
 414
 415     // XXX: This method belongs to Store::Root/StoreController, but it is here
 416     // to avoid casting Root() to StoreController until Root() API is fixed.
 417     /// removes the entry from the memory cache
 418     virtual void memoryUnlink(StoreEntry &) {}
 419
 420     // XXX: This method belongs to Store::Root/StoreController, but it is here
 421     // to avoid casting Root() to StoreController until Root() API is fixed.
 422     /// disassociates the entry from the memory cache, preserving cached data
 423     virtual void memoryDisconnect(StoreEntry &) {}
 424
 425     /// If the entry is not found, return false. Otherwise, return true after
 426     /// tying the entry to this cache and setting inSync to updateCollapsed().
 427     virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; }
 428
 429     /// update a local collapsed entry with fresh info from this cache (if any)
 430     virtual bool updateCollapsed(StoreEntry &) { return false; }
 431
 432 private:
 433     static RefCount<Store> CurrentRoot;
 434 };
 435
 436 /// \ingroup StoreAPI
 437 typedef RefCount<Store> StorePointer;
 438
 439 /// \ingroup StoreAPI
 440 size_t storeEntryInUse();
 441
 442 /// \ingroup StoreAPI
 443 const char *storeEntryFlags(const StoreEntry *);
 444
 445 /// \ingroup StoreAPI
 446 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 447
 448 /// \ingroup StoreAPI
 449 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 450
 451 /// \ingroup StoreAPI
 452 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 453
 454 /// \ingroup StoreAPI
 455 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 456
 457 /// \ingroup StoreAPI
 458 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 459 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 460
 461 /// \ingroup StoreAPI
 462 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 463 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 464
 465 /// \ingroup StoreAPI
 466 void storeInit(void);
 467
 468 /// \ingroup StoreAPI
 469 void storeConfigure(void);
 470
 471 /// \ingroup StoreAPI
 472 void storeFreeMemory(void);
 473
 474 /// \ingroup StoreAPI
 475 int expiresMoreThan(time_t, time_t);
 476
 477 /// \ingroup StoreAPI
 478 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 479
 480 /// \ingroup StoreAPI
 481 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 482
 483 /// \ingroup StoreAPI
 484 int storeTooManyDiskFilesOpen(void);
 485
 486 class SwapDir;
 487 /// \ingroup StoreAPI
 488 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 489
 490 /// \ingroup StoreAPI
 491 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 492
 493 /// \ingroup StoreAPI
 494 void storeFsInit(void);
 495
 496 /// \ingroup StoreAPI
 497 void storeFsDone(void);
 498
 499 /// \ingroup StoreAPI
 500 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 501
 502 /// \ingroup StoreAPI
 503 extern FREE destroyStoreEntry;
 504
 505 /**
 506  \ingroup StoreAPI
 507  \todo should be a subclass of Packer perhaps ?
 508  */
 509 void packerToStoreInit(Packer * p, StoreEntry * e);
 510
 511 /// \ingroup StoreAPI
 512 void storeGetMemSpace(int size);
 513
 514 #endif /* SQUID_STORE_H */
 515