src/Store.h

   1 /*
   2  *
   3  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   4  * ----------------------------------------------------------
   5  *
   6  *  Squid is the result of efforts by numerous individuals from
   7  *  the Internet community; see the CONTRIBUTORS file for full
   8  *  details.   Many organizations have provided support for Squid's
   9  *  development; see the SPONSORS file for full details.  Squid is
  10  *  Copyrighted (C) 2001 by the Regents of the University of
  11  *  California; see the COPYRIGHT file for full details.  Squid
  12  *  incorporates software developed and/or copyrighted by other
  13  *  sources; see the CREDITS file for full details.
  14  *
  15  *  This program is free software; you can redistribute it and/or modify
  16  *  it under the terms of the GNU General Public License as published by
  17  *  the Free Software Foundation; either version 2 of the License, or
  18  *  (at your option) any later version.
  19  *
  20  *  This program is distributed in the hope that it will be useful,
  21  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  *  GNU General Public License for more details.
  24  *
  25  *  You should have received a copy of the GNU General Public License
  26  *  along with this program; if not, write to the Free Software
  27  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  28  *
  29  */
  30 #ifndef SQUID_STORE_H
  31 #define SQUID_STORE_H
  32
  33 /**
  34  \defgroup StoreAPI  Store API
  35  \ingroup FileSystems
  36  */
  37
  38 #include "base/RefCount.h"
  39 #include "comm/forward.h"
  40 #include "CommRead.h"
  41 #include "hash.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequestMethod.h"
  44 #include "MemObject.h"
  45 #include "Range.h"
  46 #include "RemovalPolicy.h"
  47 #include "StoreIOBuffer.h"
  48 #include "StoreStats.h"
  49
  50 #if USE_SQUID_ESI
  51 #include "esi/Element.h"
  52 #endif
  53
  54 #include <ostream>
  55
  56 class AsyncCall;
  57 class HttpRequest;
  58 class Packer;
  59 class RequestFlags;
  60 class StoreClient;
  61 class StoreSearch;
  62 class SwapDir;
  63
  64 extern StoreIoStats store_io_stats;
  65
  66 /// maximum number of entries per cache_dir
  67 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  68
  69 /**
  70  \ingroup StoreAPI
  71  */
  72 class StoreEntry : public hash_link
  73 {
  74
  75 public:
  76     static DeferredRead::DeferrableRead DeferReader;
  77     bool checkDeferRead(int fd) const;
  78
  79     virtual const char *getMD5Text() const;
  80     StoreEntry();
  81     virtual ~StoreEntry();
  82
  83     virtual HttpReply const *getReply() const;
  84     virtual void write (StoreIOBuffer);
  85
  86     /** Check if the Store entry is emtpty
  87      * \retval true   Store contains 0 bytes of data.
  88      * \retval false  Store contains 1 or more bytes of data.
  89      * \retval false  Store contains negative content !!!!!!
  90      */
  91     virtual bool isEmpty() const {
  92         assert (mem_obj);
  93         return mem_obj->endOffset() == 0;
  94     }
  95     virtual bool isAccepting() const;
  96     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  97     virtual void complete();
  98     virtual store_client_t storeClientType() const;
  99     virtual char const *getSerialisedMetaData();
 100     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
 101     void startWriting(); ///< pack and write reply headers and, maybe, body
 102     /// whether we may start writing to disk (now or in the future)
 103     virtual bool mayStartSwapOut();
 104     virtual void trimMemory(const bool preserveSwappable);
 105
 106     // called when a decision to cache in memory has been made
 107     void memOutDecision(const bool willCacheInRam);
 108     // called when a decision to cache on disk has been made
 109     void swapOutDecision(const MemObject::SwapOut::Decision &decision);
 110
 111     void abort();
 112     void unlink();
 113     void makePublic();
 114     void makePrivate();
 115     void setPublicKey();
 116     void setPrivateKey();
 117     void expireNow();
 118     void releaseRequest();
 119     void negativeCache();
 120     void cacheNegatively();             /** \todo argh, why both? */
 121     void invokeHandlers();
 122     void purgeMem();
 123     void cacheInMemory(); ///< start or continue storing in memory cache
 124     void swapOut();
 125     /// whether we are in the process of writing this entry to disk
 126     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 127     void swapOutFileClose(int how);
 128     const char *url() const;
 129     /// Satisfies cachability requirements shared among disk and RAM caches.
 130     /// Encapsulates common checks of mayStartSwapOut() and memoryCachable().
 131     /// TODO: Rename and make private so only those two methods can call this.
 132     bool checkCachable();
 133     int checkNegativeHit() const;
 134     int locked() const;
 135     int validToSend() const;
 136     bool memoryCachable(); ///< checkCachable() and can be cached in memory
 137
 138     /// if needed, initialize mem_obj member w/o URI-related information
 139     MemObject *makeMemObject();
 140
 141     /// initialize mem_obj member (if needed) and supply URI-related info
 142     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 143
 144     void dump(int debug_lvl) const;
 145     void hashDelete();
 146     void hashInsert(const cache_key *);
 147     void registerAbort(STABH * cb, void *);
 148     void reset();
 149     void setMemStatus(mem_status_t);
 150     void timestampsSet();
 151     void unregisterAbort();
 152     void destroyMemObject();
 153     int checkTooSmall();
 154
 155     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 156
 157     void setNoDelay (bool const);
 158     bool modifiedSince(HttpRequest * request) const;
 159     /// has ETag matching at least one of the If-Match etags
 160     bool hasIfMatchEtag(const HttpRequest &request) const;
 161     /// has ETag matching at least one of the If-None-Match etags
 162     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 163     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 164     bool hasEtag(ETag &etag) const;
 165
 166     /** What store does this entry belong too ? */
 167     virtual RefCount<SwapDir> store() const;
 168
 169     MemObject *mem_obj;
 170     RemovalPolicyNode repl;
 171     /* START OF ON-DISK STORE_META_STD TLV field */
 172     time_t timestamp;
 173     time_t lastref;
 174     time_t expires;
 175     time_t lastmod;
 176     uint64_t swap_file_sz;
 177     uint16_t refcount;
 178     uint16_t flags;
 179     /* END OF ON-DISK STORE_META_STD */
 180
 181     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 182     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 183
 184     sdirno swap_dirn:7;
 185
 186     mem_status_t mem_status:3;
 187
 188     ping_status_t ping_status:3;
 189
 190     store_status_t store_status:3;
 191
 192     swap_status_t swap_status:3;
 193
 194 public:
 195     static size_t inUseCount();
 196     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 197     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 198     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 199
 200     virtual bool isNull() {
 201         return false;
 202     };
 203
 204     void *operator new(size_t byteCount);
 205     void operator delete(void *address);
 206     void setReleaseFlag();
 207 #if USE_SQUID_ESI
 208
 209     ESIElement::Pointer cachedESITree;
 210 #endif
 211     /** append bytes to the buffer */
 212     virtual void append(char const *, int len);
 213     /** disable sending content to the clients */
 214     virtual void buffer();
 215     /** flush any buffered content */
 216     virtual void flush();
 217     virtual int64_t objectLen() const;
 218     virtual int64_t contentLen() const;
 219
 220     /// claim shared ownership of this entry (for use in a given context)
 221     /// matching lock() and unlock() contexts eases leak triage but is optional
 222     void lock(const char *context);
 223
 224     /// disclaim shared ownership; may remove entry from store and delete it
 225     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 226     int unlock(const char *context);
 227
 228     /// returns a local concurrent use counter, for debugging
 229     int locks() const { return static_cast<int>(lock_count); }
 230
 231     /// update last reference timestamp and related Store metadata
 232     void touch();
 233
 234     virtual void release();
 235
 236 #if USE_ADAPTATION
 237     /// call back producer when more buffer space is available
 238     void deferProducer(const AsyncCall::Pointer &producer);
 239     /// calls back producer registered with deferProducer
 240     void kickProducer();
 241 #endif
 242
 243 protected:
 244     void transientsAbandonmentCheck();
 245
 246 private:
 247     static MemAllocator *pool;
 248
 249     unsigned short lock_count;          /* Assume < 65536! */
 250
 251 #if USE_ADAPTATION
 252     /// producer callback registered with deferProducer
 253     AsyncCall::Pointer deferredProducer;
 254 #endif
 255
 256     bool validLength() const;
 257     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 258 };
 259
 260 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 261
 262 /// \ingroup StoreAPI
 263 class NullStoreEntry:public StoreEntry
 264 {
 265
 266 public:
 267     static NullStoreEntry *getInstance();
 268     bool isNull() {
 269         return true;
 270     }
 271
 272     const char *getMD5Text() const;
 273     HttpReply const *getReply() const { return NULL; }
 274     void write (StoreIOBuffer) {}
 275
 276     bool isEmpty () const {return true;}
 277
 278     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const { return aRange.end; }
 279
 280     void operator delete(void *address);
 281     void complete() {}
 282
 283 private:
 284     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 285
 286     char const *getSerialisedMetaData();
 287     virtual bool mayStartSwapOut() { return false; }
 288
 289     void trimMemory(const bool preserveSwappable) {}
 290
 291     static NullStoreEntry _instance;
 292 };
 293
 294 /// \ingroup StoreAPI
 295 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 296
 297 /**
 298  \ingroup StoreAPI
 299  * Abstract base class that will replace the whole store and swapdir interface.
 300  */
 301 class Store : public RefCountable
 302 {
 303
 304 public:
 305     /** The root store */
 306     static Store &Root() {
 307         if (CurrentRoot == NULL)
 308             fatal("No Store Root has been set");
 309         return *CurrentRoot;
 310     }
 311     static void Root(Store *);
 312     static void Root(RefCount<Store>);
 313     static void Stats(StoreEntry * output);
 314     static void Maintain(void *unused);
 315
 316     virtual ~Store() {}
 317
 318     /** Handle pending callbacks - called by the event loop. */
 319     virtual int callback() = 0;
 320
 321     /** create the resources needed for this store to operate */
 322     virtual void create();
 323
 324     /**
 325      * Notify this store that its disk is full.
 326      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 327      */
 328     virtual void diskFull();
 329
 330     /** Retrieve a store entry from the store */
 331     virtual StoreEntry * get(const cache_key *) = 0;
 332
 333     /** \todo imeplement the async version */
 334     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 335
 336     /* prepare the store for use. The store need not be usable immediately,
 337      * it should respond to readable() and writable() with true as soon
 338      * as it can provide those services
 339      */
 340     virtual void init() = 0;
 341
 342     /**
 343      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 344      * but may throw estimates for memory etc out of whack.
 345      */
 346     virtual uint64_t maxSize() const = 0;
 347
 348     /** The minimum size the store will shrink to via normal housekeeping */
 349     virtual uint64_t minSize() const = 0;
 350
 351     /** current store size */
 352     virtual uint64_t currentSize() const = 0;
 353
 354     /** the total number of objects stored */
 355     virtual uint64_t currentCount() const = 0;
 356
 357     /** the maximum object size that can be stored, -1 if unlimited */
 358     virtual int64_t maxObjectSize() const = 0;
 359
 360     /// collect cache storage-related statistics
 361     virtual void getStats(StoreInfoStats &stats) const = 0;
 362
 363     /**
 364      * Output stats to the provided store entry.
 365      \todo make these calls asynchronous
 366      */
 367     virtual void stat(StoreEntry &) const = 0;
 368
 369     /** Sync the store prior to shutdown */
 370     virtual void sync();
 371
 372     /** remove a Store entry from the store */
 373     virtual void unlink (StoreEntry &);
 374
 375     /* search in the store */
 376     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 377
 378     /* pulled up from SwapDir for migration.... probably do not belong here */
 379     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 380
 381     /// Undo reference(), returning false iff idle e should be destroyed
 382     virtual bool dereference(StoreEntry &e, bool wantsLocalMemory) = 0;
 383
 384     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 385
 386     // XXX: This method belongs to Store::Root/StoreController, but it is here
 387     // to avoid casting Root() to StoreController until Root() API is fixed.
 388     /// informs stores that this entry will be eventually unlinked
 389     virtual void markForUnlink(StoreEntry &e) {}
 390
 391     // XXX: This method belongs to Store::Root/StoreController, but it is here
 392     // because test cases use non-StoreController derivatives as Root
 393     /// called when the entry is no longer needed by any transaction
 394     virtual void handleIdleEntry(StoreEntry &e) {}
 395
 396     // XXX: This method belongs to Store::Root/StoreController, but it is here
 397     // because test cases use non-StoreController derivatives as Root
 398     /// called to get rid of no longer needed entry data in RAM, if any
 399     virtual void memoryOut(StoreEntry &e, const bool preserveSwappable) {}
 400
 401     // XXX: This method belongs to Store::Root/StoreController, but it is here
 402     // to avoid casting Root() to StoreController until Root() API is fixed.
 403     /// makes the entry available for collapsing future requests
 404     virtual void allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) {}
 405
 406     // XXX: This method belongs to Store::Root/StoreController, but it is here
 407     // to avoid casting Root() to StoreController until Root() API is fixed.
 408     /// marks the entry completed for collapsed requests
 409     virtual void transientsCompleteWriting(StoreEntry &e) {}
 410
 411     // XXX: This method belongs to Store::Root/StoreController, but it is here
 412     // to avoid casting Root() to StoreController until Root() API is fixed.
 413     /// Update local intransit entry after changes made by appending worker.
 414     virtual void syncCollapsed(const sfileno xitIndex) {}
 415
 416     // XXX: This method belongs to Store::Root/StoreController, but it is here
 417     // to avoid casting Root() to StoreController until Root() API is fixed.
 418     /// calls Root().transients->abandon() if transients are tracked
 419     virtual void transientsAbandon(StoreEntry &e) {}
 420
 421     // XXX: This method belongs to Store::Root/StoreController, but it is here
 422     // to avoid casting Root() to StoreController until Root() API is fixed.
 423     /// number of the transient entry readers some time ago
 424     virtual int transientReaders(const StoreEntry &e) const { return 0; }
 425
 426     // XXX: This method belongs to Store::Root/StoreController, but it is here
 427     // to avoid casting Root() to StoreController until Root() API is fixed.
 428     /// disassociates the entry from the intransit table
 429     virtual void transientsDisconnect(MemObject &mem_obj) {}
 430
 431     // XXX: This method belongs to Store::Root/StoreController, but it is here
 432     // to avoid casting Root() to StoreController until Root() API is fixed.
 433     /// removes the entry from the memory cache
 434     virtual void memoryUnlink(StoreEntry &e) {}
 435
 436     // XXX: This method belongs to Store::Root/StoreController, but it is here
 437     // to avoid casting Root() to StoreController until Root() API is fixed.
 438     /// disassociates the entry from the memory cache, preserving cached data
 439     virtual void memoryDisconnect(StoreEntry &e) {}
 440
 441     /// If the entry is not found, return false. Otherwise, return true after
 442     /// tying the entry to this cache and setting inSync to updateCollapsed().
 443     virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync) { return false; }
 444
 445     /// update a local collapsed entry with fresh info from this cache (if any)
 446     virtual bool updateCollapsed(StoreEntry &collapsed) { return false; }
 447
 448 private:
 449     static RefCount<Store> CurrentRoot;
 450 };
 451
 452 /// \ingroup StoreAPI
 453 typedef RefCount<Store> StorePointer;
 454
 455 /// \ingroup StoreAPI
 456 size_t storeEntryInUse();
 457
 458 /// \ingroup StoreAPI
 459 const char *storeEntryFlags(const StoreEntry *);
 460
 461 /// \ingroup StoreAPI
 462 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 463
 464 /// \ingroup StoreAPI
 465 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 466
 467 /// \ingroup StoreAPI
 468 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 469
 470 /// \ingroup StoreAPI
 471 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 472
 473 /// \ingroup StoreAPI
 474 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 475 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 476
 477 /// \ingroup StoreAPI
 478 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 479 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 480
 481 /// \ingroup StoreAPI
 482 void storeInit(void);
 483
 484 /// \ingroup StoreAPI
 485 void storeConfigure(void);
 486
 487 /// \ingroup StoreAPI
 488 void storeFreeMemory(void);
 489
 490 /// \ingroup StoreAPI
 491 int expiresMoreThan(time_t, time_t);
 492
 493 /// \ingroup StoreAPI
 494 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 495
 496 /// \ingroup StoreAPI
 497 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 498
 499 /// \ingroup StoreAPI
 500 int storeTooManyDiskFilesOpen(void);
 501
 502 class SwapDir;
 503 /// \ingroup StoreAPI
 504 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 505
 506 /// \ingroup StoreAPI
 507 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 508
 509 /// \ingroup StoreAPI
 510 void storeFsInit(void);
 511
 512 /// \ingroup StoreAPI
 513 void storeFsDone(void);
 514
 515 /// \ingroup StoreAPI
 516 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 517
 518 /// \ingroup StoreAPI
 519 extern FREE destroyStoreEntry;
 520
 521 /**
 522  \ingroup StoreAPI
 523  \todo should be a subclass of Packer perhaps ?
 524  */
 525 void packerToStoreInit(Packer * p, StoreEntry * e);
 526
 527 /// \ingroup StoreAPI
 528 void storeGetMemSpace(int size);
 529
 530 #endif /* SQUID_STORE_H */