src/Store.h

   1 /*
   2  *
   3  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   4  * ----------------------------------------------------------
   5  *
   6  *  Squid is the result of efforts by numerous individuals from
   7  *  the Internet community; see the CONTRIBUTORS file for full
   8  *  details.   Many organizations have provided support for Squid's
   9  *  development; see the SPONSORS file for full details.  Squid is
  10  *  Copyrighted (C) 2001 by the Regents of the University of
  11  *  California; see the COPYRIGHT file for full details.  Squid
  12  *  incorporates software developed and/or copyrighted by other
  13  *  sources; see the CREDITS file for full details.
  14  *
  15  *  This program is free software; you can redistribute it and/or modify
  16  *  it under the terms of the GNU General Public License as published by
  17  *  the Free Software Foundation; either version 2 of the License, or
  18  *  (at your option) any later version.
  19  *
  20  *  This program is distributed in the hope that it will be useful,
  21  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  *  GNU General Public License for more details.
  24  *
  25  *  You should have received a copy of the GNU General Public License
  26  *  along with this program; if not, write to the Free Software
  27  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  28  *
  29  */
  30 #ifndef SQUID_STORE_H
  31 #define SQUID_STORE_H
  32
  33 /**
  34  \defgroup StoreAPI  Store API
  35  \ingroup FileSystems
  36  */
  37
  38 #include "base/RefCount.h"
  39 #include "comm/forward.h"
  40 #include "CommRead.h"
  41 #include "hash.h"
  42 #include "http/forward.h"
  43 #include "http/RequestMethod.h"
  44 #include "HttpReply.h"
  45 #include "MemObject.h"
  46 #include "Range.h"
  47 #include "RemovalPolicy.h"
  48 #include "StoreIOBuffer.h"
  49 #include "StoreStats.h"
  50
  51 #if USE_SQUID_ESI
  52 #include "esi/Element.h"
  53 #endif
  54
  55 #include <ostream>
  56
  57 class AsyncCall;
  58 class HttpRequest;
  59 class Packer;
  60 class RequestFlags;
  61 class StoreClient;
  62 class StoreSearch;
  63 class SwapDir;
  64
  65 extern StoreIoStats store_io_stats;
  66
  67 /// maximum number of entries per cache_dir
  68 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  69
  70 /**
  71  \ingroup StoreAPI
  72  */
  73 class StoreEntry : public hash_link
  74 {
  75
  76 public:
  77     static DeferredRead::DeferrableRead DeferReader;
  78     bool checkDeferRead(int fd) const;
  79
  80     virtual const char *getMD5Text() const;
  81     StoreEntry();
  82     virtual ~StoreEntry();
  83
  84     virtual HttpReply const *getReply() const;
  85     virtual void write (StoreIOBuffer);
  86
  87     /** Check if the Store entry is emtpty
  88      * \retval true   Store contains 0 bytes of data.
  89      * \retval false  Store contains 1 or more bytes of data.
  90      * \retval false  Store contains negative content !!!!!!
  91      */
  92     virtual bool isEmpty() const {
  93         assert (mem_obj);
  94         return mem_obj->endOffset() == 0;
  95     }
  96     virtual bool isAccepting() const;
  97     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  98     virtual void complete();
  99     virtual store_client_t storeClientType() const;
 100     virtual char const *getSerialisedMetaData();
 101     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
 102     void startWriting(); ///< pack and write reply headers and, maybe, body
 103     /// whether we may start writing to disk (now or in the future)
 104     virtual bool mayStartSwapOut();
 105     virtual void trimMemory(const bool preserveSwappable);
 106     void abort();
 107     void unlink();
 108     void makePublic();
 109     void makePrivate();
 110     void setPublicKey();
 111     void setPrivateKey();
 112     void expireNow();
 113     void releaseRequest();
 114     void negativeCache();
 115     void cacheNegatively();             /** \todo argh, why both? */
 116     void invokeHandlers();
 117     void purgeMem();
 118     void cacheInMemory(); ///< start or continue storing in memory cache
 119     void swapOut();
 120     /// whether we are in the process of writing this entry to disk
 121     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 122     void swapOutFileClose(int how);
 123     const char *url() const;
 124     int checkCachable();
 125     int checkNegativeHit() const;
 126     int locked() const;
 127     int validToSend() const;
 128     bool memoryCachable() const; ///< may be cached in memory
 129
 130     /// if needed, initialize mem_obj member w/o URI-related information
 131     MemObject *makeMemObject();
 132
 133     /// initialize mem_obj member (if needed) and supply URI-related info
 134     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 135
 136     void dump(int debug_lvl) const;
 137     void hashDelete();
 138     void hashInsert(const cache_key *);
 139     void registerAbort(STABH * cb, void *);
 140     void reset();
 141     void setMemStatus(mem_status_t);
 142     void timestampsSet();
 143     void unregisterAbort();
 144     void destroyMemObject();
 145     int checkTooSmall();
 146
 147     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 148
 149     void setNoDelay (bool const);
 150     bool modifiedSince(HttpRequest * request) const;
 151     /// has ETag matching at least one of the If-Match etags
 152     bool hasIfMatchEtag(const HttpRequest &request) const;
 153     /// has ETag matching at least one of the If-None-Match etags
 154     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 155     /// whether this entry has an ETag; if yes, puts ETag value into parameter
 156     bool hasEtag(ETag &etag) const;
 157
 158     /** What store does this entry belong too ? */
 159     virtual RefCount<SwapDir> store() const;
 160
 161     MemObject *mem_obj;
 162     RemovalPolicyNode repl;
 163     /* START OF ON-DISK STORE_META_STD TLV field */
 164     time_t timestamp;
 165     time_t lastref;
 166     time_t expires;
 167     time_t lastmod;
 168     uint64_t swap_file_sz;
 169     uint16_t refcount;
 170     uint16_t flags;
 171     /* END OF ON-DISK STORE_META_STD */
 172
 173     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 174     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 175
 176     sdirno swap_dirn:7;
 177
 178     mem_status_t mem_status:3;
 179
 180     ping_status_t ping_status:3;
 181
 182     store_status_t store_status:3;
 183
 184     swap_status_t swap_status:3;
 185
 186 public:
 187     static size_t inUseCount();
 188     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 189     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 190     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 191
 192     virtual bool isNull() {
 193         return false;
 194     };
 195
 196     void *operator new(size_t byteCount);
 197     void operator delete(void *address);
 198     void setReleaseFlag();
 199 #if USE_SQUID_ESI
 200
 201     ESIElement::Pointer cachedESITree;
 202 #endif
 203     /** append bytes to the buffer */
 204     virtual void append(char const *, int len);
 205     /** disable sending content to the clients */
 206     virtual void buffer();
 207     /** flush any buffered content */
 208     virtual void flush();
 209     virtual int64_t objectLen() const;
 210     virtual int64_t contentLen() const;
 211
 212     /// claim shared ownership of this entry (for use in a given context)
 213     /// matching lock() and unlock() contexts eases leak triage but is optional
 214     void lock(const char *context);
 215
 216     /// disclaim shared ownership; may remove entry from store and delete it
 217     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 218     int unlock(const char *context);
 219
 220     /// returns a local concurrent use counter, for debugging
 221     int locks() const { return static_cast<int>(lock_count); }
 222
 223     /// update last reference timestamp and related Store metadata
 224     void touch();
 225
 226     virtual void release();
 227
 228 #if USE_ADAPTATION
 229     /// call back producer when more buffer space is available
 230     void deferProducer(const AsyncCall::Pointer &producer);
 231     /// calls back producer registered with deferProducer
 232     void kickProducer();
 233 #endif
 234
 235 private:
 236     static MemAllocator *pool;
 237
 238     unsigned short lock_count;          /* Assume < 65536! */
 239
 240 #if USE_ADAPTATION
 241     /// producer callback registered with deferProducer
 242     AsyncCall::Pointer deferredProducer;
 243 #endif
 244
 245     bool validLength() const;
 246     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 247 };
 248
 249 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 250
 251 /// \ingroup StoreAPI
 252 class NullStoreEntry:public StoreEntry
 253 {
 254
 255 public:
 256     static NullStoreEntry *getInstance();
 257     bool isNull() {
 258         return true;
 259     }
 260
 261     const char *getMD5Text() const;
 262     HttpReply const *getReply() const { return NULL; }
 263     void write (StoreIOBuffer) {}
 264
 265     bool isEmpty () const {return true;}
 266
 267     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const { return aRange.end; }
 268
 269     void operator delete(void *address);
 270     void complete() {}
 271
 272 private:
 273     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 274
 275     char const *getSerialisedMetaData();
 276     bool mayStartSwapout() {return false;}
 277
 278     void trimMemory(const bool preserveSwappable) {}
 279
 280     static NullStoreEntry _instance;
 281 };
 282
 283 /// \ingroup StoreAPI
 284 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 285
 286 /**
 287  \ingroup StoreAPI
 288  * Abstract base class that will replace the whole store and swapdir interface.
 289  */
 290 class Store : public RefCountable
 291 {
 292
 293 public:
 294     /** The root store */
 295     static Store &Root() {
 296         if (CurrentRoot == NULL)
 297             fatal("No Store Root has been set");
 298         return *CurrentRoot;
 299     }
 300     static void Root(Store *);
 301     static void Root(RefCount<Store>);
 302     static void Stats(StoreEntry * output);
 303     static void Maintain(void *unused);
 304
 305     virtual ~Store() {}
 306
 307     /** Handle pending callbacks - called by the event loop. */
 308     virtual int callback() = 0;
 309
 310     /** create the resources needed for this store to operate */
 311     virtual void create();
 312
 313     /**
 314      * Notify this store that its disk is full.
 315      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 316      */
 317     virtual void diskFull();
 318
 319     /** Retrieve a store entry from the store */
 320     virtual StoreEntry * get(const cache_key *) = 0;
 321
 322     /** \todo imeplement the async version */
 323     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 324
 325     /* prepare the store for use. The store need not be usable immediately,
 326      * it should respond to readable() and writable() with true as soon
 327      * as it can provide those services
 328      */
 329     virtual void init() = 0;
 330
 331     /**
 332      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 333      * but may throw estimates for memory etc out of whack.
 334      */
 335     virtual uint64_t maxSize() const = 0;
 336
 337     /** The minimum size the store will shrink to via normal housekeeping */
 338     virtual uint64_t minSize() const = 0;
 339
 340     /** current store size */
 341     virtual uint64_t currentSize() const = 0;
 342
 343     /** the total number of objects stored */
 344     virtual uint64_t currentCount() const = 0;
 345
 346     /** the maximum object size that can be stored, -1 if unlimited */
 347     virtual int64_t maxObjectSize() const = 0;
 348
 349     /// collect cache storage-related statistics
 350     virtual void getStats(StoreInfoStats &stats) const = 0;
 351
 352     /**
 353      * Output stats to the provided store entry.
 354      \todo make these calls asynchronous
 355      */
 356     virtual void stat(StoreEntry &) const = 0;
 357
 358     /** Sync the store prior to shutdown */
 359     virtual void sync();
 360
 361     /** remove a Store entry from the store */
 362     virtual void unlink (StoreEntry &);
 363
 364     /* search in the store */
 365     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 366
 367     /* pulled up from SwapDir for migration.... probably do not belong here */
 368     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 369
 370     /// Undo reference(), returning false iff idle e should be destroyed
 371     virtual bool dereference(StoreEntry &e, bool wantsLocalMemory) = 0;
 372
 373     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 374
 375     // XXX: This method belongs to Store::Root/StoreController, but it is here
 376     // to avoid casting Root() to StoreController until Root() API is fixed.
 377     /// informs stores that this entry will be eventually unlinked
 378     virtual void markForUnlink(StoreEntry &e) {}
 379
 380     // XXX: This method belongs to Store::Root/StoreController, but it is here
 381     // because test cases use non-StoreController derivatives as Root
 382     /// called when the entry is no longer needed by any transaction
 383     virtual void handleIdleEntry(StoreEntry &e) {}
 384
 385     // XXX: This method belongs to Store::Root/StoreController, but it is here
 386     // because test cases use non-StoreController derivatives as Root
 387     /// called to get rid of no longer needed entry data in RAM, if any
 388     virtual void memoryOut(StoreEntry &e, const bool preserveSwappable) {}
 389
 390     // XXX: This method belongs to Store::Root/StoreController, but it is here
 391     // to avoid casting Root() to StoreController until Root() API is fixed.
 392     /// makes the entry available for collapsing future requests
 393     virtual void allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) {}
 394
 395     // XXX: This method belongs to Store::Root/StoreController, but it is here
 396     // to avoid casting Root() to StoreController until Root() API is fixed.
 397     /// marks the entry completed for collapsed requests
 398     virtual void transientsCompleteWriting(StoreEntry &e) {}
 399
 400     // XXX: This method belongs to Store::Root/StoreController, but it is here
 401     // to avoid casting Root() to StoreController until Root() API is fixed.
 402     /// Update local intransit entry after changes made by appending worker.
 403     virtual void syncCollapsed(const sfileno xitIndex) {}
 404
 405     // XXX: This method belongs to Store::Root/StoreController, but it is here
 406     // to avoid casting Root() to StoreController until Root() API is fixed.
 407     /// calls Root().transients->abandon() if transients are tracked
 408     virtual void transientsAbandon(StoreEntry &e) {}
 409
 410     // XXX: This method belongs to Store::Root/StoreController, but it is here
 411     // to avoid casting Root() to StoreController until Root() API is fixed.
 412     /// number of the transient entry readers some time ago
 413     virtual int transientReaders(const StoreEntry &e) const { return 0; }
 414
 415     // XXX: This method belongs to Store::Root/StoreController, but it is here
 416     // to avoid casting Root() to StoreController until Root() API is fixed.
 417     /// disassociates the entry from the intransit table
 418     virtual void transientsDisconnect(MemObject &mem_obj) {}
 419
 420     // XXX: This method belongs to Store::Root/StoreController, but it is here
 421     // to avoid casting Root() to StoreController until Root() API is fixed.
 422     /// removes the entry from the memory cache
 423     virtual void memoryUnlink(StoreEntry &e) {}
 424
 425     // XXX: This method belongs to Store::Root/StoreController, but it is here
 426     // to avoid casting Root() to StoreController until Root() API is fixed.
 427     /// disassociates the entry from the memory cache, preserving cached data
 428     virtual void memoryDisconnect(StoreEntry &e) {}
 429
 430     /// If the entry is not found, return false. Otherwise, return true after
 431     /// tying the entry to this cache and setting inSync to updateCollapsed().
 432     virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync) { return false; }
 433
 434     /// update a local collapsed entry with fresh info from this cache (if any)
 435     virtual bool updateCollapsed(StoreEntry &collapsed) { return false; }
 436
 437 private:
 438     static RefCount<Store> CurrentRoot;
 439 };
 440
 441 /// \ingroup StoreAPI
 442 typedef RefCount<Store> StorePointer;
 443
 444 /// \ingroup StoreAPI
 445 size_t storeEntryInUse();
 446
 447 /// \ingroup StoreAPI
 448 const char *storeEntryFlags(const StoreEntry *);
 449
 450 /// \ingroup StoreAPI
 451 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 452
 453 /// \ingroup StoreAPI
 454 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 455
 456 /// \ingroup StoreAPI
 457 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 458
 459 /// \ingroup StoreAPI
 460 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 461
 462 /// \ingroup StoreAPI
 463 /// Like storeCreatePureEntry(), but also locks the entry and sets entry key.
 464 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 465
 466 /// \ingroup StoreAPI
 467 /// Creates a new StoreEntry with mem_obj and sets initial flags/states.
 468 StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&);
 469
 470 /// \ingroup StoreAPI
 471 void storeInit(void);
 472
 473 /// \ingroup StoreAPI
 474 void storeConfigure(void);
 475
 476 /// \ingroup StoreAPI
 477 void storeFreeMemory(void);
 478
 479 /// \ingroup StoreAPI
 480 int expiresMoreThan(time_t, time_t);
 481
 482 /// \ingroup StoreAPI
 483 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 484
 485 /// \ingroup StoreAPI
 486 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 487
 488 /// \ingroup StoreAPI
 489 int storeTooManyDiskFilesOpen(void);
 490
 491 class SwapDir;
 492 /// \ingroup StoreAPI
 493 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 494
 495 /// \ingroup StoreAPI
 496 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 497
 498 /// \ingroup StoreAPI
 499 void storeFsInit(void);
 500
 501 /// \ingroup StoreAPI
 502 void storeFsDone(void);
 503
 504 /// \ingroup StoreAPI
 505 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 506
 507 /// \ingroup StoreAPI
 508 extern FREE destroyStoreEntry;
 509
 510 /**
 511  \ingroup StoreAPI
 512  \todo should be a subclass of Packer perhaps ?
 513  */
 514 void packerToStoreInit(Packer * p, StoreEntry * e);
 515
 516 /// \ingroup StoreAPI
 517 void storeGetMemSpace(int size);
 518
 519 #endif /* SQUID_STORE_H */