src/Store.h

   1 /*
   2  *
   3  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   4  * ----------------------------------------------------------
   5  *
   6  *  Squid is the result of efforts by numerous individuals from
   7  *  the Internet community; see the CONTRIBUTORS file for full
   8  *  details.   Many organizations have provided support for Squid's
   9  *  development; see the SPONSORS file for full details.  Squid is
  10  *  Copyrighted (C) 2001 by the Regents of the University of
  11  *  California; see the COPYRIGHT file for full details.  Squid
  12  *  incorporates software developed and/or copyrighted by other
  13  *  sources; see the CREDITS file for full details.
  14  *
  15  *  This program is free software; you can redistribute it and/or modify
  16  *  it under the terms of the GNU General Public License as published by
  17  *  the Free Software Foundation; either version 2 of the License, or
  18  *  (at your option) any later version.
  19  *
  20  *  This program is distributed in the hope that it will be useful,
  21  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  *  GNU General Public License for more details.
  24  *
  25  *  You should have received a copy of the GNU General Public License
  26  *  along with this program; if not, write to the Free Software
  27  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  28  *
  29  */
  30 #ifndef SQUID_STORE_H
  31 #define SQUID_STORE_H
  32
  33 /**
  34  \defgroup StoreAPI  Store API
  35  \ingroup FileSystems
  36  */
  37
  38 #include "base/RefCount.h"
  39 #include "comm/forward.h"
  40 #include "CommRead.h"
  41 #include "hash.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequestMethod.h"
  44 #include "Range.h"
  45 #include "RemovalPolicy.h"
  46 #include "StoreIOBuffer.h"
  47 #include "StoreStats.h"
  48
  49 #if USE_SQUID_ESI
  50 #include "esi/Element.h"
  51 #endif
  52
  53 #if HAVE_OSTREAM
  54 #include <ostream>
  55 #endif
  56
  57 class AsyncCall;
  58 class HttpRequest;
  59 class MemObject;
  60 class Packer;
  61 class RequestFlags;
  62 class StoreClient;
  63 class StoreSearch;
  64 class SwapDir;
  65
  66 extern StoreIoStats store_io_stats;
  67
  68 /// maximum number of entries per cache_dir
  69 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  70
  71 /**
  72  \ingroup StoreAPI
  73  */
  74 class StoreEntry : public hash_link
  75 {
  76
  77 public:
  78     static DeferredRead::DeferrableRead DeferReader;
  79     bool checkDeferRead(int fd) const;
  80
  81     virtual const char *getMD5Text() const;
  82     StoreEntry();
  83     virtual ~StoreEntry();
  84
  85     virtual HttpReply const *getReply() const;
  86     virtual void write (StoreIOBuffer);
  87     virtual _SQUID_INLINE_ bool isEmpty() const;
  88     virtual bool isAccepting() const;
  89     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  90     virtual void complete();
  91     virtual store_client_t storeClientType() const;
  92     virtual char const *getSerialisedMetaData();
  93     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  94     void startWriting(); ///< pack and write reply headers and, maybe, body
  95     /// whether we may start writing to disk (now or in the future)
  96     virtual bool mayStartSwapOut();
  97     virtual void trimMemory(const bool preserveSwappable);
  98     void abort();
  99     void unlink();
 100     void makePublic();
 101     void makePrivate();
 102     void setPublicKey();
 103     void setPrivateKey();
 104     void expireNow();
 105     void releaseRequest();
 106     void negativeCache();
 107     void cacheNegatively();             /** \todo argh, why both? */
 108     void invokeHandlers();
 109     void purgeMem();
 110     void cacheInMemory(); ///< start or continue storing in memory cache
 111     void swapOut();
 112     /// whether we are in the process of writing this entry to disk
 113     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 114     void swapOutFileClose(int how);
 115     const char *url() const;
 116     int checkCachable();
 117     int checkNegativeHit() const;
 118     int locked() const;
 119     int validToSend() const;
 120     bool memoryCachable() const; ///< may be cached in memory
 121
 122     /// if needed, initialize mem_obj member w/o URI-related information
 123     MemObject *makeMemObject();
 124
 125     /// initialize mem_obj member (if needed) and supply URI-related info
 126     void createMemObject(const char *storeId, const char *logUri, const HttpRequestMethod &aMethod);
 127
 128     void dump(int debug_lvl) const;
 129     void hashDelete();
 130     void hashInsert(const cache_key *);
 131     void registerAbort(STABH * cb, void *);
 132     void reset();
 133     void setMemStatus(mem_status_t);
 134     void timestampsSet();
 135     void unregisterAbort();
 136     void destroyMemObject();
 137     int checkTooSmall();
 138
 139     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 140
 141     void setNoDelay (bool const);
 142     bool modifiedSince(HttpRequest * request) const;
 143     /// has ETag matching at least one of the If-Match etags
 144     bool hasIfMatchEtag(const HttpRequest &request) const;
 145     /// has ETag matching at least one of the If-None-Match etags
 146     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 147
 148     /** What store does this entry belong too ? */
 149     virtual RefCount<SwapDir> store() const;
 150
 151     MemObject *mem_obj;
 152     RemovalPolicyNode repl;
 153     /* START OF ON-DISK STORE_META_STD TLV field */
 154     time_t timestamp;
 155     time_t lastref;
 156     time_t expires;
 157     time_t lastmod;
 158     uint64_t swap_file_sz;
 159     uint16_t refcount;
 160     uint16_t flags;
 161     /* END OF ON-DISK STORE_META_STD */
 162
 163     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 164     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 165
 166     sdirno swap_dirn:7;
 167
 168     unsigned short lock_count;          /* Assume < 65536! */
 169
 170     mem_status_t mem_status:3;
 171
 172     ping_status_t ping_status:3;
 173
 174     store_status_t store_status:3;
 175
 176     swap_status_t swap_status:3;
 177
 178 public:
 179     static size_t inUseCount();
 180     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 181     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 182     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 183
 184     virtual bool isNull() {
 185         return false;
 186     };
 187
 188     void *operator new(size_t byteCount);
 189     void operator delete(void *address);
 190     void setReleaseFlag();
 191 #if USE_SQUID_ESI
 192
 193     ESIElement::Pointer cachedESITree;
 194 #endif
 195     /** append bytes to the buffer */
 196     virtual void append(char const *, int len);
 197     /** disable sending content to the clients */
 198     virtual void buffer();
 199     /** flush any buffered content */
 200     virtual void flush();
 201     virtual int64_t objectLen() const;
 202     virtual int64_t contentLen() const;
 203
 204     /** deprecated: lock() in anonymous context folowed by touch()
 205      * RBC 20050104 this is wrong- memory ref counting
 206      * is not at all equivalent to the store 'usage' concept
 207      * which the replacement policies should be acting upon.
 208      * specifically, object iteration within stores needs
 209      * memory ref counting to prevent race conditions,
 210      * but this should not influence store replacement.
 211      */
 212     void lock() { lock("somebody"); touch(); }
 213
 214     /// claim shared ownership of this entry (for use in a given context)
 215     void lock(const char *context);
 216     /// disclaim shared ownership; may remove entry from store and delete it
 217     /// returns remaning lock level (zero for unlocked and possibly gone entry)
 218     int unlock(const char *context = "somebody");
 219     /// update last reference timestamp and related Store metadata
 220     void touch();
 221
 222     virtual void release();
 223
 224 #if USE_ADAPTATION
 225     /// call back producer when more buffer space is available
 226     void deferProducer(const AsyncCall::Pointer &producer);
 227     /// calls back producer registered with deferProducer
 228     void kickProducer();
 229 #endif
 230
 231 private:
 232     static MemAllocator *pool;
 233
 234 #if USE_ADAPTATION
 235     /// producer callback registered with deferProducer
 236     AsyncCall::Pointer deferredProducer;
 237 #endif
 238
 239     bool validLength() const;
 240     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 241 };
 242
 243 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 244
 245 /// \ingroup StoreAPI
 246 class NullStoreEntry:public StoreEntry
 247 {
 248
 249 public:
 250     static NullStoreEntry *getInstance();
 251     bool isNull() {
 252         return true;
 253     }
 254
 255     const char *getMD5Text() const;
 256     _SQUID_INLINE_ HttpReply const *getReply() const;
 257     void write (StoreIOBuffer) {}
 258
 259     bool isEmpty () const {return true;}
 260
 261     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const { return aRange.end; }
 262
 263     void operator delete(void *address);
 264     void complete() {}
 265
 266 private:
 267     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 268
 269     char const *getSerialisedMetaData();
 270     bool mayStartSwapout() {return false;}
 271
 272     void trimMemory(const bool preserveSwappable) {}
 273
 274     static NullStoreEntry _instance;
 275 };
 276
 277 /// \ingroup StoreAPI
 278 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 279
 280 /**
 281  \ingroup StoreAPI
 282  * Abstract base class that will replace the whole store and swapdir interface.
 283  */
 284 class Store : public RefCountable
 285 {
 286
 287 public:
 288     /** The root store */
 289     static _SQUID_INLINE_ Store &Root();
 290     static void Root(Store *);
 291     static void Root(RefCount<Store>);
 292     static void Stats(StoreEntry * output);
 293     static void Maintain(void *unused);
 294
 295     virtual ~Store() {}
 296
 297     /** Handle pending callbacks - called by the event loop. */
 298     virtual int callback() = 0;
 299
 300     /** create the resources needed for this store to operate */
 301     virtual void create();
 302
 303     /**
 304      * Notify this store that its disk is full.
 305      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 306      */
 307     virtual void diskFull();
 308
 309     /** Retrieve a store entry from the store */
 310     virtual StoreEntry * get(const cache_key *) = 0;
 311
 312     /** \todo imeplement the async version */
 313     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 314
 315     /* prepare the store for use. The store need not be usable immediately,
 316      * it should respond to readable() and writable() with true as soon
 317      * as it can provide those services
 318      */
 319     virtual void init() = 0;
 320
 321     /**
 322      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 323      * but may throw estimates for memory etc out of whack.
 324      */
 325     virtual uint64_t maxSize() const = 0;
 326
 327     /** The minimum size the store will shrink to via normal housekeeping */
 328     virtual uint64_t minSize() const = 0;
 329
 330     /** current store size */
 331     virtual uint64_t currentSize() const = 0;
 332
 333     /** the total number of objects stored */
 334     virtual uint64_t currentCount() const = 0;
 335
 336     /** the maximum object size that can be stored, -1 if unlimited */
 337     virtual int64_t maxObjectSize() const = 0;
 338
 339     /// collect cache storage-related statistics
 340     virtual void getStats(StoreInfoStats &stats) const = 0;
 341
 342     /**
 343      * Output stats to the provided store entry.
 344      \todo make these calls asynchronous
 345      */
 346     virtual void stat(StoreEntry &) const = 0;
 347
 348     /** Sync the store prior to shutdown */
 349     virtual void sync();
 350
 351     /** remove a Store entry from the store */
 352     virtual void unlink (StoreEntry &);
 353
 354     /* search in the store */
 355     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 356
 357     /* pulled up from SwapDir for migration.... probably do not belong here */
 358     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 359
 360     /// Undo reference(), returning false iff idle e should be destroyed
 361     virtual bool dereference(StoreEntry &e, bool wantsLocalMemory) = 0;
 362
 363     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 364
 365     // XXX: This method belongs to Store::Root/StoreController, but it is here
 366     // because test cases use non-StoreController derivatives as Root
 367     /// called when the entry is no longer needed by any transaction
 368     virtual void handleIdleEntry(StoreEntry &e) {}
 369
 370     // XXX: This method belongs to Store::Root/StoreController, but it is here
 371     // because test cases use non-StoreController derivatives as Root
 372     /// called to get rid of no longer needed entry data in RAM, if any
 373     virtual void memoryOut(StoreEntry &e, const bool preserveSwappable) {}
 374
 375     // XXX: This method belongs to Store::Root/StoreController, but it is here
 376     // to avoid casting Root() to StoreController until Root() API is fixed.
 377     /// makes the entry available for collapsing future requests
 378     virtual void allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) {}
 379
 380     // XXX: This method belongs to Store::Root/StoreController, but it is here
 381     // to avoid casting Root() to StoreController until Root() API is fixed.
 382     /// Update local intransit entry after changes made by appending worker.
 383     virtual void syncCollapsed(const cache_key *key) {}
 384
 385     // XXX: This method belongs to Store::Root/StoreController, but it is here
 386     // to avoid casting Root() to StoreController until Root() API is fixed.
 387     /// calls Root().transients->abandon() if transients are tracked
 388     virtual void transientsAbandon(StoreEntry &e) {}
 389
 390     // XXX: This method belongs to Store::Root/StoreController, but it is here
 391     // to avoid casting Root() to StoreController until Root() API is fixed.
 392     /// disassociates the entry from the intransit table
 393     virtual void transientsDisconnect(MemObject &mem_obj) {}
 394
 395     // XXX: This method belongs to Store::Root/StoreController, but it is here
 396     // to avoid casting Root() to StoreController until Root() API is fixed.
 397     /// removes the entry from the memory cache
 398     virtual void memoryUnlink(StoreEntry &e) {}
 399
 400     // XXX: This method belongs to Store::Root/StoreController, but it is here
 401     // to avoid casting Root() to StoreController until Root() API is fixed.
 402     /// disassociates the entry from the memory cache, preserving cached data
 403     virtual void memoryDisconnect(MemObject &mem_obj) {}
 404
 405     /// If the entry is not found, return false. Otherwise, return true after
 406     /// tying the entry to this cache and setting inSync to updateCollapsed().
 407     virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync) { return false; }
 408
 409     /// update a local collapsed entry with fresh info from this cache (if any)
 410     virtual bool updateCollapsed(StoreEntry &collapsed) { return false; }
 411
 412 private:
 413     static RefCount<Store> CurrentRoot;
 414 };
 415
 416 /// \ingroup StoreAPI
 417 typedef RefCount<Store> StorePointer;
 418
 419 /// \ingroup StoreAPI
 420 size_t storeEntryInUse();
 421
 422 /// \ingroup StoreAPI
 423 const char *storeEntryFlags(const StoreEntry *);
 424
 425 /// \ingroup StoreAPI
 426 void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 427
 428 /// \ingroup StoreAPI
 429 StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 430
 431 /// \ingroup StoreAPI
 432 StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 433
 434 /// \ingroup StoreAPI
 435 StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 436
 437 /// \ingroup StoreAPI
 438 StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&);
 439
 440 /// \ingroup StoreAPI
 441 void storeInit(void);
 442
 443 /// \ingroup StoreAPI
 444 void storeConfigure(void);
 445
 446 /// \ingroup StoreAPI
 447 void storeFreeMemory(void);
 448
 449 /// \ingroup StoreAPI
 450 int expiresMoreThan(time_t, time_t);
 451
 452 /// \ingroup StoreAPI
 453 void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 454
 455 /// \ingroup StoreAPI
 456 void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 457
 458 /// \ingroup StoreAPI
 459 int storeTooManyDiskFilesOpen(void);
 460
 461 class SwapDir;
 462 /// \ingroup StoreAPI
 463 void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 464
 465 /// \ingroup StoreAPI
 466 void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 467
 468 /// \ingroup StoreAPI
 469 void storeFsInit(void);
 470
 471 /// \ingroup StoreAPI
 472 void storeFsDone(void);
 473
 474 /// \ingroup StoreAPI
 475 void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 476
 477 /// \ingroup StoreAPI
 478 extern FREE destroyStoreEntry;
 479
 480 /**
 481  \ingroup StoreAPI
 482  \todo should be a subclass of Packer perhaps ?
 483  */
 484 void packerToStoreInit(Packer * p, StoreEntry * e);
 485
 486 /// \ingroup StoreAPI
 487 void storeGetMemSpace(int size);
 488
 489 #if _USE_INLINE_
 490 #include "Store.cci"
 491 #endif
 492
 493 #endif /* SQUID_STORE_H */