src/Store.h

   1 /*
   2  * $Id$
   3  *
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32 #ifndef SQUID_STORE_H
  33 #define SQUID_STORE_H
  34
  35 /**
  36  \defgroup StoreAPI  Store API
  37  \ingroup FileSystems
  38  */
  39
  40 #include "comm/forward.h"
  41 #include "CommRead.h"
  42 #include "HttpRequestMethod.h"
  43 #include "Packer.h"
  44 #include "Range.h"
  45 #include "RefCount.h"
  46 #include "RemovalPolicy.h"
  47 #include "StoreIOBuffer.h"
  48 #include "StoreStats.h"
  49
  50 #if USE_SQUID_ESI
  51 #include "esi/Element.h"
  52 #endif
  53
  54 #if HAVE_OSTREAM
  55 #include <ostream>
  56 #endif
  57
  58
  59 class AsyncCall;
  60 class HttpRequest;
  61 class MemObject;
  62 class StoreClient;
  63 class StoreSearch;
  64 class SwapDir;
  65
  66 extern StoreIoStats store_io_stats;
  67
  68 /// maximum number of entries per cache_dir
  69 enum { SwapFilenMax = 0xFFFFFF }; // keep in sync with StoreEntry::swap_filen
  70
  71 /**
  72  \ingroup StoreAPI
  73  */
  74 class StoreEntry : public hash_link
  75 {
  76
  77 public:
  78     static DeferredRead::DeferrableRead DeferReader;
  79     bool checkDeferRead(int fd) const;
  80
  81     virtual const char *getMD5Text() const;
  82     StoreEntry();
  83     StoreEntry(const char *url, const char *log_url);
  84     virtual ~StoreEntry();
  85
  86     virtual HttpReply const *getReply() const;
  87     virtual void write (StoreIOBuffer);
  88     virtual _SQUID_INLINE_ bool isEmpty() const;
  89     virtual bool isAccepting() const;
  90     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const;
  91     virtual void complete();
  92     virtual store_client_t storeClientType() const;
  93     virtual char const *getSerialisedMetaData();
  94     void replaceHttpReply(HttpReply *, bool andStartWriting = true);
  95     void startWriting(); ///< pack and write reply headers and, maybe, body
  96     /// whether we may start writing to disk (now or in the future)
  97     virtual bool mayStartSwapOut();
  98     virtual void trimMemory(const bool preserveSwappable);
  99     void abort();
 100     void unlink();
 101     void makePublic();
 102     void makePrivate();
 103     void setPublicKey();
 104     void setPrivateKey();
 105     void expireNow();
 106     void releaseRequest();
 107     void negativeCache();
 108     void cacheNegatively();             /** \todo argh, why both? */
 109     void invokeHandlers();
 110     void purgeMem();
 111     void cacheInMemory(); ///< start or continue storing in memory cache
 112     void swapOut();
 113     /// whether we are in the process of writing this entry to disk
 114     bool swappingOut() const { return swap_status == SWAPOUT_WRITING; }
 115     void swapOutFileClose(int how);
 116     const char *url() const;
 117     int checkCachable();
 118     int checkNegativeHit() const;
 119     int locked() const;
 120     int validToSend() const;
 121     bool memoryCachable() const; ///< may be cached in memory
 122     void createMemObject(const char *, const char *);
 123     void hideMemObject(); ///< no mem_obj for callers until createMemObject
 124     void dump(int debug_lvl) const;
 125     void hashDelete();
 126     void hashInsert(const cache_key *);
 127     void registerAbort(STABH * cb, void *);
 128     void reset();
 129     void setMemStatus(mem_status_t);
 130     void timestampsSet();
 131     void unregisterAbort();
 132     void destroyMemObject();
 133     int checkTooSmall();
 134
 135     void delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback);
 136
 137     void setNoDelay (bool const);
 138     bool modifiedSince(HttpRequest * request) const;
 139     /// has ETag matching at least one of the If-Match etags
 140     bool hasIfMatchEtag(const HttpRequest &request) const;
 141     /// has ETag matching at least one of the If-None-Match etags
 142     bool hasIfNoneMatchEtag(const HttpRequest &request) const;
 143
 144     /** What store does this entry belong too ? */
 145     virtual RefCount<SwapDir> store() const;
 146
 147     MemObject *mem_obj;
 148     MemObject *hidden_mem_obj; ///< mem_obj created before URLs were known
 149     RemovalPolicyNode repl;
 150     /* START OF ON-DISK STORE_META_STD TLV field */
 151     time_t timestamp;
 152     time_t lastref;
 153     time_t expires;
 154     time_t lastmod;
 155     uint64_t swap_file_sz;
 156     uint16_t refcount;
 157     uint16_t flags;
 158     /* END OF ON-DISK STORE_META_STD */
 159
 160     /// unique ID inside a cache_dir for swapped out entries; -1 for others
 161     sfileno swap_filen:25; // keep in sync with SwapFilenMax
 162
 163     sdirno swap_dirn:7;
 164
 165     unsigned short lock_count;          /* Assume < 65536! */
 166
 167     mem_status_t mem_status:3;
 168
 169     ping_status_t ping_status:3;
 170
 171     store_status_t store_status:3;
 172
 173     swap_status_t swap_status:3;
 174
 175 public:
 176     static size_t inUseCount();
 177     static void getPublicByRequestMethod(StoreClient * aClient, HttpRequest * request, const HttpRequestMethod& method);
 178     static void getPublicByRequest(StoreClient * aClient, HttpRequest * request);
 179     static void getPublic(StoreClient * aClient, const char *uri, const HttpRequestMethod& method);
 180
 181     virtual bool isNull() {
 182         return false;
 183     };
 184
 185     void *operator new(size_t byteCount);
 186     void operator delete(void *address);
 187     void setReleaseFlag();
 188 #if USE_SQUID_ESI
 189
 190     ESIElement::Pointer cachedESITree;
 191 #endif
 192     /** append bytes to the buffer */
 193     virtual void append(char const *, int len);
 194     /** disable sending content to the clients */
 195     virtual void buffer();
 196     /** flush any buffered content */
 197     virtual void flush();
 198     /** reduce the memory lock count on the entry */
 199     virtual int unlock();
 200     /** increate the memory lock count on the entry */
 201     virtual int64_t objectLen() const;
 202     virtual int64_t contentLen() const;
 203
 204     virtual void lock();
 205     virtual void release();
 206
 207 #if USE_ADAPTATION
 208     /// call back producer when more buffer space is available
 209     void deferProducer(const AsyncCall::Pointer &producer);
 210     /// calls back producer registered with deferProducer
 211     void kickProducer();
 212 #endif
 213
 214 private:
 215     static MemAllocator *pool;
 216
 217 #if USE_ADAPTATION
 218     /// producer callback registered with deferProducer
 219     AsyncCall::Pointer deferredProducer;
 220 #endif
 221
 222     bool validLength() const;
 223     bool hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const;
 224 };
 225
 226 std::ostream &operator <<(std::ostream &os, const StoreEntry &e);
 227
 228 /// \ingroup StoreAPI
 229 class NullStoreEntry:public StoreEntry
 230 {
 231
 232 public:
 233     static NullStoreEntry *getInstance();
 234     bool isNull() {
 235         return true;
 236     }
 237
 238     const char *getMD5Text() const;
 239     _SQUID_INLINE_ HttpReply const *getReply() const;
 240     void write (StoreIOBuffer) {}
 241
 242     bool isEmpty () const {return true;}
 243
 244     virtual size_t bytesWanted(Range<size_t> const aRange, bool ignoreDelayPool = false) const { return aRange.end; }
 245
 246     void operator delete(void *address);
 247     void complete() {}
 248
 249 private:
 250     store_client_t storeClientType() const {return STORE_MEM_CLIENT;}
 251
 252     char const *getSerialisedMetaData();
 253     bool mayStartSwapout() {return false;}
 254
 255     void trimMemory(const bool preserveSwappable) {}
 256
 257
 258     static NullStoreEntry _instance;
 259 };
 260
 261 /// \ingroup StoreAPI
 262 typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata);
 263
 264
 265 /**
 266  \ingroup StoreAPI
 267  * Abstract base class that will replace the whole store and swapdir interface.
 268  */
 269 class Store : public RefCountable
 270 {
 271
 272 public:
 273     /** The root store */
 274     static _SQUID_INLINE_ Store &Root();
 275     static void Root(Store *);
 276     static void Root(RefCount<Store>);
 277     static void Stats(StoreEntry * output);
 278     static void Maintain(void *unused);
 279
 280     virtual ~Store() {}
 281
 282     /** Handle pending callbacks - called by the event loop. */
 283     virtual int callback() = 0;
 284
 285     /** create the resources needed for this store to operate */
 286     virtual void create();
 287
 288     /**
 289      * Notify this store that its disk is full.
 290      \todo XXX move into a protected api call between store files and their stores, rather than a top level api call
 291      */
 292     virtual void diskFull();
 293
 294     /** Retrieve a store entry from the store */
 295     virtual StoreEntry * get(const cache_key *) = 0;
 296
 297     /** \todo imeplement the async version */
 298     virtual void get(String const key , STOREGETCLIENT callback, void *cbdata) = 0;
 299
 300     /* prepare the store for use. The store need not be usable immediately,
 301      * it should respond to readable() and writable() with true as soon
 302      * as it can provide those services
 303      */
 304     virtual void init() = 0;
 305
 306     /**
 307      * The maximum size the store will support in normal use. Inaccuracy is permitted,
 308      * but may throw estimates for memory etc out of whack.
 309      */
 310     virtual uint64_t maxSize() const = 0;
 311
 312     /** The minimum size the store will shrink to via normal housekeeping */
 313     virtual uint64_t minSize() const = 0;
 314
 315     /** current store size */
 316     virtual uint64_t currentSize() const = 0;
 317
 318     /** the total number of objects stored */
 319     virtual uint64_t currentCount() const = 0;
 320
 321     /** the maximum object size that can be stored, -1 if unlimited */
 322     virtual int64_t maxObjectSize() const = 0;
 323
 324     /// collect cache storage-related statistics
 325     virtual void getStats(StoreInfoStats &stats) const = 0;
 326
 327     /**
 328      * Output stats to the provided store entry.
 329      \todo make these calls asynchronous
 330      */
 331     virtual void stat(StoreEntry &) const = 0;
 332
 333     /** Sync the store prior to shutdown */
 334     virtual void sync();
 335
 336     /** remove a Store entry from the store */
 337     virtual void unlink (StoreEntry &);
 338
 339     /* search in the store */
 340     virtual StoreSearch *search(String const url, HttpRequest *) = 0;
 341
 342     /* pulled up from SwapDir for migration.... probably do not belong here */
 343     virtual void reference(StoreEntry &) = 0;   /* Reference this object */
 344
 345     /// Undo reference(), returning false iff idle e should be destroyed
 346     virtual bool dereference(StoreEntry &e) = 0;
 347
 348     virtual void maintain() = 0; /* perform regular maintenance should be private and self registered ... */
 349
 350     // XXX: This method belongs to Store::Root/StoreController, but it is here
 351     // because test cases use non-StoreController derivatives as Root
 352     /// called when the entry is no longer needed by any transaction
 353     virtual void handleIdleEntry(StoreEntry &e) {}
 354
 355     // XXX: This method belongs to Store::Root/StoreController, but it is here
 356     // because test cases use non-StoreController derivatives as Root
 357     /// called to get rid of no longer needed entry data in RAM, if any
 358     virtual void maybeTrimMemory(StoreEntry &e, const bool preserveSwappable) {}
 359
 360 private:
 361     static RefCount<Store> CurrentRoot;
 362 };
 363
 364 /// \ingroup StoreAPI
 365 typedef RefCount<Store> StorePointer;
 366
 367 /// \ingroup StoreAPI
 368 SQUIDCEXTERN size_t storeEntryInUse();
 369
 370 /// \ingroup StoreAPI
 371 SQUIDCEXTERN const char *storeEntryFlags(const StoreEntry *);
 372
 373 /// \ingroup StoreAPI
 374 extern void storeEntryReplaceObject(StoreEntry *, HttpReply *);
 375
 376 /// \ingroup StoreAPI
 377 SQUIDCEXTERN StoreEntry *storeGetPublic(const char *uri, const HttpRequestMethod& method);
 378
 379 /// \ingroup StoreAPI
 380 SQUIDCEXTERN StoreEntry *storeGetPublicByRequest(HttpRequest * request);
 381
 382 /// \ingroup StoreAPI
 383 SQUIDCEXTERN StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method);
 384
 385 /// \ingroup StoreAPI
 386 SQUIDCEXTERN StoreEntry *storeCreateEntry(const char *, const char *, request_flags, const HttpRequestMethod&);
 387
 388 /// \ingroup StoreAPI
 389 SQUIDCEXTERN void storeInit(void);
 390
 391 /// \ingroup StoreAPI
 392 SQUIDCEXTERN void storeConfigure(void);
 393
 394 /// \ingroup StoreAPI
 395 SQUIDCEXTERN void storeFreeMemory(void);
 396
 397 /// \ingroup StoreAPI
 398 SQUIDCEXTERN int expiresMoreThan(time_t, time_t);
 399
 400 /// \ingroup StoreAPI
 401 SQUIDCEXTERN void storeAppendPrintf(StoreEntry *, const char *,...) PRINTF_FORMAT_ARG2;
 402
 403 /// \ingroup StoreAPI
 404 extern void storeAppendVPrintf(StoreEntry *, const char *, va_list ap);
 405
 406 /// \ingroup StoreAPI
 407 SQUIDCEXTERN int storeTooManyDiskFilesOpen(void);
 408
 409 class SwapDir;
 410 /// \ingroup StoreAPI
 411 SQUIDCEXTERN void storeHeapPositionUpdate(StoreEntry *, SwapDir *);
 412
 413 /// \ingroup StoreAPI
 414 SQUIDCEXTERN void storeSwapFileNumberSet(StoreEntry * e, sfileno filn);
 415
 416 /// \ingroup StoreAPI
 417 SQUIDCEXTERN void storeFsInit(void);
 418
 419 /// \ingroup StoreAPI
 420 SQUIDCEXTERN void storeFsDone(void);
 421
 422 /// \ingroup StoreAPI
 423 SQUIDCEXTERN void storeReplAdd(const char *, REMOVALPOLICYCREATE *);
 424
 425 /// \ingroup StoreAPI
 426 extern FREE destroyStoreEntry;
 427
 428 /**
 429  \ingroup StoreAPI
 430  \todo should be a subclass of Packer perhaps ?
 431  */
 432 SQUIDCEXTERN void packerToStoreInit(Packer * p, StoreEntry * e);
 433
 434 /// \ingroup StoreAPI
 435 SQUIDCEXTERN void storeGetMemSpace(int size);
 436
 437 #if _USE_INLINE_
 438 #include "Store.cci"
 439 #endif
 440
 441 #endif /* SQUID_STORE_H */