src/MemObject.cc

   1
   2 /*
   3  * $Id: MemObject.cc,v 1.2 2003/02/05 10:36:48 robertc Exp $
   4  *
   5  * DEBUG: section 19    Store Memory Primitives
   6  * AUTHOR: Robert Collins
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "MemObject.h"
  38 #include "HttpRequest.h"
  39 #include "HttpReply.h"
  40 #include "Store.h"
  41 #include "StoreClient.h"
  42 #include "Generic.h"
  43 #if DELAY_POOLS
  44 #include "DelayPools.h"
  45 #endif
  46
  47 /* TODO: make this global or private */
  48 #if URL_CHECKSUM_DEBUG
  49 static unsigned int url_checksum(const char *url);
  50 unsigned int
  51 url_checksum(const char *url)
  52 {
  53     unsigned int ck;
  54     MD5_CTX M;
  55     static unsigned char digest[16];
  56     MD5Init(&M);
  57     MD5Update(&M, (unsigned char *) url, strlen(url));
  58     MD5Final(digest, &M);
  59     xmemcpy(&ck, digest, sizeof(ck));
  60     return ck;
  61 }
  62 #endif
  63
  64 MemPool *MemObject::pool = NULL;
  65
  66 void *
  67 MemObject::operator new (size_t byteCount)
  68 {
  69     /* derived classes with different sizes must implement their own new */
  70     assert (byteCount == sizeof (MemObject));
  71     if (!pool)
  72         pool = memPoolCreate("MemObject", sizeof (MemObject));
  73     return memPoolAlloc(pool);
  74 }
  75
  76 void
  77 MemObject::operator delete (void *address)
  78 {
  79     memPoolFree(pool, address);
  80 }
  81
  82 size_t
  83 MemObject::inUseCount()
  84 {
  85     if (!pool)
  86         return 0;
  87     MemPoolStats stats;
  88     memPoolGetStats (&stats, pool);
  89     return stats.items_inuse;
  90 }
  91
  92 MemObject::MemObject(char const *aUrl, char const *aLog_url) :
  93 _reply (httpReplyCreate())
  94 {
  95     url = xstrdup(aUrl);
  96 #if URL_CHECKSUM_DEBUG
  97     chksum = url_checksum(url);
  98 #endif
  99     log_url = xstrdup(aLog_url);
 100     object_sz = -1;
 101     fd = -1;
 102     /* XXX account log_url */
 103     debug(20, 3) ("MemObject::MemObject: initialized %p\n", this);
 104 }
 105
 106 MemObject::~MemObject()
 107 {
 108     const Ctx ctx = ctx_enter(url);
 109     debug(20, 3) ("destroy_MemObject: destroying %p\n", this);
 110 #if URL_CHECKSUM_DEBUG
 111     assert(chksum == url_checksum(url));
 112 #endif
 113     if (!shutting_down)
 114         assert(swapout.sio == NULL);
 115     data_hdr.freeContent();
 116     /*
 117      * There is no way to abort FD-less clients, so they might
 118      * still have mem->clients set if mem->fd == -1
 119      */
 120     assert(fd == -1 || clients.head == NULL);
 121     httpReplyDestroy((HttpReply *)_reply);
 122     requestUnlink(request);
 123     request = NULL;
 124     ctx_exit(ctx);              /* must exit before we free mem->url */
 125     safe_free(url);
 126     safe_free(log_url);    /* XXX account log_url */
 127     safe_free(vary_headers);
 128 }
 129
 130 void
 131 MemObject::unlinkRequest()
 132 {
 133     /* XXX Should we assert(request)? */
 134     requestUnlink(request);
 135     request = NULL;
 136 }
 137
 138 void
 139 MemObject::write ( StoreIOBuffer writeBuffer, STMCB *callback, void *callbackData)
 140 {
 141     debug(19, 6) ("memWrite: offset %lu len %d\n", writeBuffer.offset, writeBuffer.length);
 142
 143     /* the offset is into the content, not the headers */
 144     writeBuffer.offset += (_reply ? _reply->hdr_sz : 0);
 145
 146     /* We don't separate out mime headers yet, so ensure that the first
 147      * write is at offset 0 - where they start
 148      */
 149     assert (data_hdr.endOffset() || writeBuffer.offset == 0);
 150
 151     assert (data_hdr.write (writeBuffer));
 152     callback (callbackData, writeBuffer);
 153 }
 154
 155 void
 156 MemObject::dump() const
 157 {
 158     debug(20, 1) ("MemObject->data.head: %p\n",
 159         data_hdr.head);
 160     debug(20, 1) ("MemObject->data.tail: %p\n",
 161         data_hdr.tail);
 162 #if 0
 163     /* do we want this one? */
 164     debug(20, 1) ("MemObject->data.origin_offset: %d\n",
 165         data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0);
 166 #endif
 167     debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
 168         (int) start_ping.tv_sec,
 169         (int) start_ping.tv_usec);
 170     debug(20, 1) ("MemObject->inmem_hi: %d\n",
 171         (int) data_hdr.endOffset());
 172     debug(20, 1) ("MemObject->inmem_lo: %d\n",
 173         (int) inmem_lo);
 174     debug(20, 1) ("MemObject->nclients: %d\n",
 175         nclients);
 176     debug(20, 1) ("MemObject->reply: %p\n",
 177         _reply);
 178     debug(20, 1) ("MemObject->request: %p\n",
 179         request);
 180     debug(20, 1) ("MemObject->log_url: %p %s\n",
 181         log_url,
 182         checkNullString(log_url));
 183 }
 184
 185 HttpReply const *
 186 MemObject::getReply() const
 187 {
 188     return _reply;
 189 }
 190
 191 struct LowestMemReader : public unary_function<store_client, void>
 192 {
 193     LowestMemReader(off_t seed):current(seed){}
 194     void operator() (store_client const &x)
 195       {
 196         if (x.memReaderHasLowerOffset(current))
 197             current = x.copyInto.offset; }
 198     off_t current;
 199 };
 200
 201 struct StoreClientStats : public unary_function<store_client, void>
 202 {
 203     StoreClientStats(StoreEntry *anEntry):where(anEntry),index(0){}
 204     void operator()(store_client const &x) {
 205         x.dumpStats(where, index++);
 206     }
 207     StoreEntry *where;
 208     size_t index;
 209 };
 210
 211 void
 212 MemObject::stat (StoreEntry *s) const
 213 {
 214     storeAppendPrintf(s, "\t%s %s\n",
 215         RequestMethodStr[method], log_url);
 216     storeAppendPrintf(s, "\tinmem_lo: %d\n", (int) inmem_lo);
 217     storeAppendPrintf(s, "\tinmem_hi: %d\n", (int) data_hdr.endOffset());
 218     storeAppendPrintf(s, "\tswapout: %d bytes queued\n",
 219         (int) swapout.queue_offset);
 220     if (swapout.sio.getRaw())
 221         storeAppendPrintf(s, "\tswapout: %d bytes written\n",
 222             (int) swapout.sio->offset());
 223     StoreClientStats statsVisitor(s);
 224     for_each(clients, statsVisitor);
 225 }
 226
 227 off_t
 228 MemObject::endOffset () const
 229 {
 230     return data_hdr.endOffset();
 231 }
 232
 233 size_t
 234 MemObject::size() const
 235 {
 236     if (object_sz < 0)
 237         return endOffset();
 238     return object_sz;
 239 }
 240
 241 void
 242 MemObject::reset()
 243 {
 244     assert(swapout.sio == NULL);
 245     data_hdr.freeContent();
 246     inmem_lo = 0;
 247     /* Should we check for clients? */
 248 }
 249
 250
 251 off_t
 252 MemObject::lowestMemReaderOffset() const
 253 {
 254     LowestMemReader lowest (endOffset() + 1);
 255
 256     for_each (clients, lowest);
 257
 258     return lowest.current;
 259 }
 260
 261 /* XXX: This is wrong. It breaks *badly* on range combining */
 262 bool
 263 MemObject::readAheadPolicyCanRead() const
 264 {
 265     return (size_t)endOffset() - getReply()->hdr_sz < lowestMemReaderOffset() + Config.readAheadGap;
 266 }
 267
 268 void
 269 MemObject::addClient(store_client *aClient)
 270 {
 271     ++nclients;
 272     dlinkAdd(aClient, &aClient->node, &clients);
 273 }
 274
 275 #if URL_CHECKSUM_DEBUG
 276 void
 277 MemObject::checkUrlChecksum () const
 278 {
 279     assert(chksum == url_checksum(url));
 280 }
 281 #endif
 282
 283 /*
 284  * How much of the object data is on the disk?
 285  */
 286 size_t
 287 MemObject::objectBytesOnDisk() const
 288 {
 289     /*
 290      * NOTE: storeOffset() represents the disk file size,
 291      * not the amount of object data on disk.
 292      *
 293      * If we don't have at least 'swap_hdr_sz' bytes
 294      * then none of the object data is on disk.
 295      *
 296      * This should still be safe if swap_hdr_sz == 0,
 297      * meaning we haven't even opened the swapout file
 298      * yet.
 299      */
 300     if (swapout.sio.getRaw() == NULL)
 301         return 0;
 302     off_t nwritten = swapout.sio->offset();
 303     if (nwritten <= (off_t)swap_hdr_sz)
 304         return 0;
 305     return (size_t) (nwritten - swap_hdr_sz);
 306 }
 307
 308 off_t
 309 MemObject::policyLowestOffsetToKeep() const
 310 {
 311     /*
 312      * Careful.  lowest_offset can be greater than endOffset(), such
 313      * as in the case of a range request.
 314      */
 315     off_t lowest_offset = lowestMemReaderOffset();
 316     if (endOffset() < lowest_offset ||
 317         endOffset() - inmem_lo > (ssize_t)Config.Store.maxInMemObjSize)
 318         return lowest_offset;
 319
 320     return inmem_lo;
 321 }
 322
 323 void
 324 MemObject::trimSwappable()
 325 {
 326     off_t new_mem_lo = policyLowestOffsetToKeep();
 327     /*
 328      * We should only free up to what we know has been written
 329      * to disk, not what has been queued for writing.  Otherwise
 330      * there will be a chunk of the data which is not in memory
 331      * and is not yet on disk.
 332      * The -1 makes sure the page isn't freed until storeSwapOut has
 333      * walked to the next page. (mem->swapout.memnode)
 334      */
 335     off_t on_disk;
 336     if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
 337         new_mem_lo = on_disk - 1;
 338     if (new_mem_lo == -1)
 339         new_mem_lo = 0; /* the above might become -1 */
 340     data_hdr.freeDataUpto(new_mem_lo);
 341     inmem_lo = new_mem_lo;
 342 }
 343
 344 void
 345 MemObject::trimUnSwappable()
 346 {
 347     off_t new_mem_lo = policyLowestOffsetToKeep();
 348     assert (new_mem_lo > 0);
 349
 350     data_hdr.freeDataUpto(new_mem_lo);
 351     inmem_lo = new_mem_lo;
 352 }
 353
 354
 355 bool
 356 MemObject::isContiguous() const
 357 {
 358     bool result = data_hdr.hasContigousContentRange (inmem_lo, endOffset());
 359     /* XXX : make this higher level */
 360     debug (19, result ? 2 : 1) ("MemObject::isContiguous: Returning %s\n",
 361                                 result ? "true" : "false");
 362     return result;
 363 }
 364
 365 int
 366 MemObject::mostBytesWanted(int max) const
 367 {
 368 #if DELAY_POOLS
 369 #if 0
 370     int i = -1;
 371     for (dlink_node *node = clients.head; node; node = node->next) {
 372         store_client *sc = (store_client *) node->data;
 373         if (!sc->callbackPending())
 374             /* not waiting for more data */
 375             continue;
 376         if (sc->getType() != STORE_MEM_CLIENT)
 377             continue;
 378         i = sc->delayId.bytesWanted(i, XMIN(sc->copyInto.length, (size_t)max));
 379     }
 380     return XMAX(i, 0);
 381 #endif
 382     /* identify delay id with largest allowance */
 383     DelayId largestAllowance = mostBytesAllowed ();
 384     return largestAllowance.bytesWanted(0, max);
 385 #else
 386     return max;
 387 #endif
 388 }
 389
 390 #if DELAY_POOLS
 391 DelayId
 392 MemObject::mostBytesAllowed() const
 393 {
 394     int j;
 395     int jmax = -1;
 396     DelayId result;
 397     for (dlink_node *node = clients.head; node; node = node->next) {
 398         store_client *sc = (store_client *) node->data;
 399         if (!sc->callbackPending())
 400             /* not waiting for more data */
 401             continue;
 402         if (sc->getType() != STORE_MEM_CLIENT)
 403             /* reading off disk */
 404             continue;
 405         j = sc->delayId.bytesWanted(0, sc->copyInto.length);
 406         if (j > jmax) {
 407             jmax = j;
 408             result = sc->delayId;
 409         }
 410     }
 411     return result;
 412 }
 413 #endif