src/MemObject.cc

   1
   2 /*
   3  * $Id: MemObject.cc,v 1.30 2007/08/13 18:25:14 hno Exp $
   4  *
   5  * DEBUG: section 19    Store Memory Primitives
   6  * AUTHOR: Robert Collins
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "MemObject.h"
  38 #include "HttpRequest.h"
  39 #include "HttpReply.h"
  40 #include "Store.h"
  41 #include "StoreClient.h"
  42 #include "Generic.h"
  43 #if DELAY_POOLS
  44 #include "DelayPools.h"
  45 #endif
  46 #include "MemBuf.h"
  47
  48 /* TODO: make this global or private */
  49 #if URL_CHECKSUM_DEBUG
  50 static unsigned int url_checksum(const char *url);
  51 unsigned int
  52 url_checksum(const char *url)
  53 {
  54     unsigned int ck;
  55     MD5_CTX M;
  56     static unsigned char digest[16];
  57     MD5Init(&M);
  58     MD5Update(&M, (unsigned char *) url, strlen(url));
  59     MD5Final(digest, &M);
  60     xmemcpy(&ck, digest, sizeof(ck));
  61     return ck;
  62 }
  63
  64 #endif
  65
  66 RemovalPolicy * mem_policy = NULL;
  67
  68 size_t
  69 MemObject::inUseCount()
  70 {
  71     return Pool().inUseCount();
  72 }
  73
  74 MemObject::MemObject(char const *aUrl, char const *aLog_url)
  75 {
  76     debugs(20, 3, HERE << "new MemObject " << this);
  77     HttpReply *rep = new HttpReply;
  78
  79     _reply  = HTTPMSGLOCK(rep);
  80     url = xstrdup(aUrl);
  81
  82 #if URL_CHECKSUM_DEBUG
  83
  84     chksum = url_checksum(url);
  85
  86 #endif
  87
  88     log_url = xstrdup(aLog_url);
  89
  90     object_sz = -1;
  91
  92     /* XXX account log_url */
  93 }
  94
  95 MemObject::~MemObject()
  96 {
  97     debugs(20, 3, HERE << "del MemObject " << this);
  98     const Ctx ctx = ctx_enter(url);
  99 #if URL_CHECKSUM_DEBUG
 100
 101     assert(chksum == url_checksum(url));
 102 #endif
 103
 104     if (!shutting_down)
 105         assert(swapout.sio == NULL);
 106
 107     data_hdr.freeContent();
 108
 109 #if 0
 110     /*
 111      * There is no way to abort FD-less clients, so they might
 112      * still have mem->clients set.
 113      */
 114     assert(clients.head == NULL);
 115
 116 #endif
 117
 118     HTTPMSGUNLOCK(_reply);
 119
 120     HTTPMSGUNLOCK(request);
 121
 122     ctx_exit(ctx);              /* must exit before we free mem->url */
 123
 124     safe_free(url);
 125
 126     safe_free(log_url);    /* XXX account log_url */
 127
 128     safe_free(vary_headers);
 129 }
 130
 131 void
 132 MemObject::unlinkRequest()
 133 {
 134     HTTPMSGUNLOCK(request);
 135 }
 136
 137 void
 138 MemObject::write ( StoreIOBuffer writeBuffer, STMCB *callback, void *callbackData)
 139 {
 140     PROF_start(MemObject_write);
 141     debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
 142
 143     /* the offset is into the content, not the headers */
 144     writeBuffer.offset += (_reply ? _reply->hdr_sz : 0);
 145
 146     /* We don't separate out mime headers yet, so ensure that the first
 147      * write is at offset 0 - where they start
 148      */
 149     assert (data_hdr.endOffset() || writeBuffer.offset == 0);
 150
 151     assert (data_hdr.write (writeBuffer));
 152     callback (callbackData, writeBuffer);
 153     PROF_stop(MemObject_write);
 154 }
 155
 156 void
 157 MemObject::dump() const
 158 {
 159     data_hdr.dump();
 160 #if 0
 161     /* do we want this one? */
 162     debugs(20, 1, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
 163 #endif
 164
 165     debugs(20, 1, "MemObject->start_ping: " << start_ping.tv_sec  << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
 166     debugs(20, 1, "MemObject->inmem_hi: " << data_hdr.endOffset());
 167     debugs(20, 1, "MemObject->inmem_lo: " << inmem_lo);
 168     debugs(20, 1, "MemObject->nclients: " << nclients);
 169     debugs(20, 1, "MemObject->reply: " << _reply);
 170     debugs(20, 1, "MemObject->request: " << request);
 171     debugs(20, 1, "MemObject->log_url: " << log_url << " " << checkNullString(log_url));
 172 }
 173
 174 HttpReply const *
 175 MemObject::getReply() const
 176 {
 177     return _reply;
 178 }
 179
 180 void
 181 MemObject::replaceHttpReply(HttpReply *newrep)
 182 {
 183     HTTPMSGUNLOCK(_reply);
 184     _reply = HTTPMSGLOCK(newrep);
 185 }
 186
 187 struct LowestMemReader : public unary_function<store_client, void>
 188 {
 189     LowestMemReader(int64_t seed):current(seed){}
 190
 191     void operator() (store_client const &x)
 192     {
 193         if (x.memReaderHasLowerOffset(current))
 194             current = x.copyInto.offset;
 195     }
 196
 197     int64_t current;
 198 };
 199
 200 struct StoreClientStats : public unary_function<store_client, void>
 201 {
 202     StoreClientStats(MemBuf *anEntry):where(anEntry),index(0){}
 203
 204     void operator()(store_client const &x)
 205     {
 206         x.dumpStats(where, index++);
 207     }
 208
 209     MemBuf *where;
 210     size_t index;
 211 };
 212
 213 void
 214 MemObject::stat (MemBuf * mb) const
 215 {
 216     mb->Printf("\t%s %s\n",
 217                RequestMethodStr[method], log_url);
 218     mb->Printf("\tinmem_lo: %"PRId64"\n", inmem_lo);
 219     mb->Printf("\tinmem_hi: %"PRId64"\n", data_hdr.endOffset());
 220     mb->Printf("\tswapout: %"PRId64" bytes queued\n",
 221                swapout.queue_offset);
 222
 223     if (swapout.sio.getRaw())
 224         mb->Printf("\tswapout: %"PRId64" bytes written\n",
 225                    (int64_t) swapout.sio->offset());
 226
 227     StoreClientStats statsVisitor(mb);
 228
 229     for_each<StoreClientStats>(clients, statsVisitor);
 230 }
 231
 232 int64_t
 233 MemObject::endOffset () const
 234 {
 235     return data_hdr.endOffset();
 236 }
 237
 238 int64_t
 239 MemObject::size() const
 240 {
 241     if (object_sz < 0)
 242         return endOffset();
 243
 244     return object_sz;
 245 }
 246
 247 void
 248 MemObject::reset()
 249 {
 250     assert(swapout.sio == NULL);
 251     data_hdr.freeContent();
 252     inmem_lo = 0;
 253     /* Should we check for clients? */
 254 }
 255
 256
 257 int64_t
 258 MemObject::lowestMemReaderOffset() const
 259 {
 260     LowestMemReader lowest (endOffset() + 1);
 261
 262     for_each <LowestMemReader>(clients, lowest);
 263
 264     return lowest.current;
 265 }
 266
 267 /* XXX: This is wrong. It breaks *badly* on range combining */
 268 bool
 269 MemObject::readAheadPolicyCanRead() const
 270 {
 271     return endOffset() - getReply()->hdr_sz < lowestMemReaderOffset() + Config.readAheadGap;
 272 }
 273
 274 void
 275 MemObject::addClient(store_client *aClient)
 276 {
 277     ++nclients;
 278     dlinkAdd(aClient, &aClient->node, &clients);
 279 }
 280
 281 #if URL_CHECKSUM_DEBUG
 282 void
 283 MemObject::checkUrlChecksum () const
 284 {
 285     assert(chksum == url_checksum(url));
 286 }
 287
 288 #endif
 289
 290 /*
 291  * How much of the object data is on the disk?
 292  */
 293 int64_t
 294 MemObject::objectBytesOnDisk() const
 295 {
 296     /*
 297      * NOTE: storeOffset() represents the disk file size,
 298      * not the amount of object data on disk.
 299      *
 300      * If we don't have at least 'swap_hdr_sz' bytes
 301      * then none of the object data is on disk.
 302      *
 303      * This should still be safe if swap_hdr_sz == 0,
 304      * meaning we haven't even opened the swapout file
 305      * yet.
 306      */
 307
 308     if (swapout.sio.getRaw() == NULL)
 309         return 0;
 310
 311     int64_t nwritten = swapout.sio->offset();
 312
 313     if (nwritten <= (int64_t)swap_hdr_sz)
 314         return 0;
 315
 316     return (nwritten - swap_hdr_sz);
 317 }
 318
 319 int64_t
 320 MemObject::policyLowestOffsetToKeep() const
 321 {
 322     /*
 323      * Careful.  lowest_offset can be greater than endOffset(), such
 324      * as in the case of a range request.
 325      */
 326     int64_t lowest_offset = lowestMemReaderOffset();
 327
 328     if (endOffset() < lowest_offset ||
 329             endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize)
 330         return lowest_offset;
 331
 332     return inmem_lo;
 333 }
 334
 335 void
 336 MemObject::trimSwappable()
 337 {
 338     int64_t new_mem_lo = policyLowestOffsetToKeep();
 339     /*
 340      * We should only free up to what we know has been written
 341      * to disk, not what has been queued for writing.  Otherwise
 342      * there will be a chunk of the data which is not in memory
 343      * and is not yet on disk.
 344      * The -1 makes sure the page isn't freed until storeSwapOut has
 345      * walked to the next page. (mem->swapout.memnode)
 346      */
 347     int64_t on_disk;
 348
 349     if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
 350         new_mem_lo = on_disk - 1;
 351
 352     if (new_mem_lo == -1)
 353         new_mem_lo = 0; /* the above might become -1 */
 354
 355     data_hdr.freeDataUpto(new_mem_lo);
 356
 357     inmem_lo = new_mem_lo;
 358 }
 359
 360 void
 361 MemObject::trimUnSwappable()
 362 {
 363     int64_t new_mem_lo = policyLowestOffsetToKeep();
 364     assert (new_mem_lo > 0);
 365
 366     data_hdr.freeDataUpto(new_mem_lo);
 367     inmem_lo = new_mem_lo;
 368 }
 369
 370
 371 bool
 372 MemObject::isContiguous() const
 373 {
 374     bool result = data_hdr.hasContigousContentRange (Range<int64_t>(inmem_lo, endOffset()));
 375     /* XXX : make this higher level */
 376     debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
 377     return result;
 378 }
 379
 380 int
 381 MemObject::mostBytesWanted(int max) const
 382 {
 383 #if DELAY_POOLS
 384     /* identify delay id with largest allowance */
 385     DelayId largestAllowance = mostBytesAllowed ();
 386     return largestAllowance.bytesWanted(0, max);
 387 #else
 388
 389     return max;
 390 #endif
 391 }
 392
 393 void
 394 MemObject::setNoDelay(bool const newValue)
 395 {
 396 #if DELAY_POOLS
 397
 398     for (dlink_node *node = clients.head; node; node = node->next) {
 399         store_client *sc = (store_client *) node->data;
 400         sc->delayId.setNoDelay(newValue);
 401     }
 402
 403 #endif
 404 }
 405
 406 void
 407 MemObject::delayRead(DeferredRead const &aRead)
 408 {
 409     deferredReads.delayRead(aRead);
 410 }
 411
 412 void
 413 MemObject::kickReads()
 414 {
 415     deferredReads.kickReads(-1);
 416 }
 417
 418 #if DELAY_POOLS
 419 DelayId
 420 MemObject::mostBytesAllowed() const
 421 {
 422     int j;
 423     int jmax = -1;
 424     DelayId result;
 425
 426     for (dlink_node *node = clients.head; node; node = node->next) {
 427         store_client *sc = (store_client *) node->data;
 428 #if 0
 429         /* This test is invalid because the client may be writing data
 430          * and thus will want data immediately.
 431          * If we include the test, there is a race condition when too much
 432          * data is read - if all sc's are writing when a read is scheduled.
 433          * XXX: fixme.
 434          */
 435
 436         if (!sc->callbackPending())
 437             /* not waiting for more data */
 438             continue;
 439
 440 #endif
 441
 442         if (sc->getType() != STORE_MEM_CLIENT)
 443             /* reading off disk */
 444             continue;
 445
 446         j = sc->delayId.bytesWanted(0, sc->copyInto.length);
 447
 448         if (j > jmax) {
 449             jmax = j;
 450             result = sc->delayId;
 451         }
 452     }
 453
 454     return result;
 455 }
 456
 457 #endif