src/MemObject.cc

   1
   2 /*
   3  * DEBUG: section 19    Store Memory Primitives
   4  * AUTHOR: Robert Collins
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  */
  33
  34 #include "squid.h"
  35 #include "comm/Connection.h"
  36 #include "Generic.h"
  37 #include "globals.h"
  38 #include "HttpReply.h"
  39 #include "HttpRequest.h"
  40 #include "MemBuf.h"
  41 #include "MemObject.h"
  42 #include "profiler/Profiler.h"
  43 #include "SquidConfig.h"
  44 #include "Store.h"
  45 #include "StoreClient.h"
  46
  47 #if USE_DELAY_POOLS
  48 #include "DelayPools.h"
  49 #endif
  50
  51 /* TODO: make this global or private */
  52 #if URL_CHECKSUM_DEBUG
  53 static unsigned int url_checksum(const char *url);
  54 unsigned int
  55 url_checksum(const char *url)
  56 {
  57     unsigned int ck;
  58     SquidMD5_CTX M;
  59     static unsigned char digest[16];
  60     SquidMD5Init(&M);
  61     SquidMD5Update(&M, (unsigned char *) url, strlen(url));
  62     SquidMD5Final(digest, &M);
  63     memcpy(&ck, digest, sizeof(ck));
  64     return ck;
  65 }
  66
  67 #endif
  68
  69 RemovalPolicy * mem_policy = NULL;
  70
  71 size_t
  72 MemObject::inUseCount()
  73 {
  74     return Pool().inUseCount();
  75 }
  76
  77 const char *
  78 MemObject::storeId() const {
  79     if (!storeId_.defined()) {
  80         debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
  81         dump();
  82         storeId_ = "[unknown_URI]";
  83     }
  84     return storeId_.termedBuf();
  85 }
  86
  87 const char *
  88 MemObject::logUri() const {
  89     return logUri_.defined() ? logUri_.termedBuf() : storeId();
  90 }
  91
  92 bool
  93 MemObject::hasUris() const {
  94     return storeId_.defined();
  95 }
  96
  97 void
  98 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
  99 {
 100     storeId_ = aStoreId;
 101
 102     // fast pointer comparison for a common storeCreateEntry(url,url,...) case
 103     if (!aLogUri || aLogUri == aStoreId)
 104         logUri_.clean(); // use storeId_ by default to minimize copying
 105     else
 106         logUri_ = aLogUri;
 107
 108     method = aMethod;
 109
 110 #if URL_CHECKSUM_DEBUG
 111     chksum = url_checksum(urlXXX());
 112 #endif
 113 }
 114
 115 MemObject::MemObject(): smpCollapsed(false)
 116 {
 117     debugs(20, 3, HERE << "new MemObject " << this);
 118     _reply = new HttpReply;
 119     HTTPMSGLOCK(_reply);
 120
 121     object_sz = -1;
 122
 123     /* XXX account log_url */
 124
 125     swapout.decision = SwapOut::swNeedsCheck;
 126 }
 127
 128 MemObject::~MemObject()
 129 {
 130     debugs(20, 3, HERE << "del MemObject " << this);
 131     const Ctx ctx = ctx_enter(urlXXX());
 132
 133 #if URL_CHECKSUM_DEBUG
 134     checkUrlChecksum();
 135 #endif
 136
 137     if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
 138         assert(xitTable.index < 0);
 139         assert(memCache.index < 0);
 140         assert(swapout.sio == NULL);
 141     }
 142
 143     data_hdr.freeContent();
 144
 145 #if 0
 146     /*
 147      * There is no way to abort FD-less clients, so they might
 148      * still have mem->clients set.
 149      */
 150     assert(clients.head == NULL);
 151
 152 #endif
 153
 154     HTTPMSGUNLOCK(_reply);
 155
 156     HTTPMSGUNLOCK(request);
 157
 158     ctx_exit(ctx);              /* must exit before we free mem->url */
 159
 160     safe_free(vary_headers);
 161 }
 162
 163 void
 164 MemObject::unlinkRequest()
 165 {
 166     HTTPMSGUNLOCK(request);
 167 }
 168
 169 void
 170 MemObject::write(const StoreIOBuffer &writeBuffer)
 171 {
 172     PROF_start(MemObject_write);
 173     debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
 174
 175     /* We don't separate out mime headers yet, so ensure that the first
 176      * write is at offset 0 - where they start
 177      */
 178     assert (data_hdr.endOffset() || writeBuffer.offset == 0);
 179
 180     assert (data_hdr.write (writeBuffer));
 181     PROF_stop(MemObject_write);
 182 }
 183
 184 void
 185 MemObject::dump() const
 186 {
 187     data_hdr.dump();
 188 #if 0
 189     /* do we want this one? */
 190     debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
 191 #endif
 192
 193     debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec  << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
 194     debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
 195     debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
 196     debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
 197     debugs(20, DBG_IMPORTANT, "MemObject->reply: " << _reply);
 198     debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
 199     debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
 200     debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
 201 }
 202
 203 HttpReply const *
 204 MemObject::getReply() const
 205 {
 206     return _reply;
 207 }
 208
 209 void
 210 MemObject::replaceHttpReply(HttpReply *newrep)
 211 {
 212     HTTPMSGUNLOCK(_reply);
 213     _reply = newrep;
 214     HTTPMSGLOCK(_reply);
 215 }
 216
 217 struct LowestMemReader : public unary_function<store_client, void> {
 218     LowestMemReader(int64_t seed):current(seed) {}
 219
 220     void operator() (store_client const &x) {
 221         if (x.memReaderHasLowerOffset(current))
 222             current = x.copyInto.offset;
 223     }
 224
 225     int64_t current;
 226 };
 227
 228 struct StoreClientStats : public unary_function<store_client, void> {
 229     StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
 230
 231     void operator()(store_client const &x) {
 232         x.dumpStats(where, index);
 233         ++index;
 234     }
 235
 236     MemBuf *where;
 237     size_t index;
 238 };
 239
 240 void
 241 MemObject::stat(MemBuf * mb) const
 242 {
 243     mb->Printf("\t%s %s\n",
 244                RequestMethodStr(method), logUri());
 245     if (vary_headers)
 246         mb->Printf("\tvary_headers: %s\n", vary_headers);
 247     mb->Printf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
 248     mb->Printf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
 249     mb->Printf("\tswapout: %" PRId64 " bytes queued\n",
 250                swapout.queue_offset);
 251
 252     if (swapout.sio.getRaw())
 253         mb->Printf("\tswapout: %" PRId64 " bytes written\n",
 254                    (int64_t) swapout.sio->offset());
 255
 256     if (xitTable.index >= 0)
 257         mb->Printf("\ttransient index: %d state: %d\n",
 258                    xitTable.index, xitTable.io);
 259     if (memCache.index >= 0)
 260         mb->Printf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n",
 261                    memCache.index, memCache.io, memCache.offset);
 262     if (object_sz >= 0)
 263         mb->Printf("\tobject_sz: %" PRId64 "\n", object_sz);
 264     if (smpCollapsed)
 265         mb->Printf("\tsmp-collapsed\n");
 266
 267     StoreClientStats statsVisitor(mb);
 268
 269     for_each<StoreClientStats>(clients, statsVisitor);
 270 }
 271
 272 int64_t
 273 MemObject::endOffset () const
 274 {
 275     return data_hdr.endOffset();
 276 }
 277
 278 void
 279 MemObject::markEndOfReplyHeaders()
 280 {
 281     const int hdr_sz = endOffset();
 282     assert(hdr_sz >= 0);
 283     assert(_reply);
 284     _reply->hdr_sz = hdr_sz;
 285 }
 286
 287 int64_t
 288 MemObject::size() const
 289 {
 290     if (object_sz < 0)
 291         return endOffset();
 292
 293     return object_sz;
 294 }
 295
 296 int64_t
 297 MemObject::expectedReplySize() const
 298 {
 299     debugs(20, 7, HERE << "object_sz: " << object_sz);
 300     if (object_sz >= 0) // complete() has been called; we know the exact answer
 301         return object_sz;
 302
 303     if (_reply) {
 304         const int64_t clen = _reply->bodySize(method);
 305         debugs(20, 7, HERE << "clen: " << clen);
 306         if (clen >= 0 && _reply->hdr_sz > 0) // yuck: HttpMsg sets hdr_sz to 0
 307             return clen + _reply->hdr_sz;
 308     }
 309
 310     return -1; // not enough information to predict
 311 }
 312
 313 void
 314 MemObject::reset()
 315 {
 316     assert(swapout.sio == NULL);
 317     data_hdr.freeContent();
 318     inmem_lo = 0;
 319     /* Should we check for clients? */
 320 }
 321
 322 int64_t
 323 MemObject::lowestMemReaderOffset() const
 324 {
 325     LowestMemReader lowest (endOffset() + 1);
 326
 327     for_each <LowestMemReader>(clients, lowest);
 328
 329     return lowest.current;
 330 }
 331
 332 /* XXX: This is wrong. It breaks *badly* on range combining */
 333 bool
 334 MemObject::readAheadPolicyCanRead() const
 335 {
 336     const bool canRead = endOffset() - getReply()->hdr_sz <
 337         lowestMemReaderOffset() + Config.readAheadGap;
 338
 339     if (!canRead) {
 340         debugs(19, 9, "no: " << endOffset() << '-' << getReply()->hdr_sz <<
 341                " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
 342     }
 343
 344     return canRead;
 345 }
 346
 347 void
 348 MemObject::addClient(store_client *aClient)
 349 {
 350     ++nclients;
 351     dlinkAdd(aClient, &aClient->node, &clients);
 352 }
 353
 354 #if URL_CHECKSUM_DEBUG
 355 void
 356 MemObject::checkUrlChecksum () const
 357 {
 358     assert(chksum == url_checksum(urlXXX()));
 359 }
 360
 361 #endif
 362
 363 /*
 364  * How much of the object data is on the disk?
 365  */
 366 int64_t
 367 MemObject::objectBytesOnDisk() const
 368 {
 369     /*
 370      * NOTE: storeOffset() represents the disk file size,
 371      * not the amount of object data on disk.
 372      *
 373      * If we don't have at least 'swap_hdr_sz' bytes
 374      * then none of the object data is on disk.
 375      *
 376      * This should still be safe if swap_hdr_sz == 0,
 377      * meaning we haven't even opened the swapout file
 378      * yet.
 379      */
 380
 381     if (swapout.sio.getRaw() == NULL)
 382         return 0;
 383
 384     int64_t nwritten = swapout.sio->offset();
 385
 386     if (nwritten <= (int64_t)swap_hdr_sz)
 387         return 0;
 388
 389     return (nwritten - swap_hdr_sz);
 390 }
 391
 392 int64_t
 393 MemObject::policyLowestOffsetToKeep(bool swap) const
 394 {
 395     /*
 396      * Careful.  lowest_offset can be greater than endOffset(), such
 397      * as in the case of a range request.
 398      */
 399     int64_t lowest_offset = lowestMemReaderOffset();
 400
 401     if (endOffset() < lowest_offset ||
 402             endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
 403             (swap && !Config.onoff.memory_cache_first))
 404         return lowest_offset;
 405
 406     return inmem_lo;
 407 }
 408
 409 void
 410 MemObject::trimSwappable()
 411 {
 412     int64_t new_mem_lo = policyLowestOffsetToKeep(1);
 413     /*
 414      * We should only free up to what we know has been written
 415      * to disk, not what has been queued for writing.  Otherwise
 416      * there will be a chunk of the data which is not in memory
 417      * and is not yet on disk.
 418      * The -1 makes sure the page isn't freed until storeSwapOut has
 419      * walked to the next page.
 420      */
 421     int64_t on_disk;
 422
 423     if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
 424         new_mem_lo = on_disk - 1;
 425
 426     if (new_mem_lo == -1)
 427         new_mem_lo = 0; /* the above might become -1 */
 428
 429     data_hdr.freeDataUpto(new_mem_lo);
 430
 431     inmem_lo = new_mem_lo;
 432 }
 433
 434 void
 435 MemObject::trimUnSwappable()
 436 {
 437     if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
 438         assert (new_mem_lo > 0);
 439         data_hdr.freeDataUpto(new_mem_lo);
 440         inmem_lo = new_mem_lo;
 441     } // else we should not trim anything at this time
 442 }
 443
 444 bool
 445 MemObject::isContiguous() const
 446 {
 447     bool result = data_hdr.hasContigousContentRange (Range<int64_t>(inmem_lo, endOffset()));
 448     /* XXX : make this higher level */
 449     debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
 450     return result;
 451 }
 452
 453 int
 454 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
 455 {
 456 #if USE_DELAY_POOLS
 457     if (!ignoreDelayPools) {
 458         /* identify delay id with largest allowance */
 459         DelayId largestAllowance = mostBytesAllowed ();
 460         return largestAllowance.bytesWanted(0, max);
 461     }
 462 #endif
 463
 464     return max;
 465 }
 466
 467 void
 468 MemObject::setNoDelay(bool const newValue)
 469 {
 470 #if USE_DELAY_POOLS
 471
 472     for (dlink_node *node = clients.head; node; node = node->next) {
 473         store_client *sc = (store_client *) node->data;
 474         sc->delayId.setNoDelay(newValue);
 475     }
 476
 477 #endif
 478 }
 479
 480 void
 481 MemObject::delayRead(DeferredRead const &aRead)
 482 {
 483     deferredReads.delayRead(aRead);
 484 }
 485
 486 void
 487 MemObject::kickReads()
 488 {
 489     deferredReads.kickReads(-1);
 490 }
 491
 492 #if USE_DELAY_POOLS
 493 DelayId
 494 MemObject::mostBytesAllowed() const
 495 {
 496     int j;
 497     int jmax = -1;
 498     DelayId result;
 499
 500     for (dlink_node *node = clients.head; node; node = node->next) {
 501         store_client *sc = (store_client *) node->data;
 502 #if 0
 503         /* This test is invalid because the client may be writing data
 504          * and thus will want data immediately.
 505          * If we include the test, there is a race condition when too much
 506          * data is read - if all sc's are writing when a read is scheduled.
 507          * XXX: fixme.
 508          */
 509
 510         if (!sc->callbackPending())
 511             /* not waiting for more data */
 512             continue;
 513
 514 #endif
 515
 516         j = sc->delayId.bytesWanted(0, sc->copyInto.length);
 517
 518         if (j > jmax) {
 519             jmax = j;
 520             result = sc->delayId;
 521         }
 522     }
 523
 524     return result;
 525 }
 526
 527 #endif
 528
 529 int64_t
 530 MemObject::availableForSwapOut() const
 531 {
 532     return endOffset() - swapout.queue_offset;
 533 }