src/MemObject.cc

   1
   2 /*
   3  * DEBUG: section 19    Store Memory Primitives
   4  * AUTHOR: Robert Collins
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  */
  33
  34 #include "squid.h"
  35 #include "comm/Connection.h"
  36 #include "Generic.h"
  37 #include "globals.h"
  38 #include "HttpReply.h"
  39 #include "HttpRequest.h"
  40 #include "MemBuf.h"
  41 #include "MemObject.h"
  42 #include "profiler/Profiler.h"
  43 #include "SquidConfig.h"
  44 #include "Store.h"
  45 #include "StoreClient.h"
  46
  47 #if USE_DELAY_POOLS
  48 #include "DelayPools.h"
  49 #endif
  50
  51 /* TODO: make this global or private */
  52 #if URL_CHECKSUM_DEBUG
  53 static unsigned int url_checksum(const char *url);
  54 unsigned int
  55 url_checksum(const char *url)
  56 {
  57     unsigned int ck;
  58     SquidMD5_CTX M;
  59     static unsigned char digest[16];
  60     SquidMD5Init(&M);
  61     SquidMD5Update(&M, (unsigned char *) url, strlen(url));
  62     SquidMD5Final(digest, &M);
  63     memcpy(&ck, digest, sizeof(ck));
  64     return ck;
  65 }
  66
  67 #endif
  68
  69 RemovalPolicy * mem_policy = NULL;
  70
  71 size_t
  72 MemObject::inUseCount()
  73 {
  74     return Pool().inUseCount();
  75 }
  76
  77 const char *
  78 MemObject::storeId() const
  79 {
  80     if (!storeId_.size()) {
  81         debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value");
  82         dump();
  83         storeId_ = "[unknown_URI]";
  84     }
  85     return storeId_.termedBuf();
  86 }
  87
  88 const char *
  89 MemObject::logUri() const
  90 {
  91     return logUri_.size() ? logUri_.termedBuf() : storeId();
  92 }
  93
  94 bool
  95 MemObject::hasUris() const
  96 {
  97     return storeId_.size();
  98 }
  99
 100 void
 101 MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod)
 102 {
 103     storeId_ = aStoreId;
 104
 105     // fast pointer comparison for a common storeCreateEntry(url,url,...) case
 106     if (!aLogUri || aLogUri == aStoreId)
 107         logUri_.clean(); // use storeId_ by default to minimize copying
 108     else
 109         logUri_ = aLogUri;
 110
 111     method = aMethod;
 112
 113 #if URL_CHECKSUM_DEBUG
 114     chksum = url_checksum(urlXXX());
 115 #endif
 116 }
 117
 118 MemObject::MemObject(): smpCollapsed(false)
 119 {
 120     debugs(20, 3, HERE << "new MemObject " << this);
 121     _reply = new HttpReply;
 122     HTTPMSGLOCK(_reply);
 123
 124     object_sz = -1;
 125
 126     /* XXX account log_url */
 127
 128     swapout.decision = SwapOut::swNeedsCheck;
 129 }
 130
 131 MemObject::~MemObject()
 132 {
 133     debugs(20, 3, HERE << "del MemObject " << this);
 134     const Ctx ctx = ctx_enter(hasUris() ? urlXXX() : "[unknown_ctx]");
 135
 136 #if URL_CHECKSUM_DEBUG
 137     checkUrlChecksum();
 138 #endif
 139
 140     if (!shutting_down) { // Store::Root() is FATALly missing during shutdown
 141         assert(xitTable.index < 0);
 142         assert(memCache.index < 0);
 143         assert(swapout.sio == NULL);
 144     }
 145
 146     data_hdr.freeContent();
 147
 148 #if 0
 149     /*
 150      * There is no way to abort FD-less clients, so they might
 151      * still have mem->clients set.
 152      */
 153     assert(clients.head == NULL);
 154
 155 #endif
 156
 157     HTTPMSGUNLOCK(_reply);
 158
 159     HTTPMSGUNLOCK(request);
 160
 161     ctx_exit(ctx);              /* must exit before we free mem->url */
 162
 163     safe_free(vary_headers);
 164 }
 165
 166 void
 167 MemObject::unlinkRequest()
 168 {
 169     HTTPMSGUNLOCK(request);
 170 }
 171
 172 void
 173 MemObject::write(const StoreIOBuffer &writeBuffer)
 174 {
 175     PROF_start(MemObject_write);
 176     debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
 177
 178     /* We don't separate out mime headers yet, so ensure that the first
 179      * write is at offset 0 - where they start
 180      */
 181     assert (data_hdr.endOffset() || writeBuffer.offset == 0);
 182
 183     assert (data_hdr.write (writeBuffer));
 184     PROF_stop(MemObject_write);
 185 }
 186
 187 void
 188 MemObject::dump() const
 189 {
 190     data_hdr.dump();
 191 #if 0
 192     /* do we want this one? */
 193     debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
 194 #endif
 195
 196     debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec  << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
 197     debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
 198     debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
 199     debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
 200     debugs(20, DBG_IMPORTANT, "MemObject->reply: " << _reply);
 201     debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
 202     debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_);
 203     debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_);
 204 }
 205
 206 HttpReply const *
 207 MemObject::getReply() const
 208 {
 209     return _reply;
 210 }
 211
 212 void
 213 MemObject::replaceHttpReply(HttpReply *newrep)
 214 {
 215     HTTPMSGUNLOCK(_reply);
 216     _reply = newrep;
 217     HTTPMSGLOCK(_reply);
 218 }
 219
 220 struct LowestMemReader : public unary_function<store_client, void> {
 221     LowestMemReader(int64_t seed):current(seed) {}
 222
 223     void operator() (store_client const &x) {
 224         if (x.memReaderHasLowerOffset(current))
 225             current = x.copyInto.offset;
 226     }
 227
 228     int64_t current;
 229 };
 230
 231 struct StoreClientStats : public unary_function<store_client, void> {
 232     StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
 233
 234     void operator()(store_client const &x) {
 235         x.dumpStats(where, index);
 236         ++index;
 237     }
 238
 239     MemBuf *where;
 240     size_t index;
 241 };
 242
 243 void
 244 MemObject::stat(MemBuf * mb) const
 245 {
 246     mb->Printf("\t%s %s\n",
 247                RequestMethodStr(method), logUri());
 248     if (vary_headers)
 249         mb->Printf("\tvary_headers: %s\n", vary_headers);
 250     mb->Printf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
 251     mb->Printf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
 252     mb->Printf("\tswapout: %" PRId64 " bytes queued\n",
 253                swapout.queue_offset);
 254
 255     if (swapout.sio.getRaw())
 256         mb->Printf("\tswapout: %" PRId64 " bytes written\n",
 257                    (int64_t) swapout.sio->offset());
 258
 259     if (xitTable.index >= 0)
 260         mb->Printf("\ttransient index: %d state: %d\n",
 261                    xitTable.index, xitTable.io);
 262     if (memCache.index >= 0)
 263         mb->Printf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n",
 264                    memCache.index, memCache.io, memCache.offset);
 265     if (object_sz >= 0)
 266         mb->Printf("\tobject_sz: %" PRId64 "\n", object_sz);
 267     if (smpCollapsed)
 268         mb->Printf("\tsmp-collapsed\n");
 269
 270     StoreClientStats statsVisitor(mb);
 271
 272     for_each<StoreClientStats>(clients, statsVisitor);
 273 }
 274
 275 int64_t
 276 MemObject::endOffset () const
 277 {
 278     return data_hdr.endOffset();
 279 }
 280
 281 void
 282 MemObject::markEndOfReplyHeaders()
 283 {
 284     const int hdr_sz = endOffset();
 285     assert(hdr_sz >= 0);
 286     assert(_reply);
 287     _reply->hdr_sz = hdr_sz;
 288 }
 289
 290 int64_t
 291 MemObject::size() const
 292 {
 293     if (object_sz < 0)
 294         return endOffset();
 295
 296     return object_sz;
 297 }
 298
 299 int64_t
 300 MemObject::expectedReplySize() const
 301 {
 302     debugs(20, 7, HERE << "object_sz: " << object_sz);
 303     if (object_sz >= 0) // complete() has been called; we know the exact answer
 304         return object_sz;
 305
 306     if (_reply) {
 307         const int64_t clen = _reply->bodySize(method);
 308         debugs(20, 7, HERE << "clen: " << clen);
 309         if (clen >= 0 && _reply->hdr_sz > 0) // yuck: HttpMsg sets hdr_sz to 0
 310             return clen + _reply->hdr_sz;
 311     }
 312
 313     return -1; // not enough information to predict
 314 }
 315
 316 void
 317 MemObject::reset()
 318 {
 319     assert(swapout.sio == NULL);
 320     data_hdr.freeContent();
 321     inmem_lo = 0;
 322     /* Should we check for clients? */
 323 }
 324
 325 int64_t
 326 MemObject::lowestMemReaderOffset() const
 327 {
 328     LowestMemReader lowest (endOffset() + 1);
 329
 330     for_each <LowestMemReader>(clients, lowest);
 331
 332     return lowest.current;
 333 }
 334
 335 /* XXX: This is wrong. It breaks *badly* on range combining */
 336 bool
 337 MemObject::readAheadPolicyCanRead() const
 338 {
 339     const bool canRead = endOffset() - getReply()->hdr_sz <
 340                          lowestMemReaderOffset() + Config.readAheadGap;
 341
 342     if (!canRead) {
 343         debugs(19, 9, "no: " << endOffset() << '-' << getReply()->hdr_sz <<
 344                " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap);
 345     }
 346
 347     return canRead;
 348 }
 349
 350 void
 351 MemObject::addClient(store_client *aClient)
 352 {
 353     ++nclients;
 354     dlinkAdd(aClient, &aClient->node, &clients);
 355 }
 356
 357 #if URL_CHECKSUM_DEBUG
 358 void
 359 MemObject::checkUrlChecksum () const
 360 {
 361     assert(chksum == url_checksum(urlXXX()));
 362 }
 363
 364 #endif
 365
 366 /*
 367  * How much of the object data is on the disk?
 368  */
 369 int64_t
 370 MemObject::objectBytesOnDisk() const
 371 {
 372     /*
 373      * NOTE: storeOffset() represents the disk file size,
 374      * not the amount of object data on disk.
 375      *
 376      * If we don't have at least 'swap_hdr_sz' bytes
 377      * then none of the object data is on disk.
 378      *
 379      * This should still be safe if swap_hdr_sz == 0,
 380      * meaning we haven't even opened the swapout file
 381      * yet.
 382      */
 383
 384     if (swapout.sio.getRaw() == NULL)
 385         return 0;
 386
 387     int64_t nwritten = swapout.sio->offset();
 388
 389     if (nwritten <= (int64_t)swap_hdr_sz)
 390         return 0;
 391
 392     return (nwritten - swap_hdr_sz);
 393 }
 394
 395 int64_t
 396 MemObject::policyLowestOffsetToKeep(bool swap) const
 397 {
 398     /*
 399      * Careful.  lowest_offset can be greater than endOffset(), such
 400      * as in the case of a range request.
 401      */
 402     int64_t lowest_offset = lowestMemReaderOffset();
 403
 404     if (endOffset() < lowest_offset ||
 405             endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
 406             (swap && !Config.onoff.memory_cache_first))
 407         return lowest_offset;
 408
 409     return inmem_lo;
 410 }
 411
 412 void
 413 MemObject::trimSwappable()
 414 {
 415     int64_t new_mem_lo = policyLowestOffsetToKeep(1);
 416     /*
 417      * We should only free up to what we know has been written
 418      * to disk, not what has been queued for writing.  Otherwise
 419      * there will be a chunk of the data which is not in memory
 420      * and is not yet on disk.
 421      * The -1 makes sure the page isn't freed until storeSwapOut has
 422      * walked to the next page.
 423      */
 424     int64_t on_disk;
 425
 426     if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
 427         new_mem_lo = on_disk - 1;
 428
 429     if (new_mem_lo == -1)
 430         new_mem_lo = 0; /* the above might become -1 */
 431
 432     data_hdr.freeDataUpto(new_mem_lo);
 433
 434     inmem_lo = new_mem_lo;
 435 }
 436
 437 void
 438 MemObject::trimUnSwappable()
 439 {
 440     if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) {
 441         assert (new_mem_lo > 0);
 442         data_hdr.freeDataUpto(new_mem_lo);
 443         inmem_lo = new_mem_lo;
 444     } // else we should not trim anything at this time
 445 }
 446
 447 bool
 448 MemObject::isContiguous() const
 449 {
 450     bool result = data_hdr.hasContigousContentRange (Range<int64_t>(inmem_lo, endOffset()));
 451     /* XXX : make this higher level */
 452     debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
 453     return result;
 454 }
 455
 456 int
 457 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
 458 {
 459 #if USE_DELAY_POOLS
 460     if (!ignoreDelayPools) {
 461         /* identify delay id with largest allowance */
 462         DelayId largestAllowance = mostBytesAllowed ();
 463         return largestAllowance.bytesWanted(0, max);
 464     }
 465 #endif
 466
 467     return max;
 468 }
 469
 470 void
 471 MemObject::setNoDelay(bool const newValue)
 472 {
 473 #if USE_DELAY_POOLS
 474
 475     for (dlink_node *node = clients.head; node; node = node->next) {
 476         store_client *sc = (store_client *) node->data;
 477         sc->delayId.setNoDelay(newValue);
 478     }
 479
 480 #endif
 481 }
 482
 483 void
 484 MemObject::delayRead(DeferredRead const &aRead)
 485 {
 486     deferredReads.delayRead(aRead);
 487 }
 488
 489 void
 490 MemObject::kickReads()
 491 {
 492     deferredReads.kickReads(-1);
 493 }
 494
 495 #if USE_DELAY_POOLS
 496 DelayId
 497 MemObject::mostBytesAllowed() const
 498 {
 499     int j;
 500     int jmax = -1;
 501     DelayId result;
 502
 503     for (dlink_node *node = clients.head; node; node = node->next) {
 504         store_client *sc = (store_client *) node->data;
 505 #if 0
 506         /* This test is invalid because the client may be writing data
 507          * and thus will want data immediately.
 508          * If we include the test, there is a race condition when too much
 509          * data is read - if all sc's are writing when a read is scheduled.
 510          * XXX: fixme.
 511          */
 512
 513         if (!sc->callbackPending())
 514             /* not waiting for more data */
 515             continue;
 516
 517 #endif
 518
 519         j = sc->delayId.bytesWanted(0, sc->copyInto.length);
 520
 521         if (j > jmax) {
 522             jmax = j;
 523             result = sc->delayId;
 524         }
 525     }
 526
 527     return result;
 528 }
 529
 530 #endif
 531
 532 int64_t
 533 MemObject::availableForSwapOut() const
 534 {
 535     return endOffset() - swapout.queue_offset;
 536 }