src/MemObject.cc

   1
   2 /*
   3  * DEBUG: section 19    Store Memory Primitives
   4  * AUTHOR: Robert Collins
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  */
  33
  34 #include "squid.h"
  35 #include "comm/Connection.h"
  36 #include "Generic.h"
  37 #include "globals.h"
  38 #include "HttpReply.h"
  39 #include "HttpRequest.h"
  40 #include "MemBuf.h"
  41 #include "MemObject.h"
  42 #include "profiler/Profiler.h"
  43 #include "SquidConfig.h"
  44 #include "Store.h"
  45 #include "StoreClient.h"
  46
  47 #if USE_DELAY_POOLS
  48 #include "DelayPools.h"
  49 #endif
  50
  51 /* TODO: make this global or private */
  52 #if URL_CHECKSUM_DEBUG
  53 static unsigned int url_checksum(const char *url);
  54 unsigned int
  55 url_checksum(const char *url)
  56 {
  57     unsigned int ck;
  58     SquidMD5_CTX M;
  59     static unsigned char digest[16];
  60     SquidMD5Init(&M);
  61     SquidMD5Update(&M, (unsigned char *) url, strlen(url));
  62     SquidMD5Final(digest, &M);
  63     memcpy(&ck, digest, sizeof(ck));
  64     return ck;
  65 }
  66
  67 #endif
  68
  69 RemovalPolicy * mem_policy = NULL;
  70
  71 size_t
  72 MemObject::inUseCount()
  73 {
  74     return Pool().inUseCount();
  75 }
  76
  77 void
  78 MemObject::resetUrls(char const *aUrl, char const *aLog_url)
  79 {
  80     safe_free(url);
  81     safe_free(log_url);    /* XXX account log_url */
  82     log_url = xstrdup(aLog_url);
  83     url = xstrdup(aUrl);
  84 }
  85
  86 MemObject::MemObject(char const *aUrl, char const *aLog_url)
  87 {
  88     debugs(20, 3, HERE << "new MemObject " << this);
  89     HttpReply *rep = new HttpReply;
  90
  91     _reply  = HTTPMSGLOCK(rep);
  92     url = xstrdup(aUrl);
  93
  94 #if URL_CHECKSUM_DEBUG
  95
  96     chksum = url_checksum(url);
  97
  98 #endif
  99
 100     log_url = xstrdup(aLog_url);
 101
 102     object_sz = -1;
 103
 104     /* XXX account log_url */
 105
 106     swapout.decision = SwapOut::swNeedsCheck;
 107 }
 108
 109 MemObject::~MemObject()
 110 {
 111     debugs(20, 3, HERE << "del MemObject " << this);
 112     const Ctx ctx = ctx_enter(url);
 113 #if URL_CHECKSUM_DEBUG
 114
 115     assert(chksum == url_checksum(url));
 116 #endif
 117
 118     if (!shutting_down)
 119         assert(swapout.sio == NULL);
 120
 121     data_hdr.freeContent();
 122
 123 #if 0
 124     /*
 125      * There is no way to abort FD-less clients, so they might
 126      * still have mem->clients set.
 127      */
 128     assert(clients.head == NULL);
 129
 130 #endif
 131
 132     HTTPMSGUNLOCK(_reply);
 133
 134     HTTPMSGUNLOCK(request);
 135
 136     ctx_exit(ctx);              /* must exit before we free mem->url */
 137
 138     safe_free(url);
 139
 140     safe_free(log_url);    /* XXX account log_url */
 141
 142     safe_free(vary_headers);
 143 }
 144
 145 void
 146 MemObject::unlinkRequest()
 147 {
 148     HTTPMSGUNLOCK(request);
 149 }
 150
 151 void
 152 MemObject::write ( StoreIOBuffer writeBuffer, STMCB *callback, void *callbackData)
 153 {
 154     PROF_start(MemObject_write);
 155     debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length);
 156
 157     /* the offset is into the content, not the headers */
 158     writeBuffer.offset += (_reply ? _reply->hdr_sz : 0);
 159
 160     /* We don't separate out mime headers yet, so ensure that the first
 161      * write is at offset 0 - where they start
 162      */
 163     assert (data_hdr.endOffset() || writeBuffer.offset == 0);
 164
 165     assert (data_hdr.write (writeBuffer));
 166     callback (callbackData, writeBuffer);
 167     PROF_stop(MemObject_write);
 168 }
 169
 170 void
 171 MemObject::dump() const
 172 {
 173     data_hdr.dump();
 174 #if 0
 175     /* do we want this one? */
 176     debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0));
 177 #endif
 178
 179     debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec  << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec);
 180     debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset());
 181     debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo);
 182     debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients);
 183     debugs(20, DBG_IMPORTANT, "MemObject->reply: " << _reply);
 184     debugs(20, DBG_IMPORTANT, "MemObject->request: " << request);
 185     debugs(20, DBG_IMPORTANT, "MemObject->log_url: " << log_url << " " << checkNullString(log_url));
 186 }
 187
 188 HttpReply const *
 189 MemObject::getReply() const
 190 {
 191     return _reply;
 192 }
 193
 194 void
 195 MemObject::replaceHttpReply(HttpReply *newrep)
 196 {
 197     HTTPMSGUNLOCK(_reply);
 198     _reply = HTTPMSGLOCK(newrep);
 199 }
 200
 201 struct LowestMemReader : public unary_function<store_client, void> {
 202     LowestMemReader(int64_t seed):current(seed) {}
 203
 204     void operator() (store_client const &x) {
 205         if (x.memReaderHasLowerOffset(current))
 206             current = x.copyInto.offset;
 207     }
 208
 209     int64_t current;
 210 };
 211
 212 struct StoreClientStats : public unary_function<store_client, void> {
 213     StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {}
 214
 215     void operator()(store_client const &x) {
 216         x.dumpStats(where, index);
 217         ++index;
 218     }
 219
 220     MemBuf *where;
 221     size_t index;
 222 };
 223
 224 void
 225 MemObject::stat(MemBuf * mb) const
 226 {
 227     mb->Printf("\t%s %s\n",
 228                RequestMethodStr(method), log_url);
 229     if (vary_headers)
 230         mb->Printf("\tvary_headers: %s\n", vary_headers);
 231     mb->Printf("\tinmem_lo: %" PRId64 "\n", inmem_lo);
 232     mb->Printf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset());
 233     mb->Printf("\tswapout: %" PRId64 " bytes queued\n",
 234                swapout.queue_offset);
 235
 236     if (swapout.sio.getRaw())
 237         mb->Printf("\tswapout: %" PRId64 " bytes written\n",
 238                    (int64_t) swapout.sio->offset());
 239
 240     StoreClientStats statsVisitor(mb);
 241
 242     for_each<StoreClientStats>(clients, statsVisitor);
 243 }
 244
 245 int64_t
 246 MemObject::endOffset () const
 247 {
 248     return data_hdr.endOffset();
 249 }
 250
 251 void
 252 MemObject::markEndOfReplyHeaders()
 253 {
 254     const int hdr_sz = endOffset();
 255     assert(hdr_sz >= 0);
 256     assert(_reply);
 257     _reply->hdr_sz = hdr_sz;
 258 }
 259
 260 int64_t
 261 MemObject::size() const
 262 {
 263     if (object_sz < 0)
 264         return endOffset();
 265
 266     return object_sz;
 267 }
 268
 269 int64_t
 270 MemObject::expectedReplySize() const
 271 {
 272     debugs(20, 7, HERE << "object_sz: " << object_sz);
 273     if (object_sz >= 0) // complete() has been called; we know the exact answer
 274         return object_sz;
 275
 276     if (_reply) {
 277         const int64_t clen = _reply->bodySize(method);
 278         debugs(20, 7, HERE << "clen: " << clen);
 279         if (clen >= 0 && _reply->hdr_sz > 0) // yuck: HttpMsg sets hdr_sz to 0
 280             return clen + _reply->hdr_sz;
 281     }
 282
 283     return -1; // not enough information to predict
 284 }
 285
 286 void
 287 MemObject::reset()
 288 {
 289     assert(swapout.sio == NULL);
 290     data_hdr.freeContent();
 291     inmem_lo = 0;
 292     /* Should we check for clients? */
 293 }
 294
 295 int64_t
 296 MemObject::lowestMemReaderOffset() const
 297 {
 298     LowestMemReader lowest (endOffset() + 1);
 299
 300     for_each <LowestMemReader>(clients, lowest);
 301
 302     return lowest.current;
 303 }
 304
 305 /* XXX: This is wrong. It breaks *badly* on range combining */
 306 bool
 307 MemObject::readAheadPolicyCanRead() const
 308 {
 309     return endOffset() - getReply()->hdr_sz < lowestMemReaderOffset() + Config.readAheadGap;
 310 }
 311
 312 void
 313 MemObject::addClient(store_client *aClient)
 314 {
 315     ++nclients;
 316     dlinkAdd(aClient, &aClient->node, &clients);
 317 }
 318
 319 #if URL_CHECKSUM_DEBUG
 320 void
 321 MemObject::checkUrlChecksum () const
 322 {
 323     assert(chksum == url_checksum(url));
 324 }
 325
 326 #endif
 327
 328 /*
 329  * How much of the object data is on the disk?
 330  */
 331 int64_t
 332 MemObject::objectBytesOnDisk() const
 333 {
 334     /*
 335      * NOTE: storeOffset() represents the disk file size,
 336      * not the amount of object data on disk.
 337      *
 338      * If we don't have at least 'swap_hdr_sz' bytes
 339      * then none of the object data is on disk.
 340      *
 341      * This should still be safe if swap_hdr_sz == 0,
 342      * meaning we haven't even opened the swapout file
 343      * yet.
 344      */
 345
 346     if (swapout.sio.getRaw() == NULL)
 347         return 0;
 348
 349     int64_t nwritten = swapout.sio->offset();
 350
 351     if (nwritten <= (int64_t)swap_hdr_sz)
 352         return 0;
 353
 354     return (nwritten - swap_hdr_sz);
 355 }
 356
 357 int64_t
 358 MemObject::policyLowestOffsetToKeep(bool swap) const
 359 {
 360     /*
 361      * Careful.  lowest_offset can be greater than endOffset(), such
 362      * as in the case of a range request.
 363      */
 364     int64_t lowest_offset = lowestMemReaderOffset();
 365
 366     if (endOffset() < lowest_offset ||
 367             endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize ||
 368             (swap && !Config.onoff.memory_cache_first))
 369         return lowest_offset;
 370
 371     return inmem_lo;
 372 }
 373
 374 void
 375 MemObject::trimSwappable()
 376 {
 377     int64_t new_mem_lo = policyLowestOffsetToKeep(1);
 378     /*
 379      * We should only free up to what we know has been written
 380      * to disk, not what has been queued for writing.  Otherwise
 381      * there will be a chunk of the data which is not in memory
 382      * and is not yet on disk.
 383      * The -1 makes sure the page isn't freed until storeSwapOut has
 384      * walked to the next page.
 385      */
 386     int64_t on_disk;
 387
 388     if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo)
 389         new_mem_lo = on_disk - 1;
 390
 391     if (new_mem_lo == -1)
 392         new_mem_lo = 0; /* the above might become -1 */
 393
 394     data_hdr.freeDataUpto(new_mem_lo);
 395
 396     inmem_lo = new_mem_lo;
 397 }
 398
 399 void
 400 MemObject::trimUnSwappable()
 401 {
 402     int64_t new_mem_lo = policyLowestOffsetToKeep(0);
 403     assert (new_mem_lo > 0);
 404
 405     data_hdr.freeDataUpto(new_mem_lo);
 406     inmem_lo = new_mem_lo;
 407 }
 408
 409 bool
 410 MemObject::isContiguous() const
 411 {
 412     bool result = data_hdr.hasContigousContentRange (Range<int64_t>(inmem_lo, endOffset()));
 413     /* XXX : make this higher level */
 414     debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false"));
 415     return result;
 416 }
 417
 418 int
 419 MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const
 420 {
 421 #if USE_DELAY_POOLS
 422     if (!ignoreDelayPools) {
 423         /* identify delay id with largest allowance */
 424         DelayId largestAllowance = mostBytesAllowed ();
 425         return largestAllowance.bytesWanted(0, max);
 426     }
 427 #endif
 428
 429     return max;
 430 }
 431
 432 void
 433 MemObject::setNoDelay(bool const newValue)
 434 {
 435 #if USE_DELAY_POOLS
 436
 437     for (dlink_node *node = clients.head; node; node = node->next) {
 438         store_client *sc = (store_client *) node->data;
 439         sc->delayId.setNoDelay(newValue);
 440     }
 441
 442 #endif
 443 }
 444
 445 void
 446 MemObject::delayRead(DeferredRead const &aRead)
 447 {
 448     deferredReads.delayRead(aRead);
 449 }
 450
 451 void
 452 MemObject::kickReads()
 453 {
 454     deferredReads.kickReads(-1);
 455 }
 456
 457 #if USE_DELAY_POOLS
 458 DelayId
 459 MemObject::mostBytesAllowed() const
 460 {
 461     int j;
 462     int jmax = -1;
 463     DelayId result;
 464
 465     for (dlink_node *node = clients.head; node; node = node->next) {
 466         store_client *sc = (store_client *) node->data;
 467 #if 0
 468         /* This test is invalid because the client may be writing data
 469          * and thus will want data immediately.
 470          * If we include the test, there is a race condition when too much
 471          * data is read - if all sc's are writing when a read is scheduled.
 472          * XXX: fixme.
 473          */
 474
 475         if (!sc->callbackPending())
 476             /* not waiting for more data */
 477             continue;
 478
 479 #endif
 480
 481         j = sc->delayId.bytesWanted(0, sc->copyInto.length);
 482
 483         if (j > jmax) {
 484             jmax = j;
 485             result = sc->delayId;
 486         }
 487     }
 488
 489     return result;
 490 }
 491
 492 #endif
 493
 494 int64_t
 495 MemObject::availableForSwapOut() const
 496 {
 497     return endOffset() - swapout.queue_offset;
 498 }