src/store.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 20    Storage Manager
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "CacheManager.h"
  38 #include "comm/Connection.h"
  39 #include "event.h"
  40 #include "fde.h"
  41 #include "Store.h"
  42 #include "mgr/Registration.h"
  43 #include "StoreClient.h"
  44 #include "stmem.h"
  45 #include "HttpReply.h"
  46 #include "HttpRequest.h"
  47 #include "MemObject.h"
  48 #include "mem_node.h"
  49 #include "StoreMeta.h"
  50 #include "SwapDir.h"
  51 #if USE_DELAY_POOLS
  52 #include "DelayPools.h"
  53 #endif
  54 #include "Stack.h"
  55 #include "SquidTime.h"
  56 #include "swap_log_op.h"
  57 #include "mgr/StoreIoAction.h"
  58
  59 static STMCB storeWriteComplete;
  60
  61 #define REBUILD_TIMESTAMP_DELTA_MAX 2
  62
  63 #define STORE_IN_MEM_BUCKETS            (229)
  64
  65
  66 /** \todo Convert these string constants to enum string-arrays generated */
  67
  68 const char *memStatusStr[] = {
  69     "NOT_IN_MEMORY",
  70     "IN_MEMORY"
  71 };
  72
  73 const char *pingStatusStr[] = {
  74     "PING_NONE",
  75     "PING_WAITING",
  76     "PING_DONE"
  77 };
  78
  79 const char *storeStatusStr[] = {
  80     "STORE_OK",
  81     "STORE_PENDING"
  82 };
  83
  84 const char *swapStatusStr[] = {
  85     "SWAPOUT_NONE",
  86     "SWAPOUT_WRITING",
  87     "SWAPOUT_DONE"
  88 };
  89
  90
  91 /*
  92  * This defines an repl type
  93  */
  94
  95 typedef struct _storerepl_entry storerepl_entry_t;
  96
  97 struct _storerepl_entry {
  98     const char *typestr;
  99     REMOVALPOLICYCREATE *create;
 100 };
 101
 102 static storerepl_entry_t *storerepl_list = NULL;
 103
 104
 105 /*
 106  * local function prototypes
 107  */
 108 static int getKeyCounter(void);
 109 static OBJH storeCheckCachableStats;
 110 static EVH storeLateRelease;
 111
 112 /*
 113  * local variables
 114  */
 115 static Stack<StoreEntry*> LateReleaseStack;
 116 MemAllocator *StoreEntry::pool = NULL;
 117
 118 StorePointer Store::CurrentRoot = NULL;
 119
 120 void
 121 Store::Root(Store * aRoot)
 122 {
 123     CurrentRoot = aRoot;
 124 }
 125
 126 void
 127 Store::Root(StorePointer aRoot)
 128 {
 129     Root(aRoot.getRaw());
 130 }
 131
 132 void
 133 Store::Stats(StoreEntry * output)
 134 {
 135     assert (output);
 136     Root().stat(*output);
 137 }
 138
 139 void
 140 Store::create()
 141 {}
 142
 143 void
 144 Store::diskFull()
 145 {}
 146
 147 void
 148 Store::sync()
 149 {}
 150
 151 void
 152 Store::unlink (StoreEntry &anEntry)
 153 {
 154     fatal("Store::unlink on invalid Store\n");
 155 }
 156
 157 void *
 158 StoreEntry::operator new (size_t bytecount)
 159 {
 160     assert (bytecount == sizeof (StoreEntry));
 161
 162     if (!pool) {
 163         pool = memPoolCreate ("StoreEntry", bytecount);
 164         pool->setChunkSize(2048 * 1024);
 165     }
 166
 167     return pool->alloc();
 168 }
 169
 170 void
 171 StoreEntry::operator delete (void *address)
 172 {
 173     pool->freeOne(address);
 174 }
 175
 176 void
 177 StoreEntry::makePublic()
 178 {
 179     /* This object can be cached for a long time */
 180
 181     if (EBIT_TEST(flags, ENTRY_CACHABLE))
 182         setPublicKey();
 183 }
 184
 185 void
 186 StoreEntry::makePrivate()
 187 {
 188     /* This object should never be cached at all */
 189     expireNow();
 190     releaseRequest(); /* delete object when not used */
 191     /* releaseRequest clears ENTRY_CACHABLE flag */
 192 }
 193
 194 void
 195 StoreEntry::cacheNegatively()
 196 {
 197     /* This object may be negatively cached */
 198     negativeCache();
 199
 200     if (EBIT_TEST(flags, ENTRY_CACHABLE))
 201         setPublicKey();
 202 }
 203
 204 size_t
 205 StoreEntry::inUseCount()
 206 {
 207     if (!pool)
 208         return 0;
 209     return pool->getInUseCount();
 210 }
 211
 212 const char *
 213 StoreEntry::getMD5Text() const
 214 {
 215     return storeKeyText((const cache_key *)key);
 216 }
 217
 218 #include "comm.h"
 219
 220 void
 221 StoreEntry::DeferReader(void *theContext, CommRead const &aRead)
 222 {
 223     StoreEntry *anEntry = (StoreEntry *)theContext;
 224     anEntry->delayAwareRead(aRead.conn,
 225                             aRead.buf,
 226                             aRead.len,
 227                             aRead.callback);
 228 }
 229
 230 void
 231 StoreEntry::delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
 232 {
 233     size_t amountToRead = bytesWanted(Range<size_t>(0, len));
 234     /* sketch: readdeferer* = getdeferer.
 235      * ->deferRead (fd, buf, len, callback, DelayAwareRead, this)
 236      */
 237
 238     if (amountToRead == 0) {
 239         assert (mem_obj);
 240         /* read ahead limit */
 241         /* Perhaps these two calls should both live in MemObject */
 242 #if USE_DELAY_POOLS
 243         if (!mem_obj->readAheadPolicyCanRead()) {
 244 #endif
 245             mem_obj->delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 246             return;
 247 #if USE_DELAY_POOLS
 248         }
 249
 250         /* delay id limit */
 251         mem_obj->mostBytesAllowed().delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 252         return;
 253
 254 #endif
 255
 256     }
 257
 258     comm_read(conn, buf, amountToRead, callback);
 259 }
 260
 261 size_t
 262 StoreEntry::bytesWanted (Range<size_t> const aRange) const
 263 {
 264     assert (aRange.size());
 265
 266     if (mem_obj == NULL)
 267         return aRange.end - 1;
 268
 269 #if URL_CHECKSUM_DEBUG
 270
 271     mem_obj->checkUrlChecksum();
 272
 273 #endif
 274
 275     /* Always read *something* here - we haven't got the header yet */
 276     if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT))
 277         return aRange.end - 1;
 278
 279     if (!mem_obj->readAheadPolicyCanRead())
 280         return 0;
 281
 282     return mem_obj->mostBytesWanted(aRange.end - 1);
 283 }
 284
 285 bool
 286 StoreEntry::checkDeferRead(int fd) const
 287 {
 288     return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
 289 }
 290
 291 void
 292 StoreEntry::setNoDelay (bool const newValue)
 293 {
 294     if (mem_obj)
 295         mem_obj->setNoDelay(newValue);
 296 }
 297
 298 store_client_t
 299 StoreEntry::storeClientType() const
 300 {
 301     /* The needed offset isn't in memory
 302      * XXX TODO: this is wrong for range requests
 303      * as the needed offset may *not* be 0, AND
 304      * offset 0 in the memory object is the HTTP headers.
 305      */
 306
 307     if (mem_obj->inmem_lo)
 308         return STORE_DISK_CLIENT;
 309
 310     if (EBIT_TEST(flags, ENTRY_ABORTED)) {
 311         /* I don't think we should be adding clients to aborted entries */
 312         debugs(20, 1, "storeClientType: adding to ENTRY_ABORTED entry");
 313         return STORE_MEM_CLIENT;
 314     }
 315
 316     if (store_status == STORE_OK) {
 317         /* the object has completed. */
 318
 319         if (mem_obj->inmem_lo == 0 && !isEmpty()) {
 320             if (swap_status == SWAPOUT_DONE) {
 321                 if (mem_obj->endOffset() == mem_obj->object_sz) {
 322                     /* hot object fully swapped in */
 323                     return STORE_MEM_CLIENT;
 324                 }
 325             } else {
 326                 /* Memory-only, or currently being swapped out */
 327                 return STORE_MEM_CLIENT;
 328             }
 329         }
 330         return STORE_DISK_CLIENT;
 331     }
 332
 333     /* here and past, entry is STORE_PENDING */
 334     /*
 335      * If this is the first client, let it be the mem client
 336      */
 337     if (mem_obj->nclients == 1)
 338         return STORE_MEM_CLIENT;
 339
 340     /*
 341      * If there is no disk file to open yet, we must make this a
 342      * mem client.  If we can't open the swapin file before writing
 343      * to the client, there is no guarantee that we will be able
 344      * to open it later when we really need it.
 345      */
 346     if (swap_status == SWAPOUT_NONE)
 347         return STORE_MEM_CLIENT;
 348
 349     /*
 350      * otherwise, make subsequent clients read from disk so they
 351      * can not delay the first, and vice-versa.
 352      */
 353     return STORE_DISK_CLIENT;
 354 }
 355
 356 StoreEntry::StoreEntry()
 357 {
 358     debugs(20, 3, HERE << "new StoreEntry " << this);
 359     mem_obj = NULL;
 360
 361     expires = lastmod = lastref = timestamp = -1;
 362
 363     swap_filen = -1;
 364     swap_dirn = -1;
 365 }
 366
 367 StoreEntry::StoreEntry(const char *aUrl, const char *aLogUrl)
 368 {
 369     debugs(20, 3, HERE << "new StoreEntry " << this);
 370     mem_obj = new MemObject(aUrl, aLogUrl);
 371
 372     expires = lastmod = lastref = timestamp = -1;
 373
 374     swap_filen = -1;
 375     swap_dirn = -1;
 376 }
 377
 378 void
 379 StoreEntry::destroyMemObject()
 380 {
 381     debugs(20, 3, HERE << "destroyMemObject " << mem_obj);
 382     setMemStatus(NOT_IN_MEMORY);
 383     MemObject *mem = mem_obj;
 384     mem_obj = NULL;
 385     delete mem;
 386 }
 387
 388 void
 389 destroyStoreEntry(void *data)
 390 {
 391     debugs(20, 3, HERE << "destroyStoreEntry: destroying " <<  data);
 392     StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data));
 393     assert(e != NULL);
 394
 395     if (e == NullStoreEntry::getInstance())
 396         return;
 397
 398     e->destroyMemObject();
 399
 400     e->hashDelete();
 401
 402     assert(e->key == NULL);
 403
 404     delete e;
 405 }
 406
 407 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
 408
 409 void
 410 StoreEntry::hashInsert(const cache_key * someKey)
 411 {
 412     debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << this << " key '" << storeKeyText(someKey) << "'");
 413     key = storeKeyDup(someKey);
 414     hash_join(store_table, this);
 415 }
 416
 417 void
 418 StoreEntry::hashDelete()
 419 {
 420     hash_remove_link(store_table, this);
 421     storeKeyFree((const cache_key *)key);
 422     key = NULL;
 423 }
 424
 425 /* -------------------------------------------------------------------------- */
 426
 427
 428 /* get rid of memory copy of the object */
 429 void
 430 StoreEntry::purgeMem()
 431 {
 432     if (mem_obj == NULL)
 433         return;
 434
 435     debugs(20, 3, "StoreEntry::purgeMem: Freeing memory-copy of " << getMD5Text());
 436
 437     destroyMemObject();
 438
 439     if (swap_status != SWAPOUT_DONE)
 440         release();
 441 }
 442
 443 /* RBC 20050104 this is wrong- memory ref counting
 444  * is not at all equivalent to the store 'usage' concept
 445  * which the replacement policies should be acting upon.
 446  * specifically, object iteration within stores needs
 447  * memory ref counting to prevent race conditions,
 448  * but this should not influence store replacement.
 449  */
 450 void
 451
 452 StoreEntry::lock()
 453 {
 454     lock_count++;
 455     debugs(20, 3, "StoreEntry::lock: key '" << getMD5Text() <<"' count=" <<
 456            lock_count  );
 457     lastref = squid_curtime;
 458     Store::Root().reference(*this);
 459 }
 460
 461 void
 462 StoreEntry::setReleaseFlag()
 463 {
 464     if (EBIT_TEST(flags, RELEASE_REQUEST))
 465         return;
 466
 467     debugs(20, 3, "StoreEntry::setReleaseFlag: '" << getMD5Text() << "'");
 468
 469     EBIT_SET(flags, RELEASE_REQUEST);
 470 }
 471
 472 void
 473 StoreEntry::releaseRequest()
 474 {
 475     if (EBIT_TEST(flags, RELEASE_REQUEST))
 476         return;
 477
 478     setReleaseFlag();
 479
 480     /*
 481      * Clear cachable flag here because we might get called before
 482      * anyone else even looks at the cachability flag.  Also, this
 483      * prevents httpMakePublic from really setting a public key.
 484      */
 485     EBIT_CLR(flags, ENTRY_CACHABLE);
 486
 487     setPrivateKey();
 488 }
 489
 490 /* unlock object, return -1 if object get released after unlock
 491  * otherwise lock_count */
 492 int
 493 StoreEntry::unlock()
 494 {
 495     lock_count--;
 496     debugs(20, 3, "StoreEntry::unlock: key '" << getMD5Text() << "' count=" << lock_count);
 497
 498     if (lock_count)
 499         return (int) lock_count;
 500
 501     if (store_status == STORE_PENDING)
 502         setReleaseFlag();
 503
 504     assert(storePendingNClients(this) == 0);
 505
 506     if (EBIT_TEST(flags, RELEASE_REQUEST))
 507         this->release();
 508     else if (keepInMemory()) {
 509         Store::Root().dereference(*this);
 510         setMemStatus(IN_MEMORY);
 511         mem_obj->unlinkRequest();
 512     } else {
 513         Store::Root().dereference(*this);
 514
 515         if (EBIT_TEST(flags, KEY_PRIVATE))
 516             debugs(20, 1, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
 517
 518         /* StoreEntry::purgeMem may free e */
 519         purgeMem();
 520     }
 521
 522     return 0;
 523 }
 524
 525 void
 526 StoreEntry::getPublicByRequestMethod  (StoreClient *aClient, HttpRequest * request, const HttpRequestMethod& method)
 527 {
 528     assert (aClient);
 529     StoreEntry *result = storeGetPublicByRequestMethod( request, method);
 530
 531     if (!result)
 532         aClient->created (NullStoreEntry::getInstance());
 533     else
 534         aClient->created (result);
 535 }
 536
 537 void
 538 StoreEntry::getPublicByRequest (StoreClient *aClient, HttpRequest * request)
 539 {
 540     assert (aClient);
 541     StoreEntry *result = storeGetPublicByRequest (request);
 542
 543     if (!result)
 544         result = NullStoreEntry::getInstance();
 545
 546     aClient->created (result);
 547 }
 548
 549 void
 550 StoreEntry::getPublic (StoreClient *aClient, const char *uri, const HttpRequestMethod& method)
 551 {
 552     assert (aClient);
 553     StoreEntry *result = storeGetPublic (uri, method);
 554
 555     if (!result)
 556         result = NullStoreEntry::getInstance();
 557
 558     aClient->created (result);
 559 }
 560
 561 StoreEntry *
 562 storeGetPublic(const char *uri, const HttpRequestMethod& method)
 563 {
 564     return Store::Root().get(storeKeyPublic(uri, method));
 565 }
 566
 567 StoreEntry *
 568 storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method)
 569 {
 570     return Store::Root().get(storeKeyPublicByRequestMethod(req, method));
 571 }
 572
 573 StoreEntry *
 574 storeGetPublicByRequest(HttpRequest * req)
 575 {
 576     StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
 577
 578     if (e == NULL && req->method == METHOD_HEAD)
 579         /* We can generate a HEAD reply from a cached GET object */
 580         e = storeGetPublicByRequestMethod(req, METHOD_GET);
 581
 582     return e;
 583 }
 584
 585 static int
 586 getKeyCounter(void)
 587 {
 588     static int key_counter = 0;
 589
 590     if (++key_counter < 0)
 591         key_counter = 1;
 592
 593     return key_counter;
 594 }
 595
 596 /* RBC 20050104 AFAICT this should become simpler:
 597  * rather than reinserting with a special key it should be marked
 598  * as 'released' and then cleaned up when refcounting indicates.
 599  * the StoreHashIndex could well implement its 'released' in the
 600  * current manner.
 601  * Also, clean log writing should skip over ia,t
 602  * Otherwise, we need a 'remove from the index but not the store
 603  * concept'.
 604  */
 605 void
 606 StoreEntry::setPrivateKey()
 607 {
 608     const cache_key *newkey;
 609
 610     if (key && EBIT_TEST(flags, KEY_PRIVATE))
 611         return;                 /* is already private */
 612
 613     if (key) {
 614         if (swap_filen > -1)
 615             storeDirSwapLog(this, SWAP_LOG_DEL);
 616
 617         hashDelete();
 618     }
 619
 620     if (mem_obj != NULL) {
 621         mem_obj->id = getKeyCounter();
 622         newkey = storeKeyPrivate(mem_obj->url, mem_obj->method, mem_obj->id);
 623     } else {
 624         newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter());
 625     }
 626
 627     assert(hash_lookup(store_table, newkey) == NULL);
 628     EBIT_SET(flags, KEY_PRIVATE);
 629     hashInsert(newkey);
 630 }
 631
 632 void
 633 StoreEntry::setPublicKey()
 634 {
 635     StoreEntry *e2 = NULL;
 636     const cache_key *newkey;
 637
 638     if (key && !EBIT_TEST(flags, KEY_PRIVATE))
 639         return;                 /* is already public */
 640
 641     assert(mem_obj);
 642
 643     /*
 644      * We can't make RELEASE_REQUEST objects public.  Depending on
 645      * when RELEASE_REQUEST gets set, we might not be swapping out
 646      * the object.  If we're not swapping out, then subsequent
 647      * store clients won't be able to access object data which has
 648      * been freed from memory.
 649      *
 650      * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
 651      * be set, and StoreEntry::setPublicKey() should not be called.
 652      */
 653 #if MORE_DEBUG_OUTPUT
 654
 655     if (EBIT_TEST(flags, RELEASE_REQUEST))
 656         debugs(20, 1, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
 657
 658 #endif
 659
 660     assert(!EBIT_TEST(flags, RELEASE_REQUEST));
 661
 662     if (mem_obj->request) {
 663         HttpRequest *request = mem_obj->request;
 664
 665         if (!mem_obj->vary_headers) {
 666             /* First handle the case where the object no longer varies */
 667             safe_free(request->vary_headers);
 668         } else {
 669             if (request->vary_headers && strcmp(request->vary_headers, mem_obj->vary_headers) != 0) {
 670                 /* Oops.. the variance has changed. Kill the base object
 671                  * to record the new variance key
 672                  */
 673                 safe_free(request->vary_headers);       /* free old "bad" variance key */
 674                 StoreEntry *pe = storeGetPublic(mem_obj->url, mem_obj->method);
 675
 676                 if (pe)
 677                     pe->release();
 678             }
 679
 680             /* Make sure the request knows the variance status */
 681             if (!request->vary_headers) {
 682                 const char *vary = httpMakeVaryMark(request, mem_obj->getReply());
 683
 684                 if (vary)
 685                     request->vary_headers = xstrdup(vary);
 686             }
 687         }
 688
 689         if (mem_obj->vary_headers && !storeGetPublic(mem_obj->url, mem_obj->method)) {
 690             /* Create "vary" base object */
 691             String vary;
 692             StoreEntry *pe = storeCreateEntry(mem_obj->url, mem_obj->log_url, request->flags, request->method);
 693             /* We are allowed to do this typecast */
 694             HttpReply *rep = new HttpReply;
 695             rep->setHeaders(HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
 696             vary = mem_obj->getReply()->header.getList(HDR_VARY);
 697
 698             if (vary.size()) {
 699                 /* Again, we own this structure layout */
 700                 rep->header.putStr(HDR_VARY, vary.termedBuf());
 701                 vary.clean();
 702             }
 703
 704 #if X_ACCELERATOR_VARY
 705             vary = mem_obj->getReply()->header.getList(HDR_X_ACCELERATOR_VARY);
 706
 707             if (vary.defined()) {
 708                 /* Again, we own this structure layout */
 709                 rep->header.putStr(HDR_X_ACCELERATOR_VARY, vary.termedBuf());
 710                 vary.clean();
 711             }
 712
 713 #endif
 714             pe->replaceHttpReply(rep);
 715
 716             pe->timestampsSet();
 717
 718             pe->makePublic();
 719
 720             pe->complete();
 721
 722             pe->unlock();
 723         }
 724
 725         newkey = storeKeyPublicByRequest(mem_obj->request);
 726     } else
 727         newkey = storeKeyPublic(mem_obj->url, mem_obj->method);
 728
 729     if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
 730         debugs(20, 3, "StoreEntry::setPublicKey: Making old '" << mem_obj->url << "' private.");
 731         e2->setPrivateKey();
 732         e2->release();
 733
 734         if (mem_obj->request)
 735             newkey = storeKeyPublicByRequest(mem_obj->request);
 736         else
 737             newkey = storeKeyPublic(mem_obj->url, mem_obj->method);
 738     }
 739
 740     if (key)
 741         hashDelete();
 742
 743     EBIT_CLR(flags, KEY_PRIVATE);
 744
 745     hashInsert(newkey);
 746
 747     if (swap_filen > -1)
 748         storeDirSwapLog(this, SWAP_LOG_ADD);
 749 }
 750
 751 StoreEntry *
 752 storeCreateEntry(const char *url, const char *log_url, request_flags flags, const HttpRequestMethod& method)
 753 {
 754     StoreEntry *e = NULL;
 755     MemObject *mem = NULL;
 756     debugs(20, 3, "storeCreateEntry: '" << url << "'");
 757
 758     e = new StoreEntry(url, log_url);
 759     e->lock_count = 1;          /* Note lock here w/o calling storeLock() */
 760     mem = e->mem_obj;
 761     mem->method = method;
 762
 763     if (neighbors_do_private_keys || !flags.hierarchical)
 764         e->setPrivateKey();
 765     else
 766         e->setPublicKey();
 767
 768     if (flags.cachable) {
 769         EBIT_SET(e->flags, ENTRY_CACHABLE);
 770         EBIT_CLR(e->flags, RELEASE_REQUEST);
 771     } else {
 772         /* StoreEntry::releaseRequest() clears ENTRY_CACHABLE */
 773         e->releaseRequest();
 774     }
 775
 776     e->store_status = STORE_PENDING;
 777     e->setMemStatus(NOT_IN_MEMORY);
 778     e->swap_status = SWAPOUT_NONE;
 779     e->swap_filen = -1;
 780     e->swap_dirn = -1;
 781     e->refcount = 0;
 782     e->lastref = squid_curtime;
 783     e->timestamp = -1;          /* set in StoreEntry::timestampsSet() */
 784     e->ping_status = PING_NONE;
 785     EBIT_SET(e->flags, ENTRY_VALIDATED);
 786     return e;
 787 }
 788
 789 /* Mark object as expired */
 790 void
 791 StoreEntry::expireNow()
 792 {
 793     debugs(20, 3, "StoreEntry::expireNow: '" << getMD5Text() << "'");
 794     expires = squid_curtime;
 795 }
 796
 797 void
 798 storeWriteComplete (void *data, StoreIOBuffer wroteBuffer)
 799 {
 800     PROF_start(storeWriteComplete);
 801     StoreEntry *e = (StoreEntry *)data;
 802
 803     if (EBIT_TEST(e->flags, DELAY_SENDING)) {
 804         PROF_stop(storeWriteComplete);
 805         return;
 806     }
 807
 808     e->invokeHandlers();
 809     PROF_stop(storeWriteComplete);
 810 }
 811
 812 void
 813 StoreEntry::write (StoreIOBuffer writeBuffer)
 814 {
 815     assert(mem_obj != NULL);
 816     /* This assert will change when we teach the store to update */
 817     PROF_start(StoreEntry_write);
 818     assert(store_status == STORE_PENDING);
 819
 820     if (!writeBuffer.length) {
 821         /* the headers are received already, but we have not received
 822          * any body data. There are BROKEN abuses of HTTP which require
 823          * the headers to be passed along before any body data - see
 824          * http://developer.apple.com/documentation/QuickTime/QTSS/Concepts/chapter_2_section_14.html
 825          * for an example of such bad behaviour. To accomodate this, if
 826          * we have a empty write arrive, we flush to our clients.
 827          * -RBC 20060903
 828          */
 829         PROF_stop(StoreEntry_write);
 830         invokeHandlers();
 831         return;
 832     }
 833
 834     debugs(20, 5, "storeWrite: writing " << writeBuffer.length << " bytes for '" << getMD5Text() << "'");
 835     PROF_stop(StoreEntry_write);
 836     storeGetMemSpace(writeBuffer.length);
 837     mem_obj->write (writeBuffer, storeWriteComplete, this);
 838 }
 839
 840 /* Append incoming data from a primary server to an entry. */
 841 void
 842 StoreEntry::append(char const *buf, int len)
 843 {
 844     assert(mem_obj != NULL);
 845     assert(len >= 0);
 846     assert(store_status == STORE_PENDING);
 847
 848     StoreIOBuffer tempBuffer;
 849     tempBuffer.data = (char *)buf;
 850     tempBuffer.length = len;
 851     /*
 852      * XXX sigh, offset might be < 0 here, but it gets "corrected"
 853      * later.  This offset crap is such a mess.
 854      */
 855     tempBuffer.offset = mem_obj->endOffset() - (getReply() ? getReply()->hdr_sz : 0);
 856     write(tempBuffer);
 857 }
 858
 859
 860 void
 861 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
 862 {
 863     va_list args;
 864     va_start(args, fmt);
 865
 866     storeAppendVPrintf(e, fmt, args);
 867     va_end(args);
 868 }
 869
 870 /* used be storeAppendPrintf and Packer */
 871 void
 872 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
 873 {
 874     LOCAL_ARRAY(char, buf, 4096);
 875     buf[0] = '\0';
 876     vsnprintf(buf, 4096, fmt, vargs);
 877     e->append(buf, strlen(buf));
 878 }
 879
 880 struct _store_check_cachable_hist {
 881
 882     struct {
 883         int non_get;
 884         int not_entry_cachable;
 885         int wrong_content_length;
 886         int negative_cached;
 887         int too_big;
 888         int too_small;
 889         int private_key;
 890         int too_many_open_files;
 891         int too_many_open_fds;
 892     } no;
 893
 894     struct {
 895         int Default;
 896     } yes;
 897 } store_check_cachable_hist;
 898
 899 int
 900 storeTooManyDiskFilesOpen(void)
 901 {
 902     if (Config.max_open_disk_fds == 0)
 903         return 0;
 904
 905     if (store_open_disk_fd > Config.max_open_disk_fds)
 906         return 1;
 907
 908     return 0;
 909 }
 910
 911 int
 912 StoreEntry::checkTooSmall()
 913 {
 914     if (EBIT_TEST(flags, ENTRY_SPECIAL))
 915         return 0;
 916
 917     if (STORE_OK == store_status)
 918         if (mem_obj->object_sz < 0 ||
 919                 mem_obj->object_sz < Config.Store.minObjectSize)
 920             return 1;
 921     if (getReply()->content_length > -1)
 922         if (getReply()->content_length < Config.Store.minObjectSize)
 923             return 1;
 924     return 0;
 925 }
 926
 927 int
 928 StoreEntry::checkCachable()
 929 {
 930 #if CACHE_ALL_METHODS
 931
 932     if (mem_obj->method != METHOD_GET) {
 933         debugs(20, 2, "StoreEntry::checkCachable: NO: non-GET method");
 934         store_check_cachable_hist.no.non_get++;
 935     } else
 936 #endif
 937         if (store_status == STORE_OK && EBIT_TEST(flags, ENTRY_BAD_LENGTH)) {
 938             debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
 939             store_check_cachable_hist.no.wrong_content_length++;
 940         } else if (!EBIT_TEST(flags, ENTRY_CACHABLE)) {
 941             debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
 942             store_check_cachable_hist.no.not_entry_cachable++;
 943         } else if (EBIT_TEST(flags, ENTRY_NEGCACHED)) {
 944             debugs(20, 3, "StoreEntry::checkCachable: NO: negative cached");
 945             store_check_cachable_hist.no.negative_cached++;
 946             return 0;           /* avoid release call below */
 947         } else if ((getReply()->content_length > 0 &&
 948                     getReply()->content_length
 949                     > Config.Store.maxObjectSize) ||
 950                    mem_obj->endOffset() > Config.Store.maxObjectSize) {
 951             debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
 952             store_check_cachable_hist.no.too_big++;
 953         } else if (getReply()->content_length > Config.Store.maxObjectSize) {
 954             debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
 955             store_check_cachable_hist.no.too_big++;
 956         } else if (checkTooSmall()) {
 957             debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
 958             store_check_cachable_hist.no.too_small++;
 959         } else if (EBIT_TEST(flags, KEY_PRIVATE)) {
 960             debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
 961             store_check_cachable_hist.no.private_key++;
 962         } else if (swap_status != SWAPOUT_NONE) {
 963             /*
 964              * here we checked the swap_status because the remaining
 965              * cases are only relevant only if we haven't started swapping
 966              * out the object yet.
 967              */
 968             return 1;
 969         } else if (storeTooManyDiskFilesOpen()) {
 970             debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
 971             store_check_cachable_hist.no.too_many_open_files++;
 972         } else if (fdNFree() < RESERVED_FD) {
 973             debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
 974             store_check_cachable_hist.no.too_many_open_fds++;
 975         } else {
 976             store_check_cachable_hist.yes.Default++;
 977             return 1;
 978         }
 979
 980     releaseRequest();
 981     /* StoreEntry::releaseRequest() cleared ENTRY_CACHABLE */
 982     return 0;
 983 }
 984
 985 void
 986 storeCheckCachableStats(StoreEntry *sentry)
 987 {
 988     storeAppendPrintf(sentry, "Category\t Count\n");
 989
 990 #if CACHE_ALL_METHODS
 991
 992     storeAppendPrintf(sentry, "no.non_get\t%d\n",
 993                       store_check_cachable_hist.no.non_get);
 994 #endif
 995
 996     storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
 997                       store_check_cachable_hist.no.not_entry_cachable);
 998     storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
 999                       store_check_cachable_hist.no.wrong_content_length);
1000     storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
1001                       store_check_cachable_hist.no.negative_cached);
1002     storeAppendPrintf(sentry, "no.too_big\t%d\n",
1003                       store_check_cachable_hist.no.too_big);
1004     storeAppendPrintf(sentry, "no.too_small\t%d\n",
1005                       store_check_cachable_hist.no.too_small);
1006     storeAppendPrintf(sentry, "no.private_key\t%d\n",
1007                       store_check_cachable_hist.no.private_key);
1008     storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
1009                       store_check_cachable_hist.no.too_many_open_files);
1010     storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
1011                       store_check_cachable_hist.no.too_many_open_fds);
1012     storeAppendPrintf(sentry, "yes.default\t%d\n",
1013                       store_check_cachable_hist.yes.Default);
1014 }
1015
1016 void
1017 StoreEntry::complete()
1018 {
1019     debugs(20, 3, "storeComplete: '" << getMD5Text() << "'");
1020
1021     if (store_status != STORE_PENDING) {
1022         /*
1023          * if we're not STORE_PENDING, then probably we got aborted
1024          * and there should be NO clients on this entry
1025          */
1026         assert(EBIT_TEST(flags, ENTRY_ABORTED));
1027         assert(mem_obj->nclients == 0);
1028         return;
1029     }
1030
1031     /* This is suspect: mem obj offsets include the headers. do we adjust for that
1032      * in use of object_sz?
1033      */
1034     mem_obj->object_sz = mem_obj->endOffset();
1035
1036     store_status = STORE_OK;
1037
1038     assert(mem_status == NOT_IN_MEMORY);
1039
1040     if (!validLength()) {
1041         EBIT_SET(flags, ENTRY_BAD_LENGTH);
1042         releaseRequest();
1043     }
1044
1045 #if USE_CACHE_DIGESTS
1046     if (mem_obj->request)
1047         mem_obj->request->hier.store_complete_stop = current_time;
1048
1049 #endif
1050     /*
1051      * We used to call invokeHandlers, then storeSwapOut.  However,
1052      * Madhukar Reddy <myreddy@persistence.com> reported that
1053      * responses without content length would sometimes get released
1054      * in client_side, thinking that the response is incomplete.
1055      */
1056     invokeHandlers();
1057 }
1058
1059 /*
1060  * Someone wants to abort this transfer.  Set the reason in the
1061  * request structure, call the server-side callback and mark the
1062  * entry for releasing
1063  */
1064 void
1065 StoreEntry::abort()
1066 {
1067     statCounter.aborted_requests++;
1068     assert(store_status == STORE_PENDING);
1069     assert(mem_obj != NULL);
1070     debugs(20, 6, "storeAbort: " << getMD5Text());
1071
1072     lock();         /* lock while aborting */
1073     negativeCache();
1074
1075     releaseRequest();
1076
1077     EBIT_SET(flags, ENTRY_ABORTED);
1078
1079     setMemStatus(NOT_IN_MEMORY);
1080
1081     store_status = STORE_OK;
1082
1083     /*
1084      * We assign an object length here.  The only other place we assign
1085      * the object length is in storeComplete()
1086      */
1087     /* RBC: What do we need an object length for? we've just aborted the
1088      * request, the request is private and negatively cached. Surely
1089      * the object length is inappropriate to set.
1090      */
1091     mem_obj->object_sz = mem_obj->endOffset();
1092
1093     /* Notify the server side */
1094
1095     /*
1096      * DPW 2007-05-07
1097      * Should we check abort.data for validity?
1098      */
1099     if (mem_obj->abort.callback) {
1100         if (!cbdataReferenceValid(mem_obj->abort.data))
1101             debugs(20,1,HERE << "queueing event when abort.data is not valid");
1102         eventAdd("mem_obj->abort.callback",
1103                  mem_obj->abort.callback,
1104                  mem_obj->abort.data,
1105                  0.0,
1106                  true);
1107         unregisterAbort();
1108     }
1109
1110     /* XXX Should we reverse these two, so that there is no
1111      * unneeded disk swapping triggered?
1112      */
1113     /* Notify the client side */
1114     invokeHandlers();
1115
1116     /* Close any swapout file */
1117     swapOutFileClose();
1118
1119     unlock();       /* unlock */
1120 }
1121
1122 /**
1123  * Clear Memory storage to accommodate the given object len
1124  */
1125 void
1126 storeGetMemSpace(int size)
1127 {
1128     PROF_start(storeGetMemSpace);
1129     StoreEntry *e = NULL;
1130     int released = 0;
1131     static time_t last_check = 0;
1132     size_t pages_needed;
1133     RemovalPurgeWalker *walker;
1134
1135     if (squid_curtime == last_check) {
1136         PROF_stop(storeGetMemSpace);
1137         return;
1138     }
1139
1140     last_check = squid_curtime;
1141
1142     pages_needed = (size / SM_PAGE_SIZE) + 1;
1143
1144     if (mem_node::InUseCount() + pages_needed < store_pages_max) {
1145         PROF_stop(storeGetMemSpace);
1146         return;
1147     }
1148
1149     debugs(20, 2, "storeGetMemSpace: Starting, need " << pages_needed <<
1150            " pages");
1151
1152     /* XXX what to set as max_scan here? */
1153     walker = mem_policy->PurgeInit(mem_policy, 100000);
1154
1155     while ((e = walker->Next(walker))) {
1156         e->purgeMem();
1157         released++;
1158
1159         if (mem_node::InUseCount() + pages_needed < store_pages_max)
1160             break;
1161     }
1162
1163     walker->Done(walker);
1164     debugs(20, 3, "storeGetMemSpace stats:");
1165     debugs(20, 3, "  " << std::setw(6) << hot_obj_count  << " HOT objects");
1166     debugs(20, 3, "  " << std::setw(6) << released  << " were released");
1167     PROF_stop(storeGetMemSpace);
1168 }
1169
1170
1171 /* thunk through to Store::Root().maintain(). Note that this would be better still
1172  * if registered against the root store itself, but that requires more complex
1173  * update logic - bigger fish to fry first. Long term each store when
1174  * it becomes active will self register
1175  */
1176 void
1177 Store::Maintain(void *notused)
1178 {
1179     Store::Root().maintain();
1180
1181     /* Reregister a maintain event .. */
1182     eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1);
1183
1184 }
1185
1186 /* The maximum objects to scan for maintain storage space */
1187 #define MAINTAIN_MAX_SCAN       1024
1188 #define MAINTAIN_MAX_REMOVE     64
1189
1190 /*
1191  * This routine is to be called by main loop in main.c.
1192  * It removes expired objects on only one bucket for each time called.
1193  *
1194  * This should get called 1/s from main().
1195  */
1196 void
1197 StoreController::maintain()
1198 {
1199     static time_t last_warn_time = 0;
1200
1201     PROF_start(storeMaintainSwapSpace);
1202     swapDir->maintain();
1203
1204     /* this should be emitted by the oversize dir, not globally */
1205
1206     if (store_swap_size > Store::Root().maxSize()) {
1207         if (squid_curtime - last_warn_time > 10) {
1208             debugs(20, 0, "WARNING: Disk space over limit: " << store_swap_size << " KB > "
1209                    << Store::Root().maxSize() << " KB");
1210             last_warn_time = squid_curtime;
1211         }
1212     }
1213
1214     PROF_stop(storeMaintainSwapSpace);
1215 }
1216
1217 /* release an object from a cache */
1218 void
1219 StoreEntry::release()
1220 {
1221     PROF_start(storeRelease);
1222     debugs(20, 3, "storeRelease: Releasing: '" << getMD5Text() << "'");
1223     /* If, for any reason we can't discard this object because of an
1224      * outstanding request, mark it for pending release */
1225
1226     if (locked()) {
1227         expireNow();
1228         debugs(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit");
1229         releaseRequest();
1230         PROF_stop(storeRelease);
1231         return;
1232     }
1233
1234     if (StoreController::store_dirs_rebuilding && swap_filen > -1) {
1235         setPrivateKey();
1236
1237         if (mem_obj)
1238             destroyMemObject();
1239
1240         if (swap_filen > -1) {
1241             /*
1242              * Fake a call to StoreEntry->lock()  When rebuilding is done,
1243              * we'll just call StoreEntry->unlock() on these.
1244              */
1245             lock_count++;
1246             setReleaseFlag();
1247             LateReleaseStack.push_back(this);
1248             PROF_stop(storeRelease);
1249             return;
1250         } else {
1251             destroyStoreEntry(static_cast<hash_link *>(this));
1252         }
1253     }
1254
1255     storeLog(STORE_LOG_RELEASE, this);
1256
1257     if (swap_filen > -1) {
1258         unlink();
1259
1260         if (swap_status == SWAPOUT_DONE)
1261             if (EBIT_TEST(flags, ENTRY_VALIDATED))
1262                 store()->updateSize(swap_file_sz, -1);
1263
1264         if (!EBIT_TEST(flags, KEY_PRIVATE))
1265             storeDirSwapLog(this, SWAP_LOG_DEL);
1266
1267 #if 0
1268         /* From 2.4. I think we do this in storeUnlink? */
1269         storeSwapFileNumberSet(this, -1);
1270
1271 #endif
1272
1273     }
1274
1275     setMemStatus(NOT_IN_MEMORY);
1276     destroyStoreEntry(static_cast<hash_link *>(this));
1277     PROF_stop(storeRelease);
1278 }
1279
1280 static void
1281 storeLateRelease(void *unused)
1282 {
1283     StoreEntry *e;
1284     int i;
1285     static int n = 0;
1286
1287     if (StoreController::store_dirs_rebuilding) {
1288         eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1289         return;
1290     }
1291
1292     for (i = 0; i < 10; i++) {
1293         e = LateReleaseStack.pop();
1294
1295         if (e == NULL) {
1296             /* done! */
1297             debugs(20, 1, "storeLateRelease: released " << n << " objects");
1298             return;
1299         }
1300
1301         e->unlock();
1302         n++;
1303     }
1304
1305     eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
1306 }
1307
1308 /* return 1 if a store entry is locked */
1309 int
1310 StoreEntry::locked() const
1311 {
1312     if (lock_count)
1313         return 1;
1314
1315     if (swap_status == SWAPOUT_WRITING)
1316         return 1;
1317
1318     if (store_status == STORE_PENDING)
1319         return 1;
1320
1321     /*
1322      * SPECIAL, PUBLIC entries should be "locked"
1323      */
1324     if (EBIT_TEST(flags, ENTRY_SPECIAL))
1325         if (!EBIT_TEST(flags, KEY_PRIVATE))
1326             return 1;
1327
1328     return 0;
1329 }
1330
1331 bool
1332 StoreEntry::validLength() const
1333 {
1334     int64_t diff;
1335     const HttpReply *reply;
1336     assert(mem_obj != NULL);
1337     reply = getReply();
1338     debugs(20, 3, "storeEntryValidLength: Checking '" << getMD5Text() << "'");
1339     debugs(20, 5, "storeEntryValidLength:     object_len = " <<
1340            objectLen());
1341     debugs(20, 5, "storeEntryValidLength:         hdr_sz = " << reply->hdr_sz);
1342     debugs(20, 5, "storeEntryValidLength: content_length = " << reply->content_length);
1343
1344     if (reply->content_length < 0) {
1345         debugs(20, 5, "storeEntryValidLength: Unspecified content length: " << getMD5Text());
1346         return 1;
1347     }
1348
1349     if (reply->hdr_sz == 0) {
1350         debugs(20, 5, "storeEntryValidLength: Zero header size: " << getMD5Text());
1351         return 1;
1352     }
1353
1354     if (mem_obj->method == METHOD_HEAD) {
1355         debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
1356         return 1;
1357     }
1358
1359     if (reply->sline.status == HTTP_NOT_MODIFIED)
1360         return 1;
1361
1362     if (reply->sline.status == HTTP_NO_CONTENT)
1363         return 1;
1364
1365     diff = reply->hdr_sz + reply->content_length - objectLen();
1366
1367     if (diff == 0)
1368         return 1;
1369
1370     debugs(20, 3, "storeEntryValidLength: " << (diff < 0 ? -diff : diff)  << " bytes too " << (diff < 0 ? "big" : "small") <<"; '" << getMD5Text() << "'" );
1371
1372     return 0;
1373 }
1374
1375 static void
1376 storeRegisterWithCacheManager(void)
1377 {
1378     Mgr::RegisterAction("storedir", "Store Directory Stats", Store::Stats, 0, 1);
1379     Mgr::RegisterAction("store_io", "Store IO Interface Stats", &Mgr::StoreIoAction::Create, 0, 1);
1380     Mgr::RegisterAction("store_check_cachable_stats", "storeCheckCachable() Stats",
1381                         storeCheckCachableStats, 0, 1);
1382 }
1383
1384 void
1385 storeInit(void)
1386 {
1387     storeKeyInit();
1388     mem_policy = createRemovalPolicy(Config.memPolicy);
1389     storeDigestInit();
1390     storeLogOpen();
1391     eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1392     Store::Root().init();
1393     storeRebuildStart();
1394
1395     storeRegisterWithCacheManager();
1396 }
1397
1398 void
1399 storeConfigure(void)
1400 {
1401     store_swap_high = (long) (((float) Store::Root().maxSize() *
1402                                (float) Config.Swap.highWaterMark) / (float) 100);
1403     store_swap_low = (long) (((float) Store::Root().maxSize() *
1404                               (float) Config.Swap.lowWaterMark) / (float) 100);
1405     store_pages_max = Config.memMaxSize / sizeof(mem_node);
1406 }
1407
1408 int
1409 StoreEntry::keepInMemory() const
1410 {
1411     if (mem_obj == NULL)
1412         return 0;
1413
1414     if (mem_obj->data_hdr.size() == 0)
1415         return 0;
1416
1417     if (mem_obj->inmem_lo != 0)
1418         return 0;
1419
1420     if (!Config.onoff.memory_cache_first && swap_status == SWAPOUT_DONE && refcount == 1)
1421         return 0;
1422
1423     return 1;
1424 }
1425
1426 int
1427 StoreEntry::checkNegativeHit() const
1428 {
1429     if (!EBIT_TEST(flags, ENTRY_NEGCACHED))
1430         return 0;
1431
1432     if (expires <= squid_curtime)
1433         return 0;
1434
1435     if (store_status != STORE_OK)
1436         return 0;
1437
1438     return 1;
1439 }
1440
1441 /**
1442  * Set object for negative caching.
1443  * Preserves any expiry information given by the server.
1444  * In absence of proper expiry info it will set to expire immediately,
1445  * or with HTTP-violations enabled the configured negative-TTL is observed
1446  */
1447 void
1448 StoreEntry::negativeCache()
1449 {
1450     if (expires == 0)
1451 #if USE_HTTP_VIOLATIONS
1452         expires = squid_curtime + Config.negativeTtl;
1453 #else
1454         expires = squid_curtime;
1455 #endif
1456     EBIT_SET(flags, ENTRY_NEGCACHED);
1457 }
1458
1459 void
1460 storeFreeMemory(void)
1461 {
1462     Store::Root(NULL);
1463 #if USE_CACHE_DIGESTS
1464
1465     if (store_digest)
1466         cacheDigestDestroy(store_digest);
1467
1468 #endif
1469
1470     store_digest = NULL;
1471 }
1472
1473 int
1474 expiresMoreThan(time_t expires, time_t when)
1475 {
1476     if (expires < 0)            /* No Expires given */
1477         return 1;
1478
1479     return (expires > (squid_curtime + when));
1480 }
1481
1482 int
1483 StoreEntry::validToSend() const
1484 {
1485     if (EBIT_TEST(flags, RELEASE_REQUEST))
1486         return 0;
1487
1488     if (EBIT_TEST(flags, ENTRY_NEGCACHED))
1489         if (expires <= squid_curtime)
1490             return 0;
1491
1492     if (EBIT_TEST(flags, ENTRY_ABORTED))
1493         return 0;
1494
1495     return 1;
1496 }
1497
1498 void
1499 StoreEntry::timestampsSet()
1500 {
1501     const HttpReply *reply = getReply();
1502     time_t served_date = reply->date;
1503     int age = reply->header.getInt(HDR_AGE);
1504     /* Compute the timestamp, mimicking RFC2616 section 13.2.3. */
1505     /* make sure that 0 <= served_date <= squid_curtime */
1506
1507     if (served_date < 0 || served_date > squid_curtime)
1508         served_date = squid_curtime;
1509
1510     /*
1511      * Compensate with Age header if origin server clock is ahead
1512      * of us and there is a cache in between us and the origin
1513      * server.  But DONT compensate if the age value is larger than
1514      * squid_curtime because it results in a negative served_date.
1515      */
1516     if (age > squid_curtime - served_date)
1517         if (squid_curtime > age)
1518             served_date = squid_curtime - age;
1519
1520     // compensate for Squid-to-server and server-to-Squid delays
1521     if (mem_obj && mem_obj->request) {
1522         const time_t request_sent =
1523             mem_obj->request->hier.peer_http_request_sent.tv_sec;
1524         if (0 < request_sent && request_sent < squid_curtime)
1525             served_date -= (squid_curtime - request_sent);
1526     }
1527
1528     if (reply->expires > 0 && reply->date > -1)
1529         expires = served_date + (reply->expires - reply->date);
1530     else
1531         expires = reply->expires;
1532
1533     lastmod = reply->last_modified;
1534
1535     timestamp = served_date;
1536 }
1537
1538 void
1539 StoreEntry::registerAbort(STABH * cb, void *data)
1540 {
1541     assert(mem_obj);
1542     assert(mem_obj->abort.callback == NULL);
1543     mem_obj->abort.callback = cb;
1544     mem_obj->abort.data = cbdataReference(data);
1545 }
1546
1547 void
1548 StoreEntry::unregisterAbort()
1549 {
1550     assert(mem_obj);
1551     if (mem_obj->abort.callback) {
1552         mem_obj->abort.callback = NULL;
1553         cbdataReferenceDone(mem_obj->abort.data);
1554     }
1555 }
1556
1557 void
1558 StoreEntry::dump(int l) const
1559 {
1560     debugs(20, l, "StoreEntry->key: " << getMD5Text());
1561     debugs(20, l, "StoreEntry->next: " << next);
1562     debugs(20, l, "StoreEntry->mem_obj: " << mem_obj);
1563     debugs(20, l, "StoreEntry->timestamp: " << timestamp);
1564     debugs(20, l, "StoreEntry->lastref: " << lastref);
1565     debugs(20, l, "StoreEntry->expires: " << expires);
1566     debugs(20, l, "StoreEntry->lastmod: " << lastmod);
1567     debugs(20, l, "StoreEntry->swap_file_sz: " << swap_file_sz);
1568     debugs(20, l, "StoreEntry->refcount: " << refcount);
1569     debugs(20, l, "StoreEntry->flags: " << storeEntryFlags(this));
1570     debugs(20, l, "StoreEntry->swap_dirn: " << swap_dirn);
1571     debugs(20, l, "StoreEntry->swap_filen: " << swap_filen);
1572     debugs(20, l, "StoreEntry->lock_count: " << lock_count);
1573     debugs(20, l, "StoreEntry->mem_status: " << mem_status);
1574     debugs(20, l, "StoreEntry->ping_status: " << ping_status);
1575     debugs(20, l, "StoreEntry->store_status: " << store_status);
1576     debugs(20, l, "StoreEntry->swap_status: " << swap_status);
1577 }
1578
1579 /*
1580  * NOTE, this function assumes only two mem states
1581  */
1582 void
1583 StoreEntry::setMemStatus(mem_status_t new_status)
1584 {
1585     if (new_status == mem_status)
1586         return;
1587
1588     assert(mem_obj != NULL);
1589
1590     if (new_status == IN_MEMORY) {
1591         assert(mem_obj->inmem_lo == 0);
1592
1593         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1594             debugs(20, 4, "StoreEntry::setMemStatus: not inserting special " << mem_obj->url << " into policy");
1595         } else {
1596             mem_policy->Add(mem_policy, this, &mem_obj->repl);
1597             debugs(20, 4, "StoreEntry::setMemStatus: inserted mem node " << mem_obj->url);
1598         }
1599
1600         hot_obj_count++;
1601     } else {
1602         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1603             debugs(20, 4, "StoreEntry::setMemStatus: special entry " << mem_obj->url);
1604         } else {
1605             mem_policy->Remove(mem_policy, this, &mem_obj->repl);
1606             debugs(20, 4, "StoreEntry::setMemStatus: removed mem node " << mem_obj->url);
1607         }
1608
1609         hot_obj_count--;
1610     }
1611
1612     mem_status = new_status;
1613 }
1614
1615 const char *
1616 StoreEntry::url() const
1617 {
1618     if (this == NULL)
1619         return "[null_entry]";
1620     else if (mem_obj == NULL)
1621         return "[null_mem_obj]";
1622     else
1623         return mem_obj->url;
1624 }
1625
1626 void
1627 StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl)
1628 {
1629     if (mem_obj)
1630         return;
1631
1632     mem_obj = new MemObject(aUrl, aLogUrl);
1633 }
1634
1635 /* this just sets DELAY_SENDING */
1636 void
1637 StoreEntry::buffer()
1638 {
1639     EBIT_SET(flags, DELAY_SENDING);
1640 }
1641
1642 /* this just clears DELAY_SENDING and Invokes the handlers */
1643 void
1644 StoreEntry::flush()
1645 {
1646     if (EBIT_TEST(flags, DELAY_SENDING)) {
1647         EBIT_CLR(flags, DELAY_SENDING);
1648         invokeHandlers();
1649     }
1650 }
1651
1652 int64_t
1653 StoreEntry::objectLen() const
1654 {
1655     assert(mem_obj != NULL);
1656     return mem_obj->object_sz;
1657 }
1658
1659 int64_t
1660 StoreEntry::contentLen() const
1661 {
1662     assert(mem_obj != NULL);
1663     assert(getReply() != NULL);
1664     return objectLen() - getReply()->hdr_sz;
1665 }
1666
1667 HttpReply const *
1668 StoreEntry::getReply () const
1669 {
1670     if (NULL == mem_obj)
1671         return NULL;
1672
1673     return mem_obj->getReply();
1674 }
1675
1676 void
1677 StoreEntry::reset()
1678 {
1679     assert (mem_obj);
1680     debugs(20, 3, "StoreEntry::reset: " << url());
1681     mem_obj->reset();
1682     HttpReply *rep = (HttpReply *) getReply();       // bypass const
1683     rep->reset();
1684     expires = lastmod = timestamp = -1;
1685 }
1686
1687 /*
1688  * storeFsInit
1689  *
1690  * This routine calls the SETUP routine for each fs type.
1691  * I don't know where the best place for this is, and I'm not going to shuffle
1692  * around large chunks of code right now (that can be done once its working.)
1693  */
1694 void
1695 storeFsInit(void)
1696 {
1697     storeReplSetup();
1698 }
1699
1700 /*
1701  * called to add another store removal policy module
1702  */
1703 void
1704 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1705 {
1706     int i;
1707
1708     /* find the number of currently known repl types */
1709     for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
1710         if (strcmp(storerepl_list[i].typestr, type) == 0) {
1711             debugs(20, 1, "WARNING: Trying to load store replacement policy " << type << " twice.");
1712             return;
1713         }
1714     }
1715
1716     /* add the new type */
1717     storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
1718
1719     memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1720
1721     storerepl_list[i].typestr = type;
1722
1723     storerepl_list[i].create = create;
1724 }
1725
1726 /*
1727  * Create a removal policy instance
1728  */
1729 RemovalPolicy *
1730 createRemovalPolicy(RemovalPolicySettings * settings)
1731 {
1732     storerepl_entry_t *r;
1733
1734     for (r = storerepl_list; r && r->typestr; r++) {
1735         if (strcmp(r->typestr, settings->type) == 0)
1736             return r->create(settings->args);
1737     }
1738
1739     debugs(20, 1, "ERROR: Unknown policy " << settings->type);
1740     debugs(20, 1, "ERROR: Be sure to have set cache_replacement_policy");
1741     debugs(20, 1, "ERROR:   and memory_replacement_policy in squid.conf!");
1742     fatalf("ERROR: Unknown policy %s\n", settings->type);
1743     return NULL;                /* NOTREACHED */
1744 }
1745
1746 #if 0
1747 void
1748 storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
1749 {
1750     if (e->swap_file_number == filn)
1751         return;
1752
1753     if (filn < 0) {
1754         assert(-1 == filn);
1755         storeDirMapBitReset(e->swap_file_number);
1756         storeDirLRUDelete(e);
1757         e->swap_file_number = -1;
1758     } else {
1759         assert(-1 == e->swap_file_number);
1760         storeDirMapBitSet(e->swap_file_number = filn);
1761         storeDirLRUAdd(e);
1762     }
1763 }
1764
1765 #endif
1766
1767
1768 /*
1769  * Replace a store entry with
1770  * a new reply. This eats the reply.
1771  */
1772 void
1773 StoreEntry::replaceHttpReply(HttpReply *rep)
1774 {
1775     debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
1776     Packer p;
1777
1778     if (!mem_obj) {
1779         debugs(20, 0, "Attempt to replace object with no in-memory representation");
1780         return;
1781     }
1782
1783     mem_obj->replaceHttpReply(rep);
1784
1785     /* TODO: when we store headers serparately remove the header portion */
1786     /* TODO: mark the length of the headers ? */
1787     /* We ONLY want the headers */
1788     packerToStoreInit(&p, this);
1789
1790     assert (isEmpty());
1791
1792     getReply()->packHeadersInto(&p);
1793
1794     rep->hdr_sz = mem_obj->endOffset();
1795
1796     httpBodyPackInto(&getReply()->body, &p);
1797
1798     packerClean(&p);
1799 }
1800
1801
1802 char const *
1803 StoreEntry::getSerialisedMetaData()
1804 {
1805     StoreMeta *tlv_list = storeSwapMetaBuild(this);
1806     int swap_hdr_sz;
1807     char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
1808     storeSwapTLVFree(tlv_list);
1809     assert (swap_hdr_sz >= 0);
1810     mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz;
1811     return result;
1812 }
1813
1814 bool
1815 StoreEntry::swapoutPossible()
1816 {
1817     /* should we swap something out to disk? */
1818     debugs(20, 7, "storeSwapOut: " << url());
1819     debugs(20, 7, "storeSwapOut: store_status = " << storeStatusStr[store_status]);
1820
1821     if (EBIT_TEST(flags, ENTRY_ABORTED)) {
1822         assert(EBIT_TEST(flags, RELEASE_REQUEST));
1823         swapOutFileClose();
1824         return false;
1825     }
1826
1827     if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1828         debugs(20, 3, "storeSwapOut: " << url() << " SPECIAL");
1829         return false;
1830     }
1831
1832     return true;
1833 }
1834
1835 void
1836 StoreEntry::trimMemory()
1837 {
1838     /*
1839      * DPW 2007-05-09
1840      * Bug #1943.  We must not let go any data for IN_MEMORY
1841      * objects.  We have to wait until the mem_status changes.
1842      */
1843     if (mem_status == IN_MEMORY)
1844         return;
1845
1846     if (!swapOutAble()) {
1847         if (mem_obj->policyLowestOffsetToKeep(0) == 0) {
1848             /* Nothing to do */
1849             return;
1850         }
1851         /*
1852          * Its not swap-able, and we're about to delete a chunk,
1853          * so we must make it PRIVATE.  This is tricky/ugly because
1854          * for the most part, we treat swapable == cachable here.
1855          */
1856         releaseRequest();
1857         mem_obj->trimUnSwappable ();
1858     } else {
1859         mem_obj->trimSwappable ();
1860     }
1861 }
1862
1863 bool
1864 StoreEntry::modifiedSince(HttpRequest * request) const
1865 {
1866     int object_length;
1867     time_t mod_time = lastmod;
1868
1869     if (mod_time < 0)
1870         mod_time = timestamp;
1871
1872     debugs(88, 3, "modifiedSince: '" << url() << "'");
1873
1874     debugs(88, 3, "modifiedSince: mod_time = " << mod_time);
1875
1876     if (mod_time < 0)
1877         return true;
1878
1879     /* Find size of the object */
1880     object_length = getReply()->content_length;
1881
1882     if (object_length < 0)
1883         object_length = contentLen();
1884
1885     if (mod_time > request->ims) {
1886         debugs(88, 3, "--> YES: entry newer than client");
1887         return true;
1888     } else if (mod_time < request->ims) {
1889         debugs(88, 3, "-->  NO: entry older than client");
1890         return false;
1891     } else if (request->imslen < 0) {
1892         debugs(88, 3, "-->  NO: same LMT, no client length");
1893         return false;
1894     } else if (request->imslen == object_length) {
1895         debugs(88, 3, "-->  NO: same LMT, same length");
1896         return false;
1897     } else {
1898         debugs(88, 3, "--> YES: same LMT, different length");
1899         return true;
1900     }
1901 }
1902
1903 bool
1904 StoreEntry::hasIfMatchEtag(const HttpRequest &request) const
1905 {
1906     const String reqETags = request.header.getList(HDR_IF_MATCH);
1907     return hasOneOfEtags(reqETags, false);
1908 }
1909
1910 bool
1911 StoreEntry::hasIfNoneMatchEtag(const HttpRequest &request) const
1912 {
1913     const String reqETags = request.header.getList(HDR_IF_NONE_MATCH);
1914     // weak comparison is allowed only for HEAD or full-body GET requests
1915     const bool allowWeakMatch = !request.flags.range &&
1916                                 (request.method == METHOD_GET || request.method == METHOD_HEAD);
1917     return hasOneOfEtags(reqETags, allowWeakMatch);
1918 }
1919
1920 /// whether at least one of the request ETags matches entity ETag
1921 bool
1922 StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
1923 {
1924     const ETag repETag = getReply()->header.getETag(HDR_ETAG);
1925     if (!repETag.str)
1926         return strListIsMember(&reqETags, "*", ',');
1927
1928     bool matched = false;
1929     const char *pos = NULL;
1930     const char *item;
1931     int ilen;
1932     while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) {
1933         if (!strncmp(item, "*", ilen))
1934             matched = true;
1935         else {
1936             String str;
1937             str.append(item, ilen);
1938             ETag reqETag;
1939             if (etagParseInit(&reqETag, str.termedBuf())) {
1940                 matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) :
1941                           etagIsStrongEqual(repETag, reqETag);
1942             }
1943         }
1944     }
1945     return matched;
1946 }
1947
1948 StorePointer
1949 StoreEntry::store() const
1950 {
1951     assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured);
1952     return INDEXSD(swap_dirn);
1953 }
1954
1955 void
1956 StoreEntry::unlink()
1957 {
1958     store()->unlink(*this);
1959 }
1960
1961 /*
1962  * return true if the entry is in a state where
1963  * it can accept more data (ie with write() method)
1964  */
1965 bool
1966 StoreEntry::isAccepting() const
1967 {
1968     if (STORE_PENDING != store_status)
1969         return false;
1970
1971     if (EBIT_TEST(flags, ENTRY_ABORTED))
1972         return false;
1973
1974     return true;
1975 }
1976
1977 /* NullStoreEntry */
1978
1979 NullStoreEntry NullStoreEntry::_instance;
1980
1981 NullStoreEntry *
1982 NullStoreEntry::getInstance()
1983 {
1984     return &_instance;
1985 }
1986
1987 char const *
1988 NullStoreEntry::getMD5Text() const
1989 {
1990     return "N/A";
1991 }
1992
1993 void
1994 NullStoreEntry::operator delete(void*)
1995 {
1996     fatal ("Attempt to delete NullStoreEntry\n");
1997 }
1998
1999 char const *
2000 NullStoreEntry::getSerialisedMetaData()
2001 {
2002     return NULL;
2003 }
2004
2005 #if !_USE_INLINE_
2006 #include "Store.cci"
2007 #endif