src/store.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 20    Storage Manager
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "CacheManager.h"
  38 #include "comm/Connection.h"
  39 #include "ETag.h"
  40 #include "event.h"
  41 #include "fde.h"
  42 #include "Store.h"
  43 #include "mgr/Registration.h"
  44 #include "StoreClient.h"
  45 #include "stmem.h"
  46 #include "HttpReply.h"
  47 #include "HttpRequest.h"
  48 #include "MemObject.h"
  49 #include "mem_node.h"
  50 #include "StoreMeta.h"
  51 #include "SwapDir.h"
  52 #if USE_DELAY_POOLS
  53 #include "DelayPools.h"
  54 #endif
  55 #include "Stack.h"
  56 #include "SquidTime.h"
  57 #include "swap_log_op.h"
  58 #include "mgr/StoreIoAction.h"
  59
  60 static STMCB storeWriteComplete;
  61
  62 #define REBUILD_TIMESTAMP_DELTA_MAX 2
  63
  64 #define STORE_IN_MEM_BUCKETS            (229)
  65
  66
  67 /** \todo Convert these string constants to enum string-arrays generated */
  68
  69 const char *memStatusStr[] = {
  70     "NOT_IN_MEMORY",
  71     "IN_MEMORY"
  72 };
  73
  74 const char *pingStatusStr[] = {
  75     "PING_NONE",
  76     "PING_WAITING",
  77     "PING_DONE"
  78 };
  79
  80 const char *storeStatusStr[] = {
  81     "STORE_OK",
  82     "STORE_PENDING"
  83 };
  84
  85 const char *swapStatusStr[] = {
  86     "SWAPOUT_NONE",
  87     "SWAPOUT_WRITING",
  88     "SWAPOUT_DONE"
  89 };
  90
  91
  92 /*
  93  * This defines an repl type
  94  */
  95
  96 typedef struct _storerepl_entry storerepl_entry_t;
  97
  98 struct _storerepl_entry {
  99     const char *typestr;
 100     REMOVALPOLICYCREATE *create;
 101 };
 102
 103 static storerepl_entry_t *storerepl_list = NULL;
 104
 105
 106 /*
 107  * local function prototypes
 108  */
 109 static int getKeyCounter(void);
 110 static OBJH storeCheckCachableStats;
 111 static EVH storeLateRelease;
 112
 113 /*
 114  * local variables
 115  */
 116 static Stack<StoreEntry*> LateReleaseStack;
 117 MemAllocator *StoreEntry::pool = NULL;
 118
 119 StorePointer Store::CurrentRoot = NULL;
 120
 121 void
 122 Store::Root(Store * aRoot)
 123 {
 124     CurrentRoot = aRoot;
 125 }
 126
 127 void
 128 Store::Root(StorePointer aRoot)
 129 {
 130     Root(aRoot.getRaw());
 131 }
 132
 133 void
 134 Store::Stats(StoreEntry * output)
 135 {
 136     assert (output);
 137     Root().stat(*output);
 138 }
 139
 140 void
 141 Store::create()
 142 {}
 143
 144 void
 145 Store::diskFull()
 146 {}
 147
 148 void
 149 Store::sync()
 150 {}
 151
 152 void
 153 Store::unlink (StoreEntry &anEntry)
 154 {
 155     fatal("Store::unlink on invalid Store\n");
 156 }
 157
 158 void *
 159 StoreEntry::operator new (size_t bytecount)
 160 {
 161     assert (bytecount == sizeof (StoreEntry));
 162
 163     if (!pool) {
 164         pool = memPoolCreate ("StoreEntry", bytecount);
 165         pool->setChunkSize(2048 * 1024);
 166     }
 167
 168     return pool->alloc();
 169 }
 170
 171 void
 172 StoreEntry::operator delete (void *address)
 173 {
 174     pool->freeOne(address);
 175 }
 176
 177 void
 178 StoreEntry::makePublic()
 179 {
 180     /* This object can be cached for a long time */
 181
 182     if (EBIT_TEST(flags, ENTRY_CACHABLE))
 183         setPublicKey();
 184 }
 185
 186 void
 187 StoreEntry::makePrivate()
 188 {
 189     /* This object should never be cached at all */
 190     expireNow();
 191     releaseRequest(); /* delete object when not used */
 192     /* releaseRequest clears ENTRY_CACHABLE flag */
 193 }
 194
 195 void
 196 StoreEntry::cacheNegatively()
 197 {
 198     /* This object may be negatively cached */
 199     negativeCache();
 200
 201     if (EBIT_TEST(flags, ENTRY_CACHABLE))
 202         setPublicKey();
 203 }
 204
 205 size_t
 206 StoreEntry::inUseCount()
 207 {
 208     if (!pool)
 209         return 0;
 210     return pool->getInUseCount();
 211 }
 212
 213 const char *
 214 StoreEntry::getMD5Text() const
 215 {
 216     return storeKeyText((const cache_key *)key);
 217 }
 218
 219 #include "comm.h"
 220
 221 void
 222 StoreEntry::DeferReader(void *theContext, CommRead const &aRead)
 223 {
 224     StoreEntry *anEntry = (StoreEntry *)theContext;
 225     anEntry->delayAwareRead(aRead.conn,
 226                             aRead.buf,
 227                             aRead.len,
 228                             aRead.callback);
 229 }
 230
 231 void
 232 StoreEntry::delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
 233 {
 234     size_t amountToRead = bytesWanted(Range<size_t>(0, len));
 235     /* sketch: readdeferer* = getdeferer.
 236      * ->deferRead (fd, buf, len, callback, DelayAwareRead, this)
 237      */
 238
 239     if (amountToRead == 0) {
 240         assert (mem_obj);
 241         /* read ahead limit */
 242         /* Perhaps these two calls should both live in MemObject */
 243 #if USE_DELAY_POOLS
 244         if (!mem_obj->readAheadPolicyCanRead()) {
 245 #endif
 246             mem_obj->delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 247             return;
 248 #if USE_DELAY_POOLS
 249         }
 250
 251         /* delay id limit */
 252         mem_obj->mostBytesAllowed().delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 253         return;
 254
 255 #endif
 256
 257     }
 258
 259     comm_read(conn, buf, amountToRead, callback);
 260 }
 261
 262 size_t
 263 StoreEntry::bytesWanted (Range<size_t> const aRange) const
 264 {
 265     assert (aRange.size());
 266
 267     if (mem_obj == NULL)
 268         return aRange.end - 1;
 269
 270 #if URL_CHECKSUM_DEBUG
 271
 272     mem_obj->checkUrlChecksum();
 273
 274 #endif
 275
 276     /* Always read *something* here - we haven't got the header yet */
 277     if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT))
 278         return aRange.end - 1;
 279
 280     if (!mem_obj->readAheadPolicyCanRead())
 281         return 0;
 282
 283     return mem_obj->mostBytesWanted(aRange.end - 1);
 284 }
 285
 286 bool
 287 StoreEntry::checkDeferRead(int fd) const
 288 {
 289     return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
 290 }
 291
 292 void
 293 StoreEntry::setNoDelay (bool const newValue)
 294 {
 295     if (mem_obj)
 296         mem_obj->setNoDelay(newValue);
 297 }
 298
 299 store_client_t
 300 StoreEntry::storeClientType() const
 301 {
 302     /* The needed offset isn't in memory
 303      * XXX TODO: this is wrong for range requests
 304      * as the needed offset may *not* be 0, AND
 305      * offset 0 in the memory object is the HTTP headers.
 306      */
 307
 308     if (mem_obj->inmem_lo)
 309         return STORE_DISK_CLIENT;
 310
 311     if (EBIT_TEST(flags, ENTRY_ABORTED)) {
 312         /* I don't think we should be adding clients to aborted entries */
 313         debugs(20, 1, "storeClientType: adding to ENTRY_ABORTED entry");
 314         return STORE_MEM_CLIENT;
 315     }
 316
 317     if (store_status == STORE_OK) {
 318         /* the object has completed. */
 319
 320         if (mem_obj->inmem_lo == 0 && !isEmpty()) {
 321             if (swap_status == SWAPOUT_DONE) {
 322                 if (mem_obj->endOffset() == mem_obj->object_sz) {
 323                     /* hot object fully swapped in */
 324                     return STORE_MEM_CLIENT;
 325                 }
 326             } else {
 327                 /* Memory-only, or currently being swapped out */
 328                 return STORE_MEM_CLIENT;
 329             }
 330         }
 331         return STORE_DISK_CLIENT;
 332     }
 333
 334     /* here and past, entry is STORE_PENDING */
 335     /*
 336      * If this is the first client, let it be the mem client
 337      */
 338     if (mem_obj->nclients == 1)
 339         return STORE_MEM_CLIENT;
 340
 341     /*
 342      * If there is no disk file to open yet, we must make this a
 343      * mem client.  If we can't open the swapin file before writing
 344      * to the client, there is no guarantee that we will be able
 345      * to open it later when we really need it.
 346      */
 347     if (swap_status == SWAPOUT_NONE)
 348         return STORE_MEM_CLIENT;
 349
 350     /*
 351      * otherwise, make subsequent clients read from disk so they
 352      * can not delay the first, and vice-versa.
 353      */
 354     return STORE_DISK_CLIENT;
 355 }
 356
 357 StoreEntry::StoreEntry()
 358 {
 359     debugs(20, 3, HERE << "new StoreEntry " << this);
 360     mem_obj = NULL;
 361
 362     expires = lastmod = lastref = timestamp = -1;
 363
 364     swap_filen = -1;
 365     swap_dirn = -1;
 366 }
 367
 368 StoreEntry::StoreEntry(const char *aUrl, const char *aLogUrl)
 369 {
 370     debugs(20, 3, HERE << "new StoreEntry " << this);
 371     mem_obj = new MemObject(aUrl, aLogUrl);
 372
 373     expires = lastmod = lastref = timestamp = -1;
 374
 375     swap_filen = -1;
 376     swap_dirn = -1;
 377 }
 378
 379 void
 380 StoreEntry::destroyMemObject()
 381 {
 382     debugs(20, 3, HERE << "destroyMemObject " << mem_obj);
 383     setMemStatus(NOT_IN_MEMORY);
 384     MemObject *mem = mem_obj;
 385     mem_obj = NULL;
 386     delete mem;
 387 }
 388
 389 void
 390 destroyStoreEntry(void *data)
 391 {
 392     debugs(20, 3, HERE << "destroyStoreEntry: destroying " <<  data);
 393     StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data));
 394     assert(e != NULL);
 395
 396     if (e == NullStoreEntry::getInstance())
 397         return;
 398
 399     e->destroyMemObject();
 400
 401     e->hashDelete();
 402
 403     assert(e->key == NULL);
 404
 405     delete e;
 406 }
 407
 408 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
 409
 410 void
 411 StoreEntry::hashInsert(const cache_key * someKey)
 412 {
 413     debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << this << " key '" << storeKeyText(someKey) << "'");
 414     key = storeKeyDup(someKey);
 415     hash_join(store_table, this);
 416 }
 417
 418 void
 419 StoreEntry::hashDelete()
 420 {
 421     hash_remove_link(store_table, this);
 422     storeKeyFree((const cache_key *)key);
 423     key = NULL;
 424 }
 425
 426 /* -------------------------------------------------------------------------- */
 427
 428
 429 /* get rid of memory copy of the object */
 430 void
 431 StoreEntry::purgeMem()
 432 {
 433     if (mem_obj == NULL)
 434         return;
 435
 436     debugs(20, 3, "StoreEntry::purgeMem: Freeing memory-copy of " << getMD5Text());
 437
 438     destroyMemObject();
 439
 440     if (swap_status != SWAPOUT_DONE)
 441         release();
 442 }
 443
 444 /* RBC 20050104 this is wrong- memory ref counting
 445  * is not at all equivalent to the store 'usage' concept
 446  * which the replacement policies should be acting upon.
 447  * specifically, object iteration within stores needs
 448  * memory ref counting to prevent race conditions,
 449  * but this should not influence store replacement.
 450  */
 451 void
 452
 453 StoreEntry::lock()
 454 {
 455     lock_count++;
 456     debugs(20, 3, "StoreEntry::lock: key '" << getMD5Text() <<"' count=" <<
 457            lock_count  );
 458     lastref = squid_curtime;
 459     Store::Root().reference(*this);
 460 }
 461
 462 void
 463 StoreEntry::setReleaseFlag()
 464 {
 465     if (EBIT_TEST(flags, RELEASE_REQUEST))
 466         return;
 467
 468     debugs(20, 3, "StoreEntry::setReleaseFlag: '" << getMD5Text() << "'");
 469
 470     EBIT_SET(flags, RELEASE_REQUEST);
 471 }
 472
 473 void
 474 StoreEntry::releaseRequest()
 475 {
 476     if (EBIT_TEST(flags, RELEASE_REQUEST))
 477         return;
 478
 479     setReleaseFlag();
 480
 481     /*
 482      * Clear cachable flag here because we might get called before
 483      * anyone else even looks at the cachability flag.  Also, this
 484      * prevents httpMakePublic from really setting a public key.
 485      */
 486     EBIT_CLR(flags, ENTRY_CACHABLE);
 487
 488     setPrivateKey();
 489 }
 490
 491 /* unlock object, return -1 if object get released after unlock
 492  * otherwise lock_count */
 493 int
 494 StoreEntry::unlock()
 495 {
 496     lock_count--;
 497     debugs(20, 3, "StoreEntry::unlock: key '" << getMD5Text() << "' count=" << lock_count);
 498
 499     if (lock_count)
 500         return (int) lock_count;
 501
 502     if (store_status == STORE_PENDING)
 503         setReleaseFlag();
 504
 505     assert(storePendingNClients(this) == 0);
 506
 507     if (EBIT_TEST(flags, RELEASE_REQUEST))
 508         this->release();
 509     else if (keepInMemory()) {
 510         Store::Root().dereference(*this);
 511         setMemStatus(IN_MEMORY);
 512         mem_obj->unlinkRequest();
 513     } else {
 514         Store::Root().dereference(*this);
 515
 516         if (EBIT_TEST(flags, KEY_PRIVATE))
 517             debugs(20, 1, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
 518
 519         /* StoreEntry::purgeMem may free e */
 520         purgeMem();
 521     }
 522
 523     return 0;
 524 }
 525
 526 void
 527 StoreEntry::getPublicByRequestMethod  (StoreClient *aClient, HttpRequest * request, const HttpRequestMethod& method)
 528 {
 529     assert (aClient);
 530     StoreEntry *result = storeGetPublicByRequestMethod( request, method);
 531
 532     if (!result)
 533         aClient->created (NullStoreEntry::getInstance());
 534     else
 535         aClient->created (result);
 536 }
 537
 538 void
 539 StoreEntry::getPublicByRequest (StoreClient *aClient, HttpRequest * request)
 540 {
 541     assert (aClient);
 542     StoreEntry *result = storeGetPublicByRequest (request);
 543
 544     if (!result)
 545         result = NullStoreEntry::getInstance();
 546
 547     aClient->created (result);
 548 }
 549
 550 void
 551 StoreEntry::getPublic (StoreClient *aClient, const char *uri, const HttpRequestMethod& method)
 552 {
 553     assert (aClient);
 554     StoreEntry *result = storeGetPublic (uri, method);
 555
 556     if (!result)
 557         result = NullStoreEntry::getInstance();
 558
 559     aClient->created (result);
 560 }
 561
 562 StoreEntry *
 563 storeGetPublic(const char *uri, const HttpRequestMethod& method)
 564 {
 565     return Store::Root().get(storeKeyPublic(uri, method));
 566 }
 567
 568 StoreEntry *
 569 storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method)
 570 {
 571     return Store::Root().get(storeKeyPublicByRequestMethod(req, method));
 572 }
 573
 574 StoreEntry *
 575 storeGetPublicByRequest(HttpRequest * req)
 576 {
 577     StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
 578
 579     if (e == NULL && req->method == METHOD_HEAD)
 580         /* We can generate a HEAD reply from a cached GET object */
 581         e = storeGetPublicByRequestMethod(req, METHOD_GET);
 582
 583     return e;
 584 }
 585
 586 static int
 587 getKeyCounter(void)
 588 {
 589     static int key_counter = 0;
 590
 591     if (++key_counter < 0)
 592         key_counter = 1;
 593
 594     return key_counter;
 595 }
 596
 597 /* RBC 20050104 AFAICT this should become simpler:
 598  * rather than reinserting with a special key it should be marked
 599  * as 'released' and then cleaned up when refcounting indicates.
 600  * the StoreHashIndex could well implement its 'released' in the
 601  * current manner.
 602  * Also, clean log writing should skip over ia,t
 603  * Otherwise, we need a 'remove from the index but not the store
 604  * concept'.
 605  */
 606 void
 607 StoreEntry::setPrivateKey()
 608 {
 609     const cache_key *newkey;
 610
 611     if (key && EBIT_TEST(flags, KEY_PRIVATE))
 612         return;                 /* is already private */
 613
 614     if (key) {
 615         if (swap_filen > -1)
 616             storeDirSwapLog(this, SWAP_LOG_DEL);
 617
 618         hashDelete();
 619     }
 620
 621     if (mem_obj != NULL) {
 622         mem_obj->id = getKeyCounter();
 623         newkey = storeKeyPrivate(mem_obj->url, mem_obj->method, mem_obj->id);
 624     } else {
 625         newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter());
 626     }
 627
 628     assert(hash_lookup(store_table, newkey) == NULL);
 629     EBIT_SET(flags, KEY_PRIVATE);
 630     hashInsert(newkey);
 631 }
 632
 633 void
 634 StoreEntry::setPublicKey()
 635 {
 636     StoreEntry *e2 = NULL;
 637     const cache_key *newkey;
 638
 639     if (key && !EBIT_TEST(flags, KEY_PRIVATE))
 640         return;                 /* is already public */
 641
 642     assert(mem_obj);
 643
 644     /*
 645      * We can't make RELEASE_REQUEST objects public.  Depending on
 646      * when RELEASE_REQUEST gets set, we might not be swapping out
 647      * the object.  If we're not swapping out, then subsequent
 648      * store clients won't be able to access object data which has
 649      * been freed from memory.
 650      *
 651      * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
 652      * be set, and StoreEntry::setPublicKey() should not be called.
 653      */
 654 #if MORE_DEBUG_OUTPUT
 655
 656     if (EBIT_TEST(flags, RELEASE_REQUEST))
 657         debugs(20, 1, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
 658
 659 #endif
 660
 661     assert(!EBIT_TEST(flags, RELEASE_REQUEST));
 662
 663     if (mem_obj->request) {
 664         HttpRequest *request = mem_obj->request;
 665
 666         if (!mem_obj->vary_headers) {
 667             /* First handle the case where the object no longer varies */
 668             safe_free(request->vary_headers);
 669         } else {
 670             if (request->vary_headers && strcmp(request->vary_headers, mem_obj->vary_headers) != 0) {
 671                 /* Oops.. the variance has changed. Kill the base object
 672                  * to record the new variance key
 673                  */
 674                 safe_free(request->vary_headers);       /* free old "bad" variance key */
 675                 StoreEntry *pe = storeGetPublic(mem_obj->url, mem_obj->method);
 676
 677                 if (pe)
 678                     pe->release();
 679             }
 680
 681             /* Make sure the request knows the variance status */
 682             if (!request->vary_headers) {
 683                 const char *vary = httpMakeVaryMark(request, mem_obj->getReply());
 684
 685                 if (vary)
 686                     request->vary_headers = xstrdup(vary);
 687             }
 688         }
 689
 690         if (mem_obj->vary_headers && !storeGetPublic(mem_obj->url, mem_obj->method)) {
 691             /* Create "vary" base object */
 692             String vary;
 693             StoreEntry *pe = storeCreateEntry(mem_obj->url, mem_obj->log_url, request->flags, request->method);
 694             /* We are allowed to do this typecast */
 695             HttpReply *rep = new HttpReply;
 696             rep->setHeaders(HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
 697             vary = mem_obj->getReply()->header.getList(HDR_VARY);
 698
 699             if (vary.size()) {
 700                 /* Again, we own this structure layout */
 701                 rep->header.putStr(HDR_VARY, vary.termedBuf());
 702                 vary.clean();
 703             }
 704
 705 #if X_ACCELERATOR_VARY
 706             vary = mem_obj->getReply()->header.getList(HDR_X_ACCELERATOR_VARY);
 707
 708             if (vary.defined()) {
 709                 /* Again, we own this structure layout */
 710                 rep->header.putStr(HDR_X_ACCELERATOR_VARY, vary.termedBuf());
 711                 vary.clean();
 712             }
 713
 714 #endif
 715             pe->replaceHttpReply(rep);
 716
 717             pe->timestampsSet();
 718
 719             pe->makePublic();
 720
 721             pe->complete();
 722
 723             pe->unlock();
 724         }
 725
 726         newkey = storeKeyPublicByRequest(mem_obj->request);
 727     } else
 728         newkey = storeKeyPublic(mem_obj->url, mem_obj->method);
 729
 730     if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
 731         debugs(20, 3, "StoreEntry::setPublicKey: Making old '" << mem_obj->url << "' private.");
 732         e2->setPrivateKey();
 733         e2->release();
 734
 735         if (mem_obj->request)
 736             newkey = storeKeyPublicByRequest(mem_obj->request);
 737         else
 738             newkey = storeKeyPublic(mem_obj->url, mem_obj->method);
 739     }
 740
 741     if (key)
 742         hashDelete();
 743
 744     EBIT_CLR(flags, KEY_PRIVATE);
 745
 746     hashInsert(newkey);
 747
 748     if (swap_filen > -1)
 749         storeDirSwapLog(this, SWAP_LOG_ADD);
 750 }
 751
 752 StoreEntry *
 753 storeCreateEntry(const char *url, const char *log_url, request_flags flags, const HttpRequestMethod& method)
 754 {
 755     StoreEntry *e = NULL;
 756     MemObject *mem = NULL;
 757     debugs(20, 3, "storeCreateEntry: '" << url << "'");
 758
 759     e = new StoreEntry(url, log_url);
 760     e->lock_count = 1;          /* Note lock here w/o calling storeLock() */
 761     mem = e->mem_obj;
 762     mem->method = method;
 763
 764     if (neighbors_do_private_keys || !flags.hierarchical)
 765         e->setPrivateKey();
 766     else
 767         e->setPublicKey();
 768
 769     if (flags.cachable) {
 770         EBIT_SET(e->flags, ENTRY_CACHABLE);
 771         EBIT_CLR(e->flags, RELEASE_REQUEST);
 772     } else {
 773         /* StoreEntry::releaseRequest() clears ENTRY_CACHABLE */
 774         e->releaseRequest();
 775     }
 776
 777     e->store_status = STORE_PENDING;
 778     e->setMemStatus(NOT_IN_MEMORY);
 779     e->swap_status = SWAPOUT_NONE;
 780     e->swap_filen = -1;
 781     e->swap_dirn = -1;
 782     e->refcount = 0;
 783     e->lastref = squid_curtime;
 784     e->timestamp = -1;          /* set in StoreEntry::timestampsSet() */
 785     e->ping_status = PING_NONE;
 786     EBIT_SET(e->flags, ENTRY_VALIDATED);
 787     return e;
 788 }
 789
 790 /* Mark object as expired */
 791 void
 792 StoreEntry::expireNow()
 793 {
 794     debugs(20, 3, "StoreEntry::expireNow: '" << getMD5Text() << "'");
 795     expires = squid_curtime;
 796 }
 797
 798 void
 799 storeWriteComplete (void *data, StoreIOBuffer wroteBuffer)
 800 {
 801     PROF_start(storeWriteComplete);
 802     StoreEntry *e = (StoreEntry *)data;
 803
 804     if (EBIT_TEST(e->flags, DELAY_SENDING)) {
 805         PROF_stop(storeWriteComplete);
 806         return;
 807     }
 808
 809     e->invokeHandlers();
 810     PROF_stop(storeWriteComplete);
 811 }
 812
 813 void
 814 StoreEntry::write (StoreIOBuffer writeBuffer)
 815 {
 816     assert(mem_obj != NULL);
 817     /* This assert will change when we teach the store to update */
 818     PROF_start(StoreEntry_write);
 819     assert(store_status == STORE_PENDING);
 820
 821     if (!writeBuffer.length) {
 822         /* the headers are received already, but we have not received
 823          * any body data. There are BROKEN abuses of HTTP which require
 824          * the headers to be passed along before any body data - see
 825          * http://developer.apple.com/documentation/QuickTime/QTSS/Concepts/chapter_2_section_14.html
 826          * for an example of such bad behaviour. To accomodate this, if
 827          * we have a empty write arrive, we flush to our clients.
 828          * -RBC 20060903
 829          */
 830         PROF_stop(StoreEntry_write);
 831         invokeHandlers();
 832         return;
 833     }
 834
 835     debugs(20, 5, "storeWrite: writing " << writeBuffer.length << " bytes for '" << getMD5Text() << "'");
 836     PROF_stop(StoreEntry_write);
 837     storeGetMemSpace(writeBuffer.length);
 838     mem_obj->write (writeBuffer, storeWriteComplete, this);
 839 }
 840
 841 /* Append incoming data from a primary server to an entry. */
 842 void
 843 StoreEntry::append(char const *buf, int len)
 844 {
 845     assert(mem_obj != NULL);
 846     assert(len >= 0);
 847     assert(store_status == STORE_PENDING);
 848
 849     StoreIOBuffer tempBuffer;
 850     tempBuffer.data = (char *)buf;
 851     tempBuffer.length = len;
 852     /*
 853      * XXX sigh, offset might be < 0 here, but it gets "corrected"
 854      * later.  This offset crap is such a mess.
 855      */
 856     tempBuffer.offset = mem_obj->endOffset() - (getReply() ? getReply()->hdr_sz : 0);
 857     write(tempBuffer);
 858 }
 859
 860
 861 void
 862 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
 863 {
 864     va_list args;
 865     va_start(args, fmt);
 866
 867     storeAppendVPrintf(e, fmt, args);
 868     va_end(args);
 869 }
 870
 871 /* used be storeAppendPrintf and Packer */
 872 void
 873 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
 874 {
 875     LOCAL_ARRAY(char, buf, 4096);
 876     buf[0] = '\0';
 877     vsnprintf(buf, 4096, fmt, vargs);
 878     e->append(buf, strlen(buf));
 879 }
 880
 881 struct _store_check_cachable_hist {
 882
 883     struct {
 884         int non_get;
 885         int not_entry_cachable;
 886         int wrong_content_length;
 887         int negative_cached;
 888         int too_big;
 889         int too_small;
 890         int private_key;
 891         int too_many_open_files;
 892         int too_many_open_fds;
 893     } no;
 894
 895     struct {
 896         int Default;
 897     } yes;
 898 } store_check_cachable_hist;
 899
 900 int
 901 storeTooManyDiskFilesOpen(void)
 902 {
 903     if (Config.max_open_disk_fds == 0)
 904         return 0;
 905
 906     if (store_open_disk_fd > Config.max_open_disk_fds)
 907         return 1;
 908
 909     return 0;
 910 }
 911
 912 int
 913 StoreEntry::checkTooSmall()
 914 {
 915     if (EBIT_TEST(flags, ENTRY_SPECIAL))
 916         return 0;
 917
 918     if (STORE_OK == store_status)
 919         if (mem_obj->object_sz < 0 ||
 920                 mem_obj->object_sz < Config.Store.minObjectSize)
 921             return 1;
 922     if (getReply()->content_length > -1)
 923         if (getReply()->content_length < Config.Store.minObjectSize)
 924             return 1;
 925     return 0;
 926 }
 927
 928 int
 929 StoreEntry::checkCachable()
 930 {
 931 #if CACHE_ALL_METHODS
 932
 933     if (mem_obj->method != METHOD_GET) {
 934         debugs(20, 2, "StoreEntry::checkCachable: NO: non-GET method");
 935         store_check_cachable_hist.no.non_get++;
 936     } else
 937 #endif
 938         if (store_status == STORE_OK && EBIT_TEST(flags, ENTRY_BAD_LENGTH)) {
 939             debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
 940             store_check_cachable_hist.no.wrong_content_length++;
 941         } else if (!EBIT_TEST(flags, ENTRY_CACHABLE)) {
 942             debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
 943             store_check_cachable_hist.no.not_entry_cachable++;
 944         } else if (EBIT_TEST(flags, ENTRY_NEGCACHED)) {
 945             debugs(20, 3, "StoreEntry::checkCachable: NO: negative cached");
 946             store_check_cachable_hist.no.negative_cached++;
 947             return 0;           /* avoid release call below */
 948         } else if ((getReply()->content_length > 0 &&
 949                     getReply()->content_length
 950                     > Config.Store.maxObjectSize) ||
 951                    mem_obj->endOffset() > Config.Store.maxObjectSize) {
 952             debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
 953             store_check_cachable_hist.no.too_big++;
 954         } else if (getReply()->content_length > Config.Store.maxObjectSize) {
 955             debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
 956             store_check_cachable_hist.no.too_big++;
 957         } else if (checkTooSmall()) {
 958             debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
 959             store_check_cachable_hist.no.too_small++;
 960         } else if (EBIT_TEST(flags, KEY_PRIVATE)) {
 961             debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
 962             store_check_cachable_hist.no.private_key++;
 963         } else if (swap_status != SWAPOUT_NONE) {
 964             /*
 965              * here we checked the swap_status because the remaining
 966              * cases are only relevant only if we haven't started swapping
 967              * out the object yet.
 968              */
 969             return 1;
 970         } else if (storeTooManyDiskFilesOpen()) {
 971             debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
 972             store_check_cachable_hist.no.too_many_open_files++;
 973         } else if (fdNFree() < RESERVED_FD) {
 974             debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
 975             store_check_cachable_hist.no.too_many_open_fds++;
 976         } else {
 977             store_check_cachable_hist.yes.Default++;
 978             return 1;
 979         }
 980
 981     releaseRequest();
 982     /* StoreEntry::releaseRequest() cleared ENTRY_CACHABLE */
 983     return 0;
 984 }
 985
 986 void
 987 storeCheckCachableStats(StoreEntry *sentry)
 988 {
 989     storeAppendPrintf(sentry, "Category\t Count\n");
 990
 991 #if CACHE_ALL_METHODS
 992
 993     storeAppendPrintf(sentry, "no.non_get\t%d\n",
 994                       store_check_cachable_hist.no.non_get);
 995 #endif
 996
 997     storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
 998                       store_check_cachable_hist.no.not_entry_cachable);
 999     storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
1000                       store_check_cachable_hist.no.wrong_content_length);
1001     storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
1002                       store_check_cachable_hist.no.negative_cached);
1003     storeAppendPrintf(sentry, "no.too_big\t%d\n",
1004                       store_check_cachable_hist.no.too_big);
1005     storeAppendPrintf(sentry, "no.too_small\t%d\n",
1006                       store_check_cachable_hist.no.too_small);
1007     storeAppendPrintf(sentry, "no.private_key\t%d\n",
1008                       store_check_cachable_hist.no.private_key);
1009     storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
1010                       store_check_cachable_hist.no.too_many_open_files);
1011     storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
1012                       store_check_cachable_hist.no.too_many_open_fds);
1013     storeAppendPrintf(sentry, "yes.default\t%d\n",
1014                       store_check_cachable_hist.yes.Default);
1015 }
1016
1017 void
1018 StoreEntry::complete()
1019 {
1020     debugs(20, 3, "storeComplete: '" << getMD5Text() << "'");
1021
1022     if (store_status != STORE_PENDING) {
1023         /*
1024          * if we're not STORE_PENDING, then probably we got aborted
1025          * and there should be NO clients on this entry
1026          */
1027         assert(EBIT_TEST(flags, ENTRY_ABORTED));
1028         assert(mem_obj->nclients == 0);
1029         return;
1030     }
1031
1032     /* This is suspect: mem obj offsets include the headers. do we adjust for that
1033      * in use of object_sz?
1034      */
1035     mem_obj->object_sz = mem_obj->endOffset();
1036
1037     store_status = STORE_OK;
1038
1039     assert(mem_status == NOT_IN_MEMORY);
1040
1041     if (!validLength()) {
1042         EBIT_SET(flags, ENTRY_BAD_LENGTH);
1043         releaseRequest();
1044     }
1045
1046 #if USE_CACHE_DIGESTS
1047     if (mem_obj->request)
1048         mem_obj->request->hier.store_complete_stop = current_time;
1049
1050 #endif
1051     /*
1052      * We used to call invokeHandlers, then storeSwapOut.  However,
1053      * Madhukar Reddy <myreddy@persistence.com> reported that
1054      * responses without content length would sometimes get released
1055      * in client_side, thinking that the response is incomplete.
1056      */
1057     invokeHandlers();
1058 }
1059
1060 /*
1061  * Someone wants to abort this transfer.  Set the reason in the
1062  * request structure, call the server-side callback and mark the
1063  * entry for releasing
1064  */
1065 void
1066 StoreEntry::abort()
1067 {
1068     statCounter.aborted_requests++;
1069     assert(store_status == STORE_PENDING);
1070     assert(mem_obj != NULL);
1071     debugs(20, 6, "storeAbort: " << getMD5Text());
1072
1073     lock();         /* lock while aborting */
1074     negativeCache();
1075
1076     releaseRequest();
1077
1078     EBIT_SET(flags, ENTRY_ABORTED);
1079
1080     setMemStatus(NOT_IN_MEMORY);
1081
1082     store_status = STORE_OK;
1083
1084     /*
1085      * We assign an object length here.  The only other place we assign
1086      * the object length is in storeComplete()
1087      */
1088     /* RBC: What do we need an object length for? we've just aborted the
1089      * request, the request is private and negatively cached. Surely
1090      * the object length is inappropriate to set.
1091      */
1092     mem_obj->object_sz = mem_obj->endOffset();
1093
1094     /* Notify the server side */
1095
1096     /*
1097      * DPW 2007-05-07
1098      * Should we check abort.data for validity?
1099      */
1100     if (mem_obj->abort.callback) {
1101         if (!cbdataReferenceValid(mem_obj->abort.data))
1102             debugs(20,1,HERE << "queueing event when abort.data is not valid");
1103         eventAdd("mem_obj->abort.callback",
1104                  mem_obj->abort.callback,
1105                  mem_obj->abort.data,
1106                  0.0,
1107                  true);
1108         unregisterAbort();
1109     }
1110
1111     /* XXX Should we reverse these two, so that there is no
1112      * unneeded disk swapping triggered?
1113      */
1114     /* Notify the client side */
1115     invokeHandlers();
1116
1117     /* Close any swapout file */
1118     swapOutFileClose();
1119
1120     unlock();       /* unlock */
1121 }
1122
1123 /**
1124  * Clear Memory storage to accommodate the given object len
1125  */
1126 void
1127 storeGetMemSpace(int size)
1128 {
1129     PROF_start(storeGetMemSpace);
1130     StoreEntry *e = NULL;
1131     int released = 0;
1132     static time_t last_check = 0;
1133     size_t pages_needed;
1134     RemovalPurgeWalker *walker;
1135
1136     if (squid_curtime == last_check) {
1137         PROF_stop(storeGetMemSpace);
1138         return;
1139     }
1140
1141     last_check = squid_curtime;
1142
1143     pages_needed = (size / SM_PAGE_SIZE) + 1;
1144
1145     if (mem_node::InUseCount() + pages_needed < store_pages_max) {
1146         PROF_stop(storeGetMemSpace);
1147         return;
1148     }
1149
1150     debugs(20, 2, "storeGetMemSpace: Starting, need " << pages_needed <<
1151            " pages");
1152
1153     /* XXX what to set as max_scan here? */
1154     walker = mem_policy->PurgeInit(mem_policy, 100000);
1155
1156     while ((e = walker->Next(walker))) {
1157         e->purgeMem();
1158         released++;
1159
1160         if (mem_node::InUseCount() + pages_needed < store_pages_max)
1161             break;
1162     }
1163
1164     walker->Done(walker);
1165     debugs(20, 3, "storeGetMemSpace stats:");
1166     debugs(20, 3, "  " << std::setw(6) << hot_obj_count  << " HOT objects");
1167     debugs(20, 3, "  " << std::setw(6) << released  << " were released");
1168     PROF_stop(storeGetMemSpace);
1169 }
1170
1171
1172 /* thunk through to Store::Root().maintain(). Note that this would be better still
1173  * if registered against the root store itself, but that requires more complex
1174  * update logic - bigger fish to fry first. Long term each store when
1175  * it becomes active will self register
1176  */
1177 void
1178 Store::Maintain(void *notused)
1179 {
1180     Store::Root().maintain();
1181
1182     /* Reregister a maintain event .. */
1183     eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1);
1184
1185 }
1186
1187 /* The maximum objects to scan for maintain storage space */
1188 #define MAINTAIN_MAX_SCAN       1024
1189 #define MAINTAIN_MAX_REMOVE     64
1190
1191 /*
1192  * This routine is to be called by main loop in main.c.
1193  * It removes expired objects on only one bucket for each time called.
1194  *
1195  * This should get called 1/s from main().
1196  */
1197 void
1198 StoreController::maintain()
1199 {
1200     static time_t last_warn_time = 0;
1201
1202     PROF_start(storeMaintainSwapSpace);
1203     swapDir->maintain();
1204
1205     /* this should be emitted by the oversize dir, not globally */
1206
1207     if (store_swap_size > Store::Root().maxSize()) {
1208         if (squid_curtime - last_warn_time > 10) {
1209             debugs(20, 0, "WARNING: Disk space over limit: " << store_swap_size << " KB > "
1210                    << Store::Root().maxSize() << " KB");
1211             last_warn_time = squid_curtime;
1212         }
1213     }
1214
1215     PROF_stop(storeMaintainSwapSpace);
1216 }
1217
1218 /* release an object from a cache */
1219 void
1220 StoreEntry::release()
1221 {
1222     PROF_start(storeRelease);
1223     debugs(20, 3, "storeRelease: Releasing: '" << getMD5Text() << "'");
1224     /* If, for any reason we can't discard this object because of an
1225      * outstanding request, mark it for pending release */
1226
1227     if (locked()) {
1228         expireNow();
1229         debugs(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit");
1230         releaseRequest();
1231         PROF_stop(storeRelease);
1232         return;
1233     }
1234
1235     if (StoreController::store_dirs_rebuilding && swap_filen > -1) {
1236         setPrivateKey();
1237
1238         if (mem_obj)
1239             destroyMemObject();
1240
1241         if (swap_filen > -1) {
1242             /*
1243              * Fake a call to StoreEntry->lock()  When rebuilding is done,
1244              * we'll just call StoreEntry->unlock() on these.
1245              */
1246             lock_count++;
1247             setReleaseFlag();
1248             LateReleaseStack.push_back(this);
1249             PROF_stop(storeRelease);
1250             return;
1251         } else {
1252             destroyStoreEntry(static_cast<hash_link *>(this));
1253         }
1254     }
1255
1256     storeLog(STORE_LOG_RELEASE, this);
1257
1258     if (swap_filen > -1) {
1259         unlink();
1260
1261         if (swap_status == SWAPOUT_DONE)
1262             if (EBIT_TEST(flags, ENTRY_VALIDATED))
1263                 store()->updateSize(swap_file_sz, -1);
1264
1265         if (!EBIT_TEST(flags, KEY_PRIVATE))
1266             storeDirSwapLog(this, SWAP_LOG_DEL);
1267
1268 #if 0
1269         /* From 2.4. I think we do this in storeUnlink? */
1270         storeSwapFileNumberSet(this, -1);
1271
1272 #endif
1273
1274     }
1275
1276     setMemStatus(NOT_IN_MEMORY);
1277     destroyStoreEntry(static_cast<hash_link *>(this));
1278     PROF_stop(storeRelease);
1279 }
1280
1281 static void
1282 storeLateRelease(void *unused)
1283 {
1284     StoreEntry *e;
1285     int i;
1286     static int n = 0;
1287
1288     if (StoreController::store_dirs_rebuilding) {
1289         eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1290         return;
1291     }
1292
1293     for (i = 0; i < 10; i++) {
1294         e = LateReleaseStack.pop();
1295
1296         if (e == NULL) {
1297             /* done! */
1298             debugs(20, 1, "storeLateRelease: released " << n << " objects");
1299             return;
1300         }
1301
1302         e->unlock();
1303         n++;
1304     }
1305
1306     eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
1307 }
1308
1309 /* return 1 if a store entry is locked */
1310 int
1311 StoreEntry::locked() const
1312 {
1313     if (lock_count)
1314         return 1;
1315
1316     if (swap_status == SWAPOUT_WRITING)
1317         return 1;
1318
1319     if (store_status == STORE_PENDING)
1320         return 1;
1321
1322     /*
1323      * SPECIAL, PUBLIC entries should be "locked"
1324      */
1325     if (EBIT_TEST(flags, ENTRY_SPECIAL))
1326         if (!EBIT_TEST(flags, KEY_PRIVATE))
1327             return 1;
1328
1329     return 0;
1330 }
1331
1332 bool
1333 StoreEntry::validLength() const
1334 {
1335     int64_t diff;
1336     const HttpReply *reply;
1337     assert(mem_obj != NULL);
1338     reply = getReply();
1339     debugs(20, 3, "storeEntryValidLength: Checking '" << getMD5Text() << "'");
1340     debugs(20, 5, "storeEntryValidLength:     object_len = " <<
1341            objectLen());
1342     debugs(20, 5, "storeEntryValidLength:         hdr_sz = " << reply->hdr_sz);
1343     debugs(20, 5, "storeEntryValidLength: content_length = " << reply->content_length);
1344
1345     if (reply->content_length < 0) {
1346         debugs(20, 5, "storeEntryValidLength: Unspecified content length: " << getMD5Text());
1347         return 1;
1348     }
1349
1350     if (reply->hdr_sz == 0) {
1351         debugs(20, 5, "storeEntryValidLength: Zero header size: " << getMD5Text());
1352         return 1;
1353     }
1354
1355     if (mem_obj->method == METHOD_HEAD) {
1356         debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
1357         return 1;
1358     }
1359
1360     if (reply->sline.status == HTTP_NOT_MODIFIED)
1361         return 1;
1362
1363     if (reply->sline.status == HTTP_NO_CONTENT)
1364         return 1;
1365
1366     diff = reply->hdr_sz + reply->content_length - objectLen();
1367
1368     if (diff == 0)
1369         return 1;
1370
1371     debugs(20, 3, "storeEntryValidLength: " << (diff < 0 ? -diff : diff)  << " bytes too " << (diff < 0 ? "big" : "small") <<"; '" << getMD5Text() << "'" );
1372
1373     return 0;
1374 }
1375
1376 static void
1377 storeRegisterWithCacheManager(void)
1378 {
1379     Mgr::RegisterAction("storedir", "Store Directory Stats", Store::Stats, 0, 1);
1380     Mgr::RegisterAction("store_io", "Store IO Interface Stats", &Mgr::StoreIoAction::Create, 0, 1);
1381     Mgr::RegisterAction("store_check_cachable_stats", "storeCheckCachable() Stats",
1382                         storeCheckCachableStats, 0, 1);
1383 }
1384
1385 void
1386 storeInit(void)
1387 {
1388     storeKeyInit();
1389     mem_policy = createRemovalPolicy(Config.memPolicy);
1390     storeDigestInit();
1391     storeLogOpen();
1392     eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1393     Store::Root().init();
1394     storeRebuildStart();
1395
1396     storeRegisterWithCacheManager();
1397 }
1398
1399 void
1400 storeConfigure(void)
1401 {
1402     store_swap_high = (long) (((float) Store::Root().maxSize() *
1403                                (float) Config.Swap.highWaterMark) / (float) 100);
1404     store_swap_low = (long) (((float) Store::Root().maxSize() *
1405                               (float) Config.Swap.lowWaterMark) / (float) 100);
1406     store_pages_max = Config.memMaxSize / sizeof(mem_node);
1407 }
1408
1409 int
1410 StoreEntry::keepInMemory() const
1411 {
1412     if (mem_obj == NULL)
1413         return 0;
1414
1415     if (mem_obj->data_hdr.size() == 0)
1416         return 0;
1417
1418     if (mem_obj->inmem_lo != 0)
1419         return 0;
1420
1421     if (!Config.onoff.memory_cache_first && swap_status == SWAPOUT_DONE && refcount == 1)
1422         return 0;
1423
1424     return 1;
1425 }
1426
1427 int
1428 StoreEntry::checkNegativeHit() const
1429 {
1430     if (!EBIT_TEST(flags, ENTRY_NEGCACHED))
1431         return 0;
1432
1433     if (expires <= squid_curtime)
1434         return 0;
1435
1436     if (store_status != STORE_OK)
1437         return 0;
1438
1439     return 1;
1440 }
1441
1442 /**
1443  * Set object for negative caching.
1444  * Preserves any expiry information given by the server.
1445  * In absence of proper expiry info it will set to expire immediately,
1446  * or with HTTP-violations enabled the configured negative-TTL is observed
1447  */
1448 void
1449 StoreEntry::negativeCache()
1450 {
1451     if (expires == 0)
1452 #if USE_HTTP_VIOLATIONS
1453         expires = squid_curtime + Config.negativeTtl;
1454 #else
1455         expires = squid_curtime;
1456 #endif
1457     EBIT_SET(flags, ENTRY_NEGCACHED);
1458 }
1459
1460 void
1461 storeFreeMemory(void)
1462 {
1463     Store::Root(NULL);
1464 #if USE_CACHE_DIGESTS
1465
1466     if (store_digest)
1467         cacheDigestDestroy(store_digest);
1468
1469 #endif
1470
1471     store_digest = NULL;
1472 }
1473
1474 int
1475 expiresMoreThan(time_t expires, time_t when)
1476 {
1477     if (expires < 0)            /* No Expires given */
1478         return 1;
1479
1480     return (expires > (squid_curtime + when));
1481 }
1482
1483 int
1484 StoreEntry::validToSend() const
1485 {
1486     if (EBIT_TEST(flags, RELEASE_REQUEST))
1487         return 0;
1488
1489     if (EBIT_TEST(flags, ENTRY_NEGCACHED))
1490         if (expires <= squid_curtime)
1491             return 0;
1492
1493     if (EBIT_TEST(flags, ENTRY_ABORTED))
1494         return 0;
1495
1496     return 1;
1497 }
1498
1499 void
1500 StoreEntry::timestampsSet()
1501 {
1502     const HttpReply *reply = getReply();
1503     time_t served_date = reply->date;
1504     int age = reply->header.getInt(HDR_AGE);
1505     /* Compute the timestamp, mimicking RFC2616 section 13.2.3. */
1506     /* make sure that 0 <= served_date <= squid_curtime */
1507
1508     if (served_date < 0 || served_date > squid_curtime)
1509         served_date = squid_curtime;
1510
1511     /*
1512      * Compensate with Age header if origin server clock is ahead
1513      * of us and there is a cache in between us and the origin
1514      * server.  But DONT compensate if the age value is larger than
1515      * squid_curtime because it results in a negative served_date.
1516      */
1517     if (age > squid_curtime - served_date)
1518         if (squid_curtime > age)
1519             served_date = squid_curtime - age;
1520
1521     // compensate for Squid-to-server and server-to-Squid delays
1522     if (mem_obj && mem_obj->request) {
1523         const time_t request_sent =
1524             mem_obj->request->hier.peer_http_request_sent.tv_sec;
1525         if (0 < request_sent && request_sent < squid_curtime)
1526             served_date -= (squid_curtime - request_sent);
1527     }
1528
1529     if (reply->expires > 0 && reply->date > -1)
1530         expires = served_date + (reply->expires - reply->date);
1531     else
1532         expires = reply->expires;
1533
1534     lastmod = reply->last_modified;
1535
1536     timestamp = served_date;
1537 }
1538
1539 void
1540 StoreEntry::registerAbort(STABH * cb, void *data)
1541 {
1542     assert(mem_obj);
1543     assert(mem_obj->abort.callback == NULL);
1544     mem_obj->abort.callback = cb;
1545     mem_obj->abort.data = cbdataReference(data);
1546 }
1547
1548 void
1549 StoreEntry::unregisterAbort()
1550 {
1551     assert(mem_obj);
1552     if (mem_obj->abort.callback) {
1553         mem_obj->abort.callback = NULL;
1554         cbdataReferenceDone(mem_obj->abort.data);
1555     }
1556 }
1557
1558 void
1559 StoreEntry::dump(int l) const
1560 {
1561     debugs(20, l, "StoreEntry->key: " << getMD5Text());
1562     debugs(20, l, "StoreEntry->next: " << next);
1563     debugs(20, l, "StoreEntry->mem_obj: " << mem_obj);
1564     debugs(20, l, "StoreEntry->timestamp: " << timestamp);
1565     debugs(20, l, "StoreEntry->lastref: " << lastref);
1566     debugs(20, l, "StoreEntry->expires: " << expires);
1567     debugs(20, l, "StoreEntry->lastmod: " << lastmod);
1568     debugs(20, l, "StoreEntry->swap_file_sz: " << swap_file_sz);
1569     debugs(20, l, "StoreEntry->refcount: " << refcount);
1570     debugs(20, l, "StoreEntry->flags: " << storeEntryFlags(this));
1571     debugs(20, l, "StoreEntry->swap_dirn: " << swap_dirn);
1572     debugs(20, l, "StoreEntry->swap_filen: " << swap_filen);
1573     debugs(20, l, "StoreEntry->lock_count: " << lock_count);
1574     debugs(20, l, "StoreEntry->mem_status: " << mem_status);
1575     debugs(20, l, "StoreEntry->ping_status: " << ping_status);
1576     debugs(20, l, "StoreEntry->store_status: " << store_status);
1577     debugs(20, l, "StoreEntry->swap_status: " << swap_status);
1578 }
1579
1580 /*
1581  * NOTE, this function assumes only two mem states
1582  */
1583 void
1584 StoreEntry::setMemStatus(mem_status_t new_status)
1585 {
1586     if (new_status == mem_status)
1587         return;
1588
1589     assert(mem_obj != NULL);
1590
1591     if (new_status == IN_MEMORY) {
1592         assert(mem_obj->inmem_lo == 0);
1593
1594         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1595             debugs(20, 4, "StoreEntry::setMemStatus: not inserting special " << mem_obj->url << " into policy");
1596         } else {
1597             mem_policy->Add(mem_policy, this, &mem_obj->repl);
1598             debugs(20, 4, "StoreEntry::setMemStatus: inserted mem node " << mem_obj->url);
1599         }
1600
1601         hot_obj_count++;
1602     } else {
1603         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1604             debugs(20, 4, "StoreEntry::setMemStatus: special entry " << mem_obj->url);
1605         } else {
1606             mem_policy->Remove(mem_policy, this, &mem_obj->repl);
1607             debugs(20, 4, "StoreEntry::setMemStatus: removed mem node " << mem_obj->url);
1608         }
1609
1610         hot_obj_count--;
1611     }
1612
1613     mem_status = new_status;
1614 }
1615
1616 const char *
1617 StoreEntry::url() const
1618 {
1619     if (this == NULL)
1620         return "[null_entry]";
1621     else if (mem_obj == NULL)
1622         return "[null_mem_obj]";
1623     else
1624         return mem_obj->url;
1625 }
1626
1627 void
1628 StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl)
1629 {
1630     if (mem_obj)
1631         return;
1632
1633     mem_obj = new MemObject(aUrl, aLogUrl);
1634 }
1635
1636 /* this just sets DELAY_SENDING */
1637 void
1638 StoreEntry::buffer()
1639 {
1640     EBIT_SET(flags, DELAY_SENDING);
1641 }
1642
1643 /* this just clears DELAY_SENDING and Invokes the handlers */
1644 void
1645 StoreEntry::flush()
1646 {
1647     if (EBIT_TEST(flags, DELAY_SENDING)) {
1648         EBIT_CLR(flags, DELAY_SENDING);
1649         invokeHandlers();
1650     }
1651 }
1652
1653 int64_t
1654 StoreEntry::objectLen() const
1655 {
1656     assert(mem_obj != NULL);
1657     return mem_obj->object_sz;
1658 }
1659
1660 int64_t
1661 StoreEntry::contentLen() const
1662 {
1663     assert(mem_obj != NULL);
1664     assert(getReply() != NULL);
1665     return objectLen() - getReply()->hdr_sz;
1666 }
1667
1668 HttpReply const *
1669 StoreEntry::getReply () const
1670 {
1671     if (NULL == mem_obj)
1672         return NULL;
1673
1674     return mem_obj->getReply();
1675 }
1676
1677 void
1678 StoreEntry::reset()
1679 {
1680     assert (mem_obj);
1681     debugs(20, 3, "StoreEntry::reset: " << url());
1682     mem_obj->reset();
1683     HttpReply *rep = (HttpReply *) getReply();       // bypass const
1684     rep->reset();
1685     expires = lastmod = timestamp = -1;
1686 }
1687
1688 /*
1689  * storeFsInit
1690  *
1691  * This routine calls the SETUP routine for each fs type.
1692  * I don't know where the best place for this is, and I'm not going to shuffle
1693  * around large chunks of code right now (that can be done once its working.)
1694  */
1695 void
1696 storeFsInit(void)
1697 {
1698     storeReplSetup();
1699 }
1700
1701 /*
1702  * called to add another store removal policy module
1703  */
1704 void
1705 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1706 {
1707     int i;
1708
1709     /* find the number of currently known repl types */
1710     for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
1711         if (strcmp(storerepl_list[i].typestr, type) == 0) {
1712             debugs(20, 1, "WARNING: Trying to load store replacement policy " << type << " twice.");
1713             return;
1714         }
1715     }
1716
1717     /* add the new type */
1718     storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
1719
1720     memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1721
1722     storerepl_list[i].typestr = type;
1723
1724     storerepl_list[i].create = create;
1725 }
1726
1727 /*
1728  * Create a removal policy instance
1729  */
1730 RemovalPolicy *
1731 createRemovalPolicy(RemovalPolicySettings * settings)
1732 {
1733     storerepl_entry_t *r;
1734
1735     for (r = storerepl_list; r && r->typestr; r++) {
1736         if (strcmp(r->typestr, settings->type) == 0)
1737             return r->create(settings->args);
1738     }
1739
1740     debugs(20, 1, "ERROR: Unknown policy " << settings->type);
1741     debugs(20, 1, "ERROR: Be sure to have set cache_replacement_policy");
1742     debugs(20, 1, "ERROR:   and memory_replacement_policy in squid.conf!");
1743     fatalf("ERROR: Unknown policy %s\n", settings->type);
1744     return NULL;                /* NOTREACHED */
1745 }
1746
1747 #if 0
1748 void
1749 storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
1750 {
1751     if (e->swap_file_number == filn)
1752         return;
1753
1754     if (filn < 0) {
1755         assert(-1 == filn);
1756         storeDirMapBitReset(e->swap_file_number);
1757         storeDirLRUDelete(e);
1758         e->swap_file_number = -1;
1759     } else {
1760         assert(-1 == e->swap_file_number);
1761         storeDirMapBitSet(e->swap_file_number = filn);
1762         storeDirLRUAdd(e);
1763     }
1764 }
1765
1766 #endif
1767
1768
1769 /*
1770  * Replace a store entry with
1771  * a new reply. This eats the reply.
1772  */
1773 void
1774 StoreEntry::replaceHttpReply(HttpReply *rep)
1775 {
1776     debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
1777     Packer p;
1778
1779     if (!mem_obj) {
1780         debugs(20, 0, "Attempt to replace object with no in-memory representation");
1781         return;
1782     }
1783
1784     mem_obj->replaceHttpReply(rep);
1785
1786     /* TODO: when we store headers serparately remove the header portion */
1787     /* TODO: mark the length of the headers ? */
1788     /* We ONLY want the headers */
1789     packerToStoreInit(&p, this);
1790
1791     assert (isEmpty());
1792
1793     getReply()->packHeadersInto(&p);
1794
1795     rep->hdr_sz = mem_obj->endOffset();
1796
1797     httpBodyPackInto(&getReply()->body, &p);
1798
1799     packerClean(&p);
1800 }
1801
1802
1803 char const *
1804 StoreEntry::getSerialisedMetaData()
1805 {
1806     StoreMeta *tlv_list = storeSwapMetaBuild(this);
1807     int swap_hdr_sz;
1808     char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
1809     storeSwapTLVFree(tlv_list);
1810     assert (swap_hdr_sz >= 0);
1811     mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz;
1812     return result;
1813 }
1814
1815 bool
1816 StoreEntry::swapoutPossible()
1817 {
1818     /* should we swap something out to disk? */
1819     debugs(20, 7, "storeSwapOut: " << url());
1820     debugs(20, 7, "storeSwapOut: store_status = " << storeStatusStr[store_status]);
1821
1822     if (EBIT_TEST(flags, ENTRY_ABORTED)) {
1823         assert(EBIT_TEST(flags, RELEASE_REQUEST));
1824         swapOutFileClose();
1825         return false;
1826     }
1827
1828     if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1829         debugs(20, 3, "storeSwapOut: " << url() << " SPECIAL");
1830         return false;
1831     }
1832
1833     return true;
1834 }
1835
1836 void
1837 StoreEntry::trimMemory()
1838 {
1839     /*
1840      * DPW 2007-05-09
1841      * Bug #1943.  We must not let go any data for IN_MEMORY
1842      * objects.  We have to wait until the mem_status changes.
1843      */
1844     if (mem_status == IN_MEMORY)
1845         return;
1846
1847     if (!swapOutAble()) {
1848         if (mem_obj->policyLowestOffsetToKeep(0) == 0) {
1849             /* Nothing to do */
1850             return;
1851         }
1852         /*
1853          * Its not swap-able, and we're about to delete a chunk,
1854          * so we must make it PRIVATE.  This is tricky/ugly because
1855          * for the most part, we treat swapable == cachable here.
1856          */
1857         releaseRequest();
1858         mem_obj->trimUnSwappable ();
1859     } else {
1860         mem_obj->trimSwappable ();
1861     }
1862 }
1863
1864 bool
1865 StoreEntry::modifiedSince(HttpRequest * request) const
1866 {
1867     int object_length;
1868     time_t mod_time = lastmod;
1869
1870     if (mod_time < 0)
1871         mod_time = timestamp;
1872
1873     debugs(88, 3, "modifiedSince: '" << url() << "'");
1874
1875     debugs(88, 3, "modifiedSince: mod_time = " << mod_time);
1876
1877     if (mod_time < 0)
1878         return true;
1879
1880     /* Find size of the object */
1881     object_length = getReply()->content_length;
1882
1883     if (object_length < 0)
1884         object_length = contentLen();
1885
1886     if (mod_time > request->ims) {
1887         debugs(88, 3, "--> YES: entry newer than client");
1888         return true;
1889     } else if (mod_time < request->ims) {
1890         debugs(88, 3, "-->  NO: entry older than client");
1891         return false;
1892     } else if (request->imslen < 0) {
1893         debugs(88, 3, "-->  NO: same LMT, no client length");
1894         return false;
1895     } else if (request->imslen == object_length) {
1896         debugs(88, 3, "-->  NO: same LMT, same length");
1897         return false;
1898     } else {
1899         debugs(88, 3, "--> YES: same LMT, different length");
1900         return true;
1901     }
1902 }
1903
1904 bool
1905 StoreEntry::hasIfMatchEtag(const HttpRequest &request) const
1906 {
1907     const String reqETags = request.header.getList(HDR_IF_MATCH);
1908     return hasOneOfEtags(reqETags, false);
1909 }
1910
1911 bool
1912 StoreEntry::hasIfNoneMatchEtag(const HttpRequest &request) const
1913 {
1914     const String reqETags = request.header.getList(HDR_IF_NONE_MATCH);
1915     // weak comparison is allowed only for HEAD or full-body GET requests
1916     const bool allowWeakMatch = !request.flags.range &&
1917                                 (request.method == METHOD_GET || request.method == METHOD_HEAD);
1918     return hasOneOfEtags(reqETags, allowWeakMatch);
1919 }
1920
1921 /// whether at least one of the request ETags matches entity ETag
1922 bool
1923 StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
1924 {
1925     const ETag repETag = getReply()->header.getETag(HDR_ETAG);
1926     if (!repETag.str)
1927         return strListIsMember(&reqETags, "*", ',');
1928
1929     bool matched = false;
1930     const char *pos = NULL;
1931     const char *item;
1932     int ilen;
1933     while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) {
1934         if (!strncmp(item, "*", ilen))
1935             matched = true;
1936         else {
1937             String str;
1938             str.append(item, ilen);
1939             ETag reqETag;
1940             if (etagParseInit(&reqETag, str.termedBuf())) {
1941                 matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) :
1942                           etagIsStrongEqual(repETag, reqETag);
1943             }
1944         }
1945     }
1946     return matched;
1947 }
1948
1949 StorePointer
1950 StoreEntry::store() const
1951 {
1952     assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured);
1953     return INDEXSD(swap_dirn);
1954 }
1955
1956 void
1957 StoreEntry::unlink()
1958 {
1959     store()->unlink(*this);
1960 }
1961
1962 /*
1963  * return true if the entry is in a state where
1964  * it can accept more data (ie with write() method)
1965  */
1966 bool
1967 StoreEntry::isAccepting() const
1968 {
1969     if (STORE_PENDING != store_status)
1970         return false;
1971
1972     if (EBIT_TEST(flags, ENTRY_ABORTED))
1973         return false;
1974
1975     return true;
1976 }
1977
1978 /* NullStoreEntry */
1979
1980 NullStoreEntry NullStoreEntry::_instance;
1981
1982 NullStoreEntry *
1983 NullStoreEntry::getInstance()
1984 {
1985     return &_instance;
1986 }
1987
1988 char const *
1989 NullStoreEntry::getMD5Text() const
1990 {
1991     return "N/A";
1992 }
1993
1994 void
1995 NullStoreEntry::operator delete(void*)
1996 {
1997     fatal ("Attempt to delete NullStoreEntry\n");
1998 }
1999
2000 char const *
2001 NullStoreEntry::getSerialisedMetaData()
2002 {
2003     return NULL;
2004 }
2005
2006 #if !_USE_INLINE_
2007 #include "Store.cci"
2008 #endif