src/store.cc

   1
   2 /*
   3  * DEBUG: section 20    Storage Manager
   4  * AUTHOR: Harvest Derived
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  */
  33
  34 #include "squid.h"
  35 #include "CacheDigest.h"
  36 #include "CacheManager.h"
  37 #include "comm/Connection.h"
  38 #include "ETag.h"
  39 #include "event.h"
  40 #include "fde.h"
  41 #include "globals.h"
  42 #include "http.h"
  43 #include "HttpReply.h"
  44 #include "HttpRequest.h"
  45 #include "mem_node.h"
  46 #include "MemObject.h"
  47 #include "mgr/Registration.h"
  48 #include "mgr/StoreIoAction.h"
  49 #include "profiler/Profiler.h"
  50 #include "repl_modules.h"
  51 #include "RequestFlags.h"
  52 #include "SquidConfig.h"
  53 #include "SquidTime.h"
  54 #include "Stack.h"
  55 #include "StatCounters.h"
  56 #include "stmem.h"
  57 #include "store_digest.h"
  58 #include "store_key_md5.h"
  59 #include "store_key_md5.h"
  60 #include "store_log.h"
  61 #include "store_rebuild.h"
  62 #include "Store.h"
  63 #include "StoreClient.h"
  64 #include "StoreIOState.h"
  65 #include "StoreMeta.h"
  66 #include "StrList.h"
  67 #include "swap_log_op.h"
  68 #include "SwapDir.h"
  69 #include "tools.h"
  70 #if USE_DELAY_POOLS
  71 #include "DelayPools.h"
  72 #endif
  73 #if HAVE_LIMITS_H
  74 #include <limits.h>
  75 #endif
  76
  77 #define REBUILD_TIMESTAMP_DELTA_MAX 2
  78
  79 #define STORE_IN_MEM_BUCKETS            (229)
  80
  81 /** \todo Convert these string constants to enum string-arrays generated */
  82
  83 const char *memStatusStr[] = {
  84     "NOT_IN_MEMORY",
  85     "IN_MEMORY"
  86 };
  87
  88 const char *pingStatusStr[] = {
  89     "PING_NONE",
  90     "PING_WAITING",
  91     "PING_DONE"
  92 };
  93
  94 const char *storeStatusStr[] = {
  95     "STORE_OK",
  96     "STORE_PENDING"
  97 };
  98
  99 const char *swapStatusStr[] = {
 100     "SWAPOUT_NONE",
 101     "SWAPOUT_WRITING",
 102     "SWAPOUT_DONE"
 103 };
 104
 105 /*
 106  * This defines an repl type
 107  */
 108
 109 typedef struct _storerepl_entry storerepl_entry_t;
 110
 111 struct _storerepl_entry {
 112     const char *typestr;
 113     REMOVALPOLICYCREATE *create;
 114 };
 115
 116 static storerepl_entry_t *storerepl_list = NULL;
 117
 118 /*
 119  * local function prototypes
 120  */
 121 static int getKeyCounter(void);
 122 static OBJH storeCheckCachableStats;
 123 static EVH storeLateRelease;
 124
 125 /*
 126  * local variables
 127  */
 128 static Stack<StoreEntry*> LateReleaseStack;
 129 MemAllocator *StoreEntry::pool = NULL;
 130
 131 StorePointer Store::CurrentRoot = NULL;
 132
 133 void
 134 Store::Root(Store * aRoot)
 135 {
 136     CurrentRoot = aRoot;
 137 }
 138
 139 void
 140 Store::Root(StorePointer aRoot)
 141 {
 142     Root(aRoot.getRaw());
 143 }
 144
 145 void
 146 Store::Stats(StoreEntry * output)
 147 {
 148     assert (output);
 149     Root().stat(*output);
 150 }
 151
 152 void
 153 Store::create()
 154 {}
 155
 156 void
 157 Store::diskFull()
 158 {}
 159
 160 void
 161 Store::sync()
 162 {}
 163
 164 void
 165 Store::unlink (StoreEntry &anEntry)
 166 {
 167     fatal("Store::unlink on invalid Store\n");
 168 }
 169
 170 void *
 171 StoreEntry::operator new (size_t bytecount)
 172 {
 173     assert (bytecount == sizeof (StoreEntry));
 174
 175     if (!pool) {
 176         pool = memPoolCreate ("StoreEntry", bytecount);
 177         pool->setChunkSize(2048 * 1024);
 178     }
 179
 180     return pool->alloc();
 181 }
 182
 183 void
 184 StoreEntry::operator delete (void *address)
 185 {
 186     pool->freeOne(address);
 187 }
 188
 189 void
 190 StoreEntry::makePublic()
 191 {
 192     /* This object can be cached for a long time */
 193
 194     if (!EBIT_TEST(flags, RELEASE_REQUEST))
 195         setPublicKey();
 196 }
 197
 198 void
 199 StoreEntry::makePrivate()
 200 {
 201     /* This object should never be cached at all */
 202     expireNow();
 203     releaseRequest(); /* delete object when not used */
 204 }
 205
 206 void
 207 StoreEntry::cacheNegatively()
 208 {
 209     /* This object may be negatively cached */
 210     negativeCache();
 211     makePublic();
 212 }
 213
 214 size_t
 215 StoreEntry::inUseCount()
 216 {
 217     if (!pool)
 218         return 0;
 219     return pool->getInUseCount();
 220 }
 221
 222 const char *
 223 StoreEntry::getMD5Text() const
 224 {
 225     return storeKeyText((const cache_key *)key);
 226 }
 227
 228 #include "comm.h"
 229
 230 void
 231 StoreEntry::DeferReader(void *theContext, CommRead const &aRead)
 232 {
 233     StoreEntry *anEntry = (StoreEntry *)theContext;
 234     anEntry->delayAwareRead(aRead.conn,
 235                             aRead.buf,
 236                             aRead.len,
 237                             aRead.callback);
 238 }
 239
 240 void
 241 StoreEntry::delayAwareRead(const Comm::ConnectionPointer &conn, char *buf, int len, AsyncCall::Pointer callback)
 242 {
 243     size_t amountToRead = bytesWanted(Range<size_t>(0, len));
 244     /* sketch: readdeferer* = getdeferer.
 245      * ->deferRead (fd, buf, len, callback, DelayAwareRead, this)
 246      */
 247
 248     if (amountToRead == 0) {
 249         assert (mem_obj);
 250         /* read ahead limit */
 251         /* Perhaps these two calls should both live in MemObject */
 252 #if USE_DELAY_POOLS
 253         if (!mem_obj->readAheadPolicyCanRead()) {
 254 #endif
 255             mem_obj->delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 256             return;
 257 #if USE_DELAY_POOLS
 258         }
 259
 260         /* delay id limit */
 261         mem_obj->mostBytesAllowed().delayRead(DeferredRead(DeferReader, this, CommRead(conn, buf, len, callback)));
 262         return;
 263
 264 #endif
 265
 266     }
 267
 268     if (fd_table[conn->fd].closing()) {
 269         // Readers must have closing callbacks if they want to be notified. No
 270         // readers appeared to care around 2009/12/14 as they skipped reading
 271         // for other reasons. Closing may already be true at the delyaAwareRead
 272         // call time or may happen while we wait after delayRead() above.
 273         debugs(20, 3, HERE << "wont read from closing " << conn << " for " <<
 274                callback);
 275         return; // the read callback will never be called
 276     }
 277
 278     comm_read(conn, buf, amountToRead, callback);
 279 }
 280
 281 size_t
 282 StoreEntry::bytesWanted (Range<size_t> const aRange, bool ignoreDelayPools) const
 283 {
 284     if (mem_obj == NULL)
 285         return aRange.end;
 286
 287 #if URL_CHECKSUM_DEBUG
 288
 289     mem_obj->checkUrlChecksum();
 290
 291 #endif
 292
 293     if (!mem_obj->readAheadPolicyCanRead())
 294         return 0;
 295
 296     return mem_obj->mostBytesWanted(aRange.end, ignoreDelayPools);
 297 }
 298
 299 bool
 300 StoreEntry::checkDeferRead(int fd) const
 301 {
 302     return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0);
 303 }
 304
 305 void
 306 StoreEntry::setNoDelay (bool const newValue)
 307 {
 308     if (mem_obj)
 309         mem_obj->setNoDelay(newValue);
 310 }
 311
 312 store_client_t
 313 StoreEntry::storeClientType() const
 314 {
 315     /* The needed offset isn't in memory
 316      * XXX TODO: this is wrong for range requests
 317      * as the needed offset may *not* be 0, AND
 318      * offset 0 in the memory object is the HTTP headers.
 319      */
 320
 321     if (mem_status == IN_MEMORY && Config.memShared && IamWorkerProcess()) {
 322         // clients of an object cached in shared memory are memory clients
 323         return STORE_MEM_CLIENT;
 324     }
 325
 326     assert(mem_obj);
 327
 328     if (mem_obj->inmem_lo)
 329         return STORE_DISK_CLIENT;
 330
 331     if (EBIT_TEST(flags, ENTRY_ABORTED)) {
 332         /* I don't think we should be adding clients to aborted entries */
 333         debugs(20, DBG_IMPORTANT, "storeClientType: adding to ENTRY_ABORTED entry");
 334         return STORE_MEM_CLIENT;
 335     }
 336
 337     if (store_status == STORE_OK) {
 338         /* the object has completed. */
 339
 340         if (mem_obj->inmem_lo == 0 && !isEmpty()) {
 341             if (swap_status == SWAPOUT_DONE) {
 342                 debugs(20,7, HERE << mem_obj << " lo: " << mem_obj->inmem_lo << " hi: " << mem_obj->endOffset() << " size: " << mem_obj->object_sz);
 343                 if (mem_obj->endOffset() == mem_obj->object_sz) {
 344                     /* hot object fully swapped in */
 345                     return STORE_MEM_CLIENT;
 346                 }
 347             } else {
 348                 /* Memory-only, or currently being swapped out */
 349                 return STORE_MEM_CLIENT;
 350             }
 351         }
 352         return STORE_DISK_CLIENT;
 353     }
 354
 355     /* here and past, entry is STORE_PENDING */
 356     /*
 357      * If this is the first client, let it be the mem client
 358      */
 359     if (mem_obj->nclients == 1)
 360         return STORE_MEM_CLIENT;
 361
 362     /*
 363      * If there is no disk file to open yet, we must make this a
 364      * mem client.  If we can't open the swapin file before writing
 365      * to the client, there is no guarantee that we will be able
 366      * to open it later when we really need it.
 367      */
 368     if (swap_status == SWAPOUT_NONE)
 369         return STORE_MEM_CLIENT;
 370
 371     /*
 372      * otherwise, make subsequent clients read from disk so they
 373      * can not delay the first, and vice-versa.
 374      */
 375     return STORE_DISK_CLIENT;
 376 }
 377
 378 StoreEntry::StoreEntry() :
 379         mem_obj(NULL),
 380         timestamp(-1),
 381         lastref(-1),
 382         expires(-1),
 383         lastmod(-1),
 384         swap_file_sz(0),
 385         refcount(0),
 386         flags(0),
 387         swap_filen(-1),
 388         swap_dirn(-1),
 389         mem_status(NOT_IN_MEMORY),
 390         ping_status(PING_NONE),
 391         store_status(STORE_PENDING),
 392         swap_status(SWAPOUT_NONE),
 393         lock_count(0)
 394 {
 395     debugs(20, 3, HERE << "new StoreEntry " << this);
 396 }
 397
 398 StoreEntry::~StoreEntry()
 399 {
 400     if (swap_filen >= 0) {
 401         SwapDir &sd = dynamic_cast<SwapDir&>(*store());
 402         sd.disconnect(*this);
 403     }
 404 }
 405
 406 #if USE_ADAPTATION
 407 void
 408 StoreEntry::deferProducer(const AsyncCall::Pointer &producer)
 409 {
 410     if (!deferredProducer)
 411         deferredProducer = producer;
 412     else
 413         debugs(20, 5, HERE << "Deferred producer call is allready set to: " <<
 414                *deferredProducer << ", requested call: " << *producer);
 415 }
 416
 417 void
 418 StoreEntry::kickProducer()
 419 {
 420     if (deferredProducer != NULL) {
 421         ScheduleCallHere(deferredProducer);
 422         deferredProducer = NULL;
 423     }
 424 }
 425 #endif
 426
 427 void
 428 StoreEntry::destroyMemObject()
 429 {
 430     debugs(20, 3, HERE << "destroyMemObject " << mem_obj);
 431     setMemStatus(NOT_IN_MEMORY);
 432     MemObject *mem = mem_obj;
 433     mem_obj = NULL;
 434     delete mem;
 435 }
 436
 437 void
 438 destroyStoreEntry(void *data)
 439 {
 440     debugs(20, 3, HERE << "destroyStoreEntry: destroying " <<  data);
 441     StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data));
 442     assert(e != NULL);
 443
 444     if (e == NullStoreEntry::getInstance())
 445         return;
 446
 447     e->destroyMemObject();
 448
 449     e->hashDelete();
 450
 451     assert(e->key == NULL);
 452
 453     delete e;
 454 }
 455
 456 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
 457
 458 void
 459 StoreEntry::hashInsert(const cache_key * someKey)
 460 {
 461     debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << *this << " key '" << storeKeyText(someKey) << "'");
 462     key = storeKeyDup(someKey);
 463     hash_join(store_table, this);
 464 }
 465
 466 void
 467 StoreEntry::hashDelete()
 468 {
 469     hash_remove_link(store_table, this);
 470     storeKeyFree((const cache_key *)key);
 471     key = NULL;
 472 }
 473
 474 /* -------------------------------------------------------------------------- */
 475
 476 /* get rid of memory copy of the object */
 477 void
 478 StoreEntry::purgeMem()
 479 {
 480     if (mem_obj == NULL)
 481         return;
 482
 483     debugs(20, 3, "StoreEntry::purgeMem: Freeing memory-copy of " << getMD5Text());
 484
 485     Store::Root().memoryUnlink(*this);
 486
 487     if (swap_status != SWAPOUT_DONE)
 488         release();
 489 }
 490
 491 void
 492 StoreEntry::lock(const char *context)
 493 {
 494     ++lock_count;
 495     debugs(20, 3, context << " locked key " << getMD5Text() << ' ' << *this);
 496 }
 497
 498 void
 499 StoreEntry::touch() {
 500     lastref = squid_curtime;
 501     Store::Root().reference(*this);
 502 }
 503
 504 void
 505 StoreEntry::setReleaseFlag()
 506 {
 507     if (EBIT_TEST(flags, RELEASE_REQUEST))
 508         return;
 509
 510     debugs(20, 3, "StoreEntry::setReleaseFlag: '" << getMD5Text() << "'");
 511
 512     EBIT_SET(flags, RELEASE_REQUEST);
 513
 514     Store::Root().markForUnlink(*this);
 515 }
 516
 517 void
 518 StoreEntry::releaseRequest()
 519 {
 520     if (EBIT_TEST(flags, RELEASE_REQUEST))
 521         return;
 522
 523     setReleaseFlag(); // makes validToSend() false, preventing future hits
 524
 525     setPrivateKey();
 526 }
 527
 528 int
 529 StoreEntry::unlock(const char *context)
 530 {
 531     debugs(20, 3, (context ? context : "somebody") <<
 532            " unlocking key " << getMD5Text() << ' ' << *this);
 533     --lock_count;
 534
 535     if (lock_count)
 536         return (int) lock_count;
 537
 538     if (store_status == STORE_PENDING)
 539         setReleaseFlag();
 540
 541     assert(storePendingNClients(this) == 0);
 542
 543     if (EBIT_TEST(flags, RELEASE_REQUEST)) {
 544         this->release();
 545         return 0;
 546     }
 547
 548     if (EBIT_TEST(flags, KEY_PRIVATE))
 549         debugs(20, DBG_IMPORTANT, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE");
 550
 551     Store::Root().handleIdleEntry(*this); // may delete us
 552     return 0;
 553 }
 554
 555 void
 556 StoreEntry::getPublicByRequestMethod  (StoreClient *aClient, HttpRequest * request, const HttpRequestMethod& method)
 557 {
 558     assert (aClient);
 559     StoreEntry *result = storeGetPublicByRequestMethod( request, method);
 560
 561     if (!result)
 562         aClient->created (NullStoreEntry::getInstance());
 563     else
 564         aClient->created (result);
 565 }
 566
 567 void
 568 StoreEntry::getPublicByRequest (StoreClient *aClient, HttpRequest * request)
 569 {
 570     assert (aClient);
 571     StoreEntry *result = storeGetPublicByRequest (request);
 572
 573     if (!result)
 574         result = NullStoreEntry::getInstance();
 575
 576     aClient->created (result);
 577 }
 578
 579 void
 580 StoreEntry::getPublic (StoreClient *aClient, const char *uri, const HttpRequestMethod& method)
 581 {
 582     assert (aClient);
 583     StoreEntry *result = storeGetPublic (uri, method);
 584
 585     if (!result)
 586         result = NullStoreEntry::getInstance();
 587
 588     aClient->created (result);
 589 }
 590
 591 StoreEntry *
 592 storeGetPublic(const char *uri, const HttpRequestMethod& method)
 593 {
 594     return Store::Root().get(storeKeyPublic(uri, method));
 595 }
 596
 597 StoreEntry *
 598 storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method)
 599 {
 600     return Store::Root().get(storeKeyPublicByRequestMethod(req, method));
 601 }
 602
 603 StoreEntry *
 604 storeGetPublicByRequest(HttpRequest * req)
 605 {
 606     StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
 607
 608     if (e == NULL && req->method == Http::METHOD_HEAD)
 609         /* We can generate a HEAD reply from a cached GET object */
 610         e = storeGetPublicByRequestMethod(req, Http::METHOD_GET);
 611
 612     return e;
 613 }
 614
 615 static int
 616 getKeyCounter(void)
 617 {
 618     static int key_counter = 0;
 619
 620     if (++key_counter < 0)
 621         key_counter = 1;
 622
 623     return key_counter;
 624 }
 625
 626 /* RBC 20050104 AFAICT this should become simpler:
 627  * rather than reinserting with a special key it should be marked
 628  * as 'released' and then cleaned up when refcounting indicates.
 629  * the StoreHashIndex could well implement its 'released' in the
 630  * current manner.
 631  * Also, clean log writing should skip over ia,t
 632  * Otherwise, we need a 'remove from the index but not the store
 633  * concept'.
 634  */
 635 void
 636 StoreEntry::setPrivateKey()
 637 {
 638     const cache_key *newkey;
 639
 640     if (key && EBIT_TEST(flags, KEY_PRIVATE))
 641         return;                 /* is already private */
 642
 643     if (key) {
 644         setReleaseFlag(); // will markForUnlink(); all caches/workers will know
 645
 646         // TODO: move into SwapDir::markForUnlink() already called by Root()
 647         if (swap_filen > -1)
 648             storeDirSwapLog(this, SWAP_LOG_DEL);
 649
 650         hashDelete();
 651     }
 652
 653     if (mem_obj && mem_obj->hasUris()) {
 654         mem_obj->id = getKeyCounter();
 655         newkey = storeKeyPrivate(mem_obj->storeId(), mem_obj->method, mem_obj->id);
 656     } else {
 657         newkey = storeKeyPrivate("JUNK", Http::METHOD_NONE, getKeyCounter());
 658     }
 659
 660     assert(hash_lookup(store_table, newkey) == NULL);
 661     EBIT_SET(flags, KEY_PRIVATE);
 662     hashInsert(newkey);
 663 }
 664
 665 void
 666 StoreEntry::setPublicKey()
 667 {
 668     const cache_key *newkey;
 669
 670     if (key && !EBIT_TEST(flags, KEY_PRIVATE))
 671         return;                 /* is already public */
 672
 673     assert(mem_obj);
 674
 675     /*
 676      * We can't make RELEASE_REQUEST objects public.  Depending on
 677      * when RELEASE_REQUEST gets set, we might not be swapping out
 678      * the object.  If we're not swapping out, then subsequent
 679      * store clients won't be able to access object data which has
 680      * been freed from memory.
 681      *
 682      * If RELEASE_REQUEST is set, setPublicKey() should not be called.
 683      */
 684 #if MORE_DEBUG_OUTPUT
 685
 686     if (EBIT_TEST(flags, RELEASE_REQUEST))
 687         debugs(20, DBG_IMPORTANT, "assertion failed: RELEASE key " << key << ", url " << mem_obj->url);
 688
 689 #endif
 690
 691     assert(!EBIT_TEST(flags, RELEASE_REQUEST));
 692
 693     if (mem_obj->request) {
 694         HttpRequest *request = mem_obj->request;
 695
 696         if (!mem_obj->vary_headers) {
 697             /* First handle the case where the object no longer varies */
 698             safe_free(request->vary_headers);
 699         } else {
 700             if (request->vary_headers && strcmp(request->vary_headers, mem_obj->vary_headers) != 0) {
 701                 /* Oops.. the variance has changed. Kill the base object
 702                  * to record the new variance key
 703                  */
 704                 safe_free(request->vary_headers);       /* free old "bad" variance key */
 705                 if (StoreEntry *pe = storeGetPublic(mem_obj->storeId(), mem_obj->method))
 706                     pe->release();
 707             }
 708
 709             /* Make sure the request knows the variance status */
 710             if (!request->vary_headers) {
 711                 const char *vary = httpMakeVaryMark(request, mem_obj->getReply());
 712
 713                 if (vary)
 714                     request->vary_headers = xstrdup(vary);
 715             }
 716         }
 717
 718         // TODO: storeGetPublic() calls below may create unlocked entries.
 719         // We should add/use storeHas() API or lock/unlock those entries.
 720         if (mem_obj->vary_headers && !storeGetPublic(mem_obj->storeId(), mem_obj->method)) {
 721             /* Create "vary" base object */
 722             String vary;
 723             StoreEntry *pe = storeCreateEntry(mem_obj->storeId(), mem_obj->logUri(), request->flags, request->method);
 724             /* We are allowed to do this typecast */
 725             HttpReply *rep = new HttpReply;
 726             rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
 727             vary = mem_obj->getReply()->header.getList(HDR_VARY);
 728
 729             if (vary.size()) {
 730                 /* Again, we own this structure layout */
 731                 rep->header.putStr(HDR_VARY, vary.termedBuf());
 732                 vary.clean();
 733             }
 734
 735 #if X_ACCELERATOR_VARY
 736             vary = mem_obj->getReply()->header.getList(HDR_X_ACCELERATOR_VARY);
 737
 738             if (vary.defined()) {
 739                 /* Again, we own this structure layout */
 740                 rep->header.putStr(HDR_X_ACCELERATOR_VARY, vary.termedBuf());
 741                 vary.clean();
 742             }
 743
 744 #endif
 745             pe->replaceHttpReply(rep);
 746
 747             pe->timestampsSet();
 748
 749             pe->makePublic();
 750
 751             pe->complete();
 752
 753             pe->unlock("StoreEntry::setPublicKey+Vary");
 754         }
 755
 756         newkey = storeKeyPublicByRequest(mem_obj->request);
 757     } else
 758         newkey = storeKeyPublic(mem_obj->storeId(), mem_obj->method);
 759
 760     if (StoreEntry *e2 = (StoreEntry *)hash_lookup(store_table, newkey)) {
 761         debugs(20, 3, "Making old " << *e2 << " private.");
 762         e2->setPrivateKey();
 763         e2->release();
 764
 765         if (mem_obj->request)
 766             newkey = storeKeyPublicByRequest(mem_obj->request);
 767         else
 768             newkey = storeKeyPublic(mem_obj->storeId(), mem_obj->method);
 769     }
 770
 771     if (key)
 772         hashDelete();
 773
 774     EBIT_CLR(flags, KEY_PRIVATE);
 775
 776     hashInsert(newkey);
 777
 778     if (swap_filen > -1)
 779         storeDirSwapLog(this, SWAP_LOG_ADD);
 780 }
 781
 782 StoreEntry *
 783 storeCreatePureEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method)
 784 {
 785     StoreEntry *e = NULL;
 786     debugs(20, 3, "storeCreateEntry: '" << url << "'");
 787
 788     e = new StoreEntry();
 789     e->makeMemObject();
 790     e->mem_obj->setUris(url, log_url, method);
 791
 792     if (flags.cachable) {
 793         EBIT_CLR(e->flags, RELEASE_REQUEST);
 794     } else {
 795         e->releaseRequest();
 796     }
 797
 798     e->store_status = STORE_PENDING;
 799     e->refcount = 0;
 800     e->lastref = squid_curtime;
 801     e->timestamp = -1;          /* set in StoreEntry::timestampsSet() */
 802     e->ping_status = PING_NONE;
 803     EBIT_SET(e->flags, ENTRY_VALIDATED);
 804     return e;
 805 }
 806
 807 StoreEntry *
 808 storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod& method)
 809 {
 810     StoreEntry *e = storeCreatePureEntry(url, logUrl, flags, method);
 811     e->lock("storeCreateEntry");
 812
 813     if (neighbors_do_private_keys || !flags.hierarchical)
 814         e->setPrivateKey();
 815     else
 816         e->setPublicKey();
 817
 818     return e;
 819 }
 820
 821 /* Mark object as expired */
 822 void
 823 StoreEntry::expireNow()
 824 {
 825     debugs(20, 3, "StoreEntry::expireNow: '" << getMD5Text() << "'");
 826     expires = squid_curtime;
 827 }
 828
 829 void
 830 StoreEntry::write (StoreIOBuffer writeBuffer)
 831 {
 832     assert(mem_obj != NULL);
 833     /* This assert will change when we teach the store to update */
 834     PROF_start(StoreEntry_write);
 835     assert(store_status == STORE_PENDING);
 836
 837     // XXX: caller uses content offset, but we also store headers
 838     if (const HttpReply *reply = mem_obj->getReply())
 839         writeBuffer.offset += reply->hdr_sz;
 840
 841     debugs(20, 5, "storeWrite: writing " << writeBuffer.length << " bytes for '" << getMD5Text() << "'");
 842     PROF_stop(StoreEntry_write);
 843     storeGetMemSpace(writeBuffer.length);
 844     mem_obj->write(writeBuffer);
 845
 846     if (!EBIT_TEST(flags, DELAY_SENDING))
 847         invokeHandlers();
 848 }
 849
 850 /* Append incoming data from a primary server to an entry. */
 851 void
 852 StoreEntry::append(char const *buf, int len)
 853 {
 854     assert(mem_obj != NULL);
 855     assert(len >= 0);
 856     assert(store_status == STORE_PENDING);
 857
 858     StoreIOBuffer tempBuffer;
 859     tempBuffer.data = (char *)buf;
 860     tempBuffer.length = len;
 861     /*
 862      * XXX sigh, offset might be < 0 here, but it gets "corrected"
 863      * later.  This offset crap is such a mess.
 864      */
 865     tempBuffer.offset = mem_obj->endOffset() - (getReply() ? getReply()->hdr_sz : 0);
 866     write(tempBuffer);
 867 }
 868
 869 void
 870 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
 871 {
 872     va_list args;
 873     va_start(args, fmt);
 874
 875     storeAppendVPrintf(e, fmt, args);
 876     va_end(args);
 877 }
 878
 879 /* used be storeAppendPrintf and Packer */
 880 void
 881 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
 882 {
 883     LOCAL_ARRAY(char, buf, 4096);
 884     buf[0] = '\0';
 885     vsnprintf(buf, 4096, fmt, vargs);
 886     e->append(buf, strlen(buf));
 887 }
 888
 889 struct _store_check_cachable_hist {
 890
 891     struct {
 892         int non_get;
 893         int not_entry_cachable;
 894         int wrong_content_length;
 895         int negative_cached;
 896         int too_big;
 897         int too_small;
 898         int private_key;
 899         int too_many_open_files;
 900         int too_many_open_fds;
 901     } no;
 902
 903     struct {
 904         int Default;
 905     } yes;
 906 } store_check_cachable_hist;
 907
 908 int
 909 storeTooManyDiskFilesOpen(void)
 910 {
 911     if (Config.max_open_disk_fds == 0)
 912         return 0;
 913
 914     if (store_open_disk_fd > Config.max_open_disk_fds)
 915         return 1;
 916
 917     return 0;
 918 }
 919
 920 int
 921 StoreEntry::checkTooSmall()
 922 {
 923     if (EBIT_TEST(flags, ENTRY_SPECIAL))
 924         return 0;
 925
 926     if (STORE_OK == store_status)
 927         if (mem_obj->object_sz < 0 ||
 928                 mem_obj->object_sz < Config.Store.minObjectSize)
 929             return 1;
 930     if (getReply()->content_length > -1)
 931         if (getReply()->content_length < Config.Store.minObjectSize)
 932             return 1;
 933     return 0;
 934 }
 935
 936 // TODO: remove checks already performed by swapoutPossible()
 937 // TODO: move "too many open..." checks outside -- we are called too early/late
 938 int
 939 StoreEntry::checkCachable()
 940 {
 941 #if CACHE_ALL_METHODS
 942
 943     if (mem_obj->method != Http::METHOD_GET) {
 944         debugs(20, 2, "StoreEntry::checkCachable: NO: non-GET method");
 945         ++store_check_cachable_hist.no.non_get;
 946     } else
 947 #endif
 948         if (store_status == STORE_OK && EBIT_TEST(flags, ENTRY_BAD_LENGTH)) {
 949             debugs(20, 2, "StoreEntry::checkCachable: NO: wrong content-length");
 950             ++store_check_cachable_hist.no.wrong_content_length;
 951         } else if (EBIT_TEST(flags, RELEASE_REQUEST)) {
 952             debugs(20, 2, "StoreEntry::checkCachable: NO: not cachable");
 953             ++store_check_cachable_hist.no.not_entry_cachable; // TODO: rename?
 954         } else if (EBIT_TEST(flags, ENTRY_NEGCACHED)) {
 955             debugs(20, 3, "StoreEntry::checkCachable: NO: negative cached");
 956             ++store_check_cachable_hist.no.negative_cached;
 957             return 0;           /* avoid release call below */
 958         } else if ((getReply()->content_length > 0 &&
 959                     getReply()->content_length > store_maxobjsize) ||
 960                    mem_obj->endOffset() > store_maxobjsize) {
 961             debugs(20, 2, "StoreEntry::checkCachable: NO: too big");
 962             ++store_check_cachable_hist.no.too_big;
 963         } else if (checkTooSmall()) {
 964             debugs(20, 2, "StoreEntry::checkCachable: NO: too small");
 965             ++store_check_cachable_hist.no.too_small;
 966         } else if (EBIT_TEST(flags, KEY_PRIVATE)) {
 967             debugs(20, 3, "StoreEntry::checkCachable: NO: private key");
 968             ++store_check_cachable_hist.no.private_key;
 969         } else if (swap_status != SWAPOUT_NONE) {
 970             /*
 971              * here we checked the swap_status because the remaining
 972              * cases are only relevant only if we haven't started swapping
 973              * out the object yet.
 974              */
 975             return 1;
 976         } else if (storeTooManyDiskFilesOpen()) {
 977             debugs(20, 2, "StoreEntry::checkCachable: NO: too many disk files open");
 978             ++store_check_cachable_hist.no.too_many_open_files;
 979         } else if (fdNFree() < RESERVED_FD) {
 980             debugs(20, 2, "StoreEntry::checkCachable: NO: too many FD's open");
 981             ++store_check_cachable_hist.no.too_many_open_fds;
 982         } else {
 983             ++store_check_cachable_hist.yes.Default;
 984             return 1;
 985         }
 986
 987     releaseRequest();
 988     return 0;
 989 }
 990
 991 void
 992 storeCheckCachableStats(StoreEntry *sentry)
 993 {
 994     storeAppendPrintf(sentry, "Category\t Count\n");
 995
 996 #if CACHE_ALL_METHODS
 997
 998     storeAppendPrintf(sentry, "no.non_get\t%d\n",
 999                       store_check_cachable_hist.no.non_get);
1000 #endif
1001
1002     storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
1003                       store_check_cachable_hist.no.not_entry_cachable);
1004     storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
1005                       store_check_cachable_hist.no.wrong_content_length);
1006     storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
1007                       store_check_cachable_hist.no.negative_cached);
1008     storeAppendPrintf(sentry, "no.too_big\t%d\n",
1009                       store_check_cachable_hist.no.too_big);
1010     storeAppendPrintf(sentry, "no.too_small\t%d\n",
1011                       store_check_cachable_hist.no.too_small);
1012     storeAppendPrintf(sentry, "no.private_key\t%d\n",
1013                       store_check_cachable_hist.no.private_key);
1014     storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
1015                       store_check_cachable_hist.no.too_many_open_files);
1016     storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
1017                       store_check_cachable_hist.no.too_many_open_fds);
1018     storeAppendPrintf(sentry, "yes.default\t%d\n",
1019                       store_check_cachable_hist.yes.Default);
1020 }
1021
1022 void
1023 StoreEntry::complete()
1024 {
1025     debugs(20, 3, "storeComplete: '" << getMD5Text() << "'");
1026
1027     if (store_status != STORE_PENDING) {
1028         /*
1029          * if we're not STORE_PENDING, then probably we got aborted
1030          * and there should be NO clients on this entry
1031          */
1032         assert(EBIT_TEST(flags, ENTRY_ABORTED));
1033         assert(mem_obj->nclients == 0);
1034         return;
1035     }
1036
1037     /* This is suspect: mem obj offsets include the headers. do we adjust for that
1038      * in use of object_sz?
1039      */
1040     mem_obj->object_sz = mem_obj->endOffset();
1041
1042     store_status = STORE_OK;
1043
1044     assert(mem_status == NOT_IN_MEMORY);
1045
1046     if (!validLength()) {
1047         EBIT_SET(flags, ENTRY_BAD_LENGTH);
1048         releaseRequest();
1049     }
1050
1051 #if USE_CACHE_DIGESTS
1052     if (mem_obj->request)
1053         mem_obj->request->hier.store_complete_stop = current_time;
1054
1055 #endif
1056     /*
1057      * We used to call invokeHandlers, then storeSwapOut.  However,
1058      * Madhukar Reddy <myreddy@persistence.com> reported that
1059      * responses without content length would sometimes get released
1060      * in client_side, thinking that the response is incomplete.
1061      */
1062     invokeHandlers();
1063 }
1064
1065 /*
1066  * Someone wants to abort this transfer.  Set the reason in the
1067  * request structure, call the server-side callback and mark the
1068  * entry for releasing
1069  */
1070 void
1071 StoreEntry::abort()
1072 {
1073     ++statCounter.aborted_requests;
1074     assert(store_status == STORE_PENDING);
1075     assert(mem_obj != NULL);
1076     debugs(20, 6, "storeAbort: " << getMD5Text());
1077
1078     lock("StoreEntry::abort");         /* lock while aborting */
1079     negativeCache();
1080
1081     releaseRequest();
1082
1083     EBIT_SET(flags, ENTRY_ABORTED);
1084
1085     setMemStatus(NOT_IN_MEMORY);
1086
1087     store_status = STORE_OK;
1088
1089     /* Notify the server side */
1090
1091     /*
1092      * DPW 2007-05-07
1093      * Should we check abort.data for validity?
1094      */
1095     if (mem_obj->abort.callback) {
1096         if (!cbdataReferenceValid(mem_obj->abort.data))
1097             debugs(20, DBG_IMPORTANT,HERE << "queueing event when abort.data is not valid");
1098         eventAdd("mem_obj->abort.callback",
1099                  mem_obj->abort.callback,
1100                  mem_obj->abort.data,
1101                  0.0,
1102                  true);
1103         unregisterAbort();
1104     }
1105
1106     /* XXX Should we reverse these two, so that there is no
1107      * unneeded disk swapping triggered?
1108      */
1109     /* Notify the client side */
1110     invokeHandlers();
1111
1112     // abort swap out, invalidating what was created so far (release follows)
1113     swapOutFileClose(StoreIOState::writerGone);
1114
1115     unlock("StoreEntry::abort");       /* unlock */
1116 }
1117
1118 /**
1119  * Clear Memory storage to accommodate the given object len
1120  */
1121 void
1122 storeGetMemSpace(int size)
1123 {
1124     PROF_start(storeGetMemSpace);
1125     StoreEntry *e = NULL;
1126     int released = 0;
1127     static time_t last_check = 0;
1128     size_t pages_needed;
1129     RemovalPurgeWalker *walker;
1130
1131     if (squid_curtime == last_check) {
1132         PROF_stop(storeGetMemSpace);
1133         return;
1134     }
1135
1136     last_check = squid_curtime;
1137
1138     pages_needed = (size + SM_PAGE_SIZE-1) / SM_PAGE_SIZE;
1139
1140     if (mem_node::InUseCount() + pages_needed < store_pages_max) {
1141         PROF_stop(storeGetMemSpace);
1142         return;
1143     }
1144
1145     debugs(20, 2, "storeGetMemSpace: Starting, need " << pages_needed <<
1146            " pages");
1147
1148     /* XXX what to set as max_scan here? */
1149     walker = mem_policy->PurgeInit(mem_policy, 100000);
1150
1151     while ((e = walker->Next(walker))) {
1152         e->purgeMem();
1153         ++released;
1154
1155         if (mem_node::InUseCount() + pages_needed < store_pages_max)
1156             break;
1157     }
1158
1159     walker->Done(walker);
1160     debugs(20, 3, "storeGetMemSpace stats:");
1161     debugs(20, 3, "  " << std::setw(6) << hot_obj_count  << " HOT objects");
1162     debugs(20, 3, "  " << std::setw(6) << released  << " were released");
1163     PROF_stop(storeGetMemSpace);
1164 }
1165
1166 /* thunk through to Store::Root().maintain(). Note that this would be better still
1167  * if registered against the root store itself, but that requires more complex
1168  * update logic - bigger fish to fry first. Long term each store when
1169  * it becomes active will self register
1170  */
1171 void
1172 Store::Maintain(void *notused)
1173 {
1174     Store::Root().maintain();
1175
1176     /* Reregister a maintain event .. */
1177     eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1);
1178
1179 }
1180
1181 /* The maximum objects to scan for maintain storage space */
1182 #define MAINTAIN_MAX_SCAN       1024
1183 #define MAINTAIN_MAX_REMOVE     64
1184
1185 /*
1186  * This routine is to be called by main loop in main.c.
1187  * It removes expired objects on only one bucket for each time called.
1188  *
1189  * This should get called 1/s from main().
1190  */
1191 void
1192 StoreController::maintain()
1193 {
1194     static time_t last_warn_time = 0;
1195
1196     PROF_start(storeMaintainSwapSpace);
1197     swapDir->maintain();
1198
1199     /* this should be emitted by the oversize dir, not globally */
1200
1201     if (Store::Root().currentSize() > Store::Root().maxSize()) {
1202         if (squid_curtime - last_warn_time > 10) {
1203             debugs(20, DBG_CRITICAL, "WARNING: Disk space over limit: "
1204                    << Store::Root().currentSize() / 1024.0 << " KB > "
1205                    << (Store::Root().maxSize() >> 10) << " KB");
1206             last_warn_time = squid_curtime;
1207         }
1208     }
1209
1210     PROF_stop(storeMaintainSwapSpace);
1211 }
1212
1213 /* release an object from a cache */
1214 void
1215 StoreEntry::release()
1216 {
1217     PROF_start(storeRelease);
1218     debugs(20, 3, "releasing " << *this << ' ' << getMD5Text());
1219     /* If, for any reason we can't discard this object because of an
1220      * outstanding request, mark it for pending release */
1221
1222     if (locked()) {
1223         expireNow();
1224         debugs(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit");
1225         releaseRequest();
1226         PROF_stop(storeRelease);
1227         return;
1228     }
1229
1230     Store::Root().memoryUnlink(*this);
1231
1232     if (StoreController::store_dirs_rebuilding && swap_filen > -1) {
1233         setPrivateKey();
1234
1235         if (swap_filen > -1) {
1236             /*
1237              * Fake a call to StoreEntry->lock()  When rebuilding is done,
1238              * we'll just call StoreEntry->unlock() on these.
1239              */
1240             lock("StoreEntry::release+rebuilding");
1241             setReleaseFlag();
1242             LateReleaseStack.push_back(this);
1243         } else {
1244             destroyStoreEntry(static_cast<hash_link *>(this));
1245             // "this" is no longer valid
1246         }
1247
1248         PROF_stop(storeRelease);
1249         return;
1250     }
1251
1252     storeLog(STORE_LOG_RELEASE, this);
1253
1254     if (swap_filen > -1) {
1255         // log before unlink() below clears swap_filen
1256         if (!EBIT_TEST(flags, KEY_PRIVATE))
1257             storeDirSwapLog(this, SWAP_LOG_DEL);
1258
1259         unlink();
1260     }
1261
1262     destroyStoreEntry(static_cast<hash_link *>(this));
1263     PROF_stop(storeRelease);
1264 }
1265
1266 static void
1267 storeLateRelease(void *unused)
1268 {
1269     StoreEntry *e;
1270     int i;
1271     static int n = 0;
1272
1273     if (StoreController::store_dirs_rebuilding) {
1274         eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1275         return;
1276     }
1277
1278     for (i = 0; i < 10; ++i) {
1279         e = LateReleaseStack.count ? LateReleaseStack.pop() : NULL;
1280
1281         if (e == NULL) {
1282             /* done! */
1283             debugs(20, DBG_IMPORTANT, "storeLateRelease: released " << n << " objects");
1284             return;
1285         }
1286
1287         e->unlock("storeLateRelease");
1288         ++n;
1289     }
1290
1291     eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
1292 }
1293
1294 /* return 1 if a store entry is locked */
1295 int
1296 StoreEntry::locked() const
1297 {
1298     if (lock_count)
1299         return 1;
1300
1301     /*
1302      * SPECIAL, PUBLIC entries should be "locked";
1303      * XXX: Their owner should lock them then instead of relying on this hack.
1304      */
1305     if (EBIT_TEST(flags, ENTRY_SPECIAL))
1306         if (!EBIT_TEST(flags, KEY_PRIVATE))
1307             return 1;
1308
1309     return 0;
1310 }
1311
1312 bool
1313 StoreEntry::validLength() const
1314 {
1315     int64_t diff;
1316     const HttpReply *reply;
1317     assert(mem_obj != NULL);
1318     reply = getReply();
1319     debugs(20, 3, "storeEntryValidLength: Checking '" << getMD5Text() << "'");
1320     debugs(20, 5, "storeEntryValidLength:     object_len = " <<
1321            objectLen());
1322     debugs(20, 5, "storeEntryValidLength:         hdr_sz = " << reply->hdr_sz);
1323     debugs(20, 5, "storeEntryValidLength: content_length = " << reply->content_length);
1324
1325     if (reply->content_length < 0) {
1326         debugs(20, 5, "storeEntryValidLength: Unspecified content length: " << getMD5Text());
1327         return 1;
1328     }
1329
1330     if (reply->hdr_sz == 0) {
1331         debugs(20, 5, "storeEntryValidLength: Zero header size: " << getMD5Text());
1332         return 1;
1333     }
1334
1335     if (mem_obj->method == Http::METHOD_HEAD) {
1336         debugs(20, 5, "storeEntryValidLength: HEAD request: " << getMD5Text());
1337         return 1;
1338     }
1339
1340     if (reply->sline.status() == Http::scNotModified)
1341         return 1;
1342
1343     if (reply->sline.status() == Http::scNoContent)
1344         return 1;
1345
1346     diff = reply->hdr_sz + reply->content_length - objectLen();
1347
1348     if (diff == 0)
1349         return 1;
1350
1351     debugs(20, 3, "storeEntryValidLength: " << (diff < 0 ? -diff : diff)  << " bytes too " << (diff < 0 ? "big" : "small") <<"; '" << getMD5Text() << "'" );
1352
1353     return 0;
1354 }
1355
1356 static void
1357 storeRegisterWithCacheManager(void)
1358 {
1359     Mgr::RegisterAction("storedir", "Store Directory Stats", Store::Stats, 0, 1);
1360     Mgr::RegisterAction("store_io", "Store IO Interface Stats", &Mgr::StoreIoAction::Create, 0, 1);
1361     Mgr::RegisterAction("store_check_cachable_stats", "storeCheckCachable() Stats",
1362                         storeCheckCachableStats, 0, 1);
1363 }
1364
1365 void
1366 storeInit(void)
1367 {
1368     storeKeyInit();
1369     mem_policy = createRemovalPolicy(Config.memPolicy);
1370     storeDigestInit();
1371     storeLogOpen();
1372     eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1373     Store::Root().init();
1374     storeRebuildStart();
1375
1376     storeRegisterWithCacheManager();
1377 }
1378
1379 void
1380 storeConfigure(void)
1381 {
1382     store_swap_high = (long) (((float) Store::Root().maxSize() *
1383                                (float) Config.Swap.highWaterMark) / (float) 100);
1384     store_swap_low = (long) (((float) Store::Root().maxSize() *
1385                               (float) Config.Swap.lowWaterMark) / (float) 100);
1386     store_pages_max = Config.memMaxSize / sizeof(mem_node);
1387 }
1388
1389 bool
1390 StoreEntry::memoryCachable() const
1391 {
1392     if (mem_obj == NULL)
1393         return 0;
1394
1395     if (mem_obj->data_hdr.size() == 0)
1396         return 0;
1397
1398     if (mem_obj->inmem_lo != 0)
1399         return 0;
1400
1401     if (!Config.onoff.memory_cache_first && swap_status == SWAPOUT_DONE && refcount == 1)
1402         return 0;
1403
1404     return 1;
1405 }
1406
1407 int
1408 StoreEntry::checkNegativeHit() const
1409 {
1410     if (!EBIT_TEST(flags, ENTRY_NEGCACHED))
1411         return 0;
1412
1413     if (expires <= squid_curtime)
1414         return 0;
1415
1416     if (store_status != STORE_OK)
1417         return 0;
1418
1419     return 1;
1420 }
1421
1422 /**
1423  * Set object for negative caching.
1424  * Preserves any expiry information given by the server.
1425  * In absence of proper expiry info it will set to expire immediately,
1426  * or with HTTP-violations enabled the configured negative-TTL is observed
1427  */
1428 void
1429 StoreEntry::negativeCache()
1430 {
1431     // XXX: should make the default for expires 0 instead of -1
1432     //      so we can distinguish "Expires: -1" from nothing.
1433     if (expires <= 0)
1434 #if USE_HTTP_VIOLATIONS
1435         expires = squid_curtime + Config.negativeTtl;
1436 #else
1437         expires = squid_curtime;
1438 #endif
1439     EBIT_SET(flags, ENTRY_NEGCACHED);
1440 }
1441
1442 void
1443 storeFreeMemory(void)
1444 {
1445     Store::Root(NULL);
1446 #if USE_CACHE_DIGESTS
1447
1448     if (store_digest)
1449         cacheDigestDestroy(store_digest);
1450
1451 #endif
1452
1453     store_digest = NULL;
1454 }
1455
1456 int
1457 expiresMoreThan(time_t expires, time_t when)
1458 {
1459     if (expires < 0)            /* No Expires given */
1460         return 1;
1461
1462     return (expires > (squid_curtime + when));
1463 }
1464
1465 int
1466 StoreEntry::validToSend() const
1467 {
1468     if (EBIT_TEST(flags, RELEASE_REQUEST))
1469         return 0;
1470
1471     if (EBIT_TEST(flags, ENTRY_NEGCACHED))
1472         if (expires <= squid_curtime)
1473             return 0;
1474
1475     if (EBIT_TEST(flags, ENTRY_ABORTED))
1476         return 0;
1477
1478     return 1;
1479 }
1480
1481 void
1482 StoreEntry::timestampsSet()
1483 {
1484     const HttpReply *reply = getReply();
1485     time_t served_date = reply->date;
1486     int age = reply->header.getInt(HDR_AGE);
1487     /* Compute the timestamp, mimicking RFC2616 section 13.2.3. */
1488     /* make sure that 0 <= served_date <= squid_curtime */
1489
1490     if (served_date < 0 || served_date > squid_curtime)
1491         served_date = squid_curtime;
1492
1493     /* Bug 1791:
1494      * If the returned Date: is more than 24 hours older than
1495      * the squid_curtime, then one of us needs to use NTP to set our
1496      * clock.  We'll pretend that our clock is right.
1497      */
1498     else if (served_date < (squid_curtime - 24 * 60 * 60) )
1499         served_date = squid_curtime;
1500
1501     /*
1502      * Compensate with Age header if origin server clock is ahead
1503      * of us and there is a cache in between us and the origin
1504      * server.  But DONT compensate if the age value is larger than
1505      * squid_curtime because it results in a negative served_date.
1506      */
1507     if (age > squid_curtime - served_date)
1508         if (squid_curtime > age)
1509             served_date = squid_curtime - age;
1510
1511     // compensate for Squid-to-server and server-to-Squid delays
1512     if (mem_obj && mem_obj->request) {
1513         const time_t request_sent =
1514             mem_obj->request->hier.peer_http_request_sent.tv_sec;
1515         if (0 < request_sent && request_sent < squid_curtime)
1516             served_date -= (squid_curtime - request_sent);
1517     }
1518
1519     if (reply->expires > 0 && reply->date > -1)
1520         expires = served_date + (reply->expires - reply->date);
1521     else
1522         expires = reply->expires;
1523
1524     lastmod = reply->last_modified;
1525
1526     timestamp = served_date;
1527 }
1528
1529 void
1530 StoreEntry::registerAbort(STABH * cb, void *data)
1531 {
1532     assert(mem_obj);
1533     assert(mem_obj->abort.callback == NULL);
1534     mem_obj->abort.callback = cb;
1535     mem_obj->abort.data = cbdataReference(data);
1536 }
1537
1538 void
1539 StoreEntry::unregisterAbort()
1540 {
1541     assert(mem_obj);
1542     if (mem_obj->abort.callback) {
1543         mem_obj->abort.callback = NULL;
1544         cbdataReferenceDone(mem_obj->abort.data);
1545     }
1546 }
1547
1548 void
1549 StoreEntry::dump(int l) const
1550 {
1551     debugs(20, l, "StoreEntry->key: " << getMD5Text());
1552     debugs(20, l, "StoreEntry->next: " << next);
1553     debugs(20, l, "StoreEntry->mem_obj: " << mem_obj);
1554     debugs(20, l, "StoreEntry->timestamp: " << timestamp);
1555     debugs(20, l, "StoreEntry->lastref: " << lastref);
1556     debugs(20, l, "StoreEntry->expires: " << expires);
1557     debugs(20, l, "StoreEntry->lastmod: " << lastmod);
1558     debugs(20, l, "StoreEntry->swap_file_sz: " << swap_file_sz);
1559     debugs(20, l, "StoreEntry->refcount: " << refcount);
1560     debugs(20, l, "StoreEntry->flags: " << storeEntryFlags(this));
1561     debugs(20, l, "StoreEntry->swap_dirn: " << swap_dirn);
1562     debugs(20, l, "StoreEntry->swap_filen: " << swap_filen);
1563     debugs(20, l, "StoreEntry->lock_count: " << lock_count);
1564     debugs(20, l, "StoreEntry->mem_status: " << mem_status);
1565     debugs(20, l, "StoreEntry->ping_status: " << ping_status);
1566     debugs(20, l, "StoreEntry->store_status: " << store_status);
1567     debugs(20, l, "StoreEntry->swap_status: " << swap_status);
1568 }
1569
1570 /*
1571  * NOTE, this function assumes only two mem states
1572  */
1573 void
1574 StoreEntry::setMemStatus(mem_status_t new_status)
1575 {
1576     if (new_status == mem_status)
1577         return;
1578
1579     // are we using a shared memory cache?
1580     if (Config.memShared && IamWorkerProcess()) {
1581         // This method was designed to update replacement policy, not to
1582         // actually purge something from the memory cache (TODO: rename?).
1583         // Shared memory cache does not have a policy that needs updates.
1584         mem_status = new_status;
1585         return;
1586     }
1587
1588     assert(mem_obj != NULL);
1589
1590     if (new_status == IN_MEMORY) {
1591         assert(mem_obj->inmem_lo == 0);
1592
1593         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1594             debugs(20, 4, "not inserting special " << *this << " into policy");
1595         } else {
1596             mem_policy->Add(mem_policy, this, &mem_obj->repl);
1597             debugs(20, 4, "inserted " << *this << " key: " << getMD5Text());
1598         }
1599
1600         ++hot_obj_count; // TODO: maintain for the shared hot cache as well
1601     } else {
1602         if (EBIT_TEST(flags, ENTRY_SPECIAL)) {
1603             debugs(20, 4, "not removing special " << *this << " from policy");
1604         } else {
1605             mem_policy->Remove(mem_policy, this, &mem_obj->repl);
1606             debugs(20, 4, "removed " << *this);
1607         }
1608
1609         --hot_obj_count;
1610     }
1611
1612     mem_status = new_status;
1613 }
1614
1615 const char *
1616 StoreEntry::url() const
1617 {
1618     if (this == NULL)
1619         return "[null_entry]";
1620     else if (mem_obj == NULL)
1621         return "[null_mem_obj]";
1622     else
1623         return mem_obj->storeId();
1624 }
1625
1626 MemObject *
1627 StoreEntry::makeMemObject()
1628 {
1629     if (!mem_obj)
1630         mem_obj = new MemObject();
1631     return mem_obj;
1632 }
1633
1634 void
1635 StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl, const HttpRequestMethod &aMethod)
1636 {
1637     makeMemObject();
1638     mem_obj->setUris(aUrl, aLogUrl, aMethod);
1639 }
1640
1641 /* this just sets DELAY_SENDING */
1642 void
1643 StoreEntry::buffer()
1644 {
1645     EBIT_SET(flags, DELAY_SENDING);
1646 }
1647
1648 /* this just clears DELAY_SENDING and Invokes the handlers */
1649 void
1650 StoreEntry::flush()
1651 {
1652     if (EBIT_TEST(flags, DELAY_SENDING)) {
1653         EBIT_CLR(flags, DELAY_SENDING);
1654         invokeHandlers();
1655     }
1656 }
1657
1658 int64_t
1659 StoreEntry::objectLen() const
1660 {
1661     assert(mem_obj != NULL);
1662     return mem_obj->object_sz;
1663 }
1664
1665 int64_t
1666 StoreEntry::contentLen() const
1667 {
1668     assert(mem_obj != NULL);
1669     assert(getReply() != NULL);
1670     return objectLen() - getReply()->hdr_sz;
1671 }
1672
1673 HttpReply const *
1674 StoreEntry::getReply () const
1675 {
1676     if (NULL == mem_obj)
1677         return NULL;
1678
1679     return mem_obj->getReply();
1680 }
1681
1682 void
1683 StoreEntry::reset()
1684 {
1685     assert (mem_obj);
1686     debugs(20, 3, "StoreEntry::reset: " << url());
1687     mem_obj->reset();
1688     HttpReply *rep = (HttpReply *) getReply();       // bypass const
1689     rep->reset();
1690     expires = lastmod = timestamp = -1;
1691 }
1692
1693 /*
1694  * storeFsInit
1695  *
1696  * This routine calls the SETUP routine for each fs type.
1697  * I don't know where the best place for this is, and I'm not going to shuffle
1698  * around large chunks of code right now (that can be done once its working.)
1699  */
1700 void
1701 storeFsInit(void)
1702 {
1703     storeReplSetup();
1704 }
1705
1706 /*
1707  * called to add another store removal policy module
1708  */
1709 void
1710 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1711 {
1712     int i;
1713
1714     /* find the number of currently known repl types */
1715     for (i = 0; storerepl_list && storerepl_list[i].typestr; ++i) {
1716         if (strcmp(storerepl_list[i].typestr, type) == 0) {
1717             debugs(20, DBG_IMPORTANT, "WARNING: Trying to load store replacement policy " << type << " twice.");
1718             return;
1719         }
1720     }
1721
1722     /* add the new type */
1723     storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)));
1724
1725     memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1726
1727     storerepl_list[i].typestr = type;
1728
1729     storerepl_list[i].create = create;
1730 }
1731
1732 /*
1733  * Create a removal policy instance
1734  */
1735 RemovalPolicy *
1736 createRemovalPolicy(RemovalPolicySettings * settings)
1737 {
1738     storerepl_entry_t *r;
1739
1740     for (r = storerepl_list; r && r->typestr; ++r) {
1741         if (strcmp(r->typestr, settings->type) == 0)
1742             return r->create(settings->args);
1743     }
1744
1745     debugs(20, DBG_IMPORTANT, "ERROR: Unknown policy " << settings->type);
1746     debugs(20, DBG_IMPORTANT, "ERROR: Be sure to have set cache_replacement_policy");
1747     debugs(20, DBG_IMPORTANT, "ERROR:   and memory_replacement_policy in squid.conf!");
1748     fatalf("ERROR: Unknown policy %s\n", settings->type);
1749     return NULL;                /* NOTREACHED */
1750 }
1751
1752 #if 0
1753 void
1754 storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
1755 {
1756     if (e->swap_file_number == filn)
1757         return;
1758
1759     if (filn < 0) {
1760         assert(-1 == filn);
1761         storeDirMapBitReset(e->swap_file_number);
1762         storeDirLRUDelete(e);
1763         e->swap_file_number = -1;
1764     } else {
1765         assert(-1 == e->swap_file_number);
1766         storeDirMapBitSet(e->swap_file_number = filn);
1767         storeDirLRUAdd(e);
1768     }
1769 }
1770
1771 #endif
1772
1773 /*
1774  * Replace a store entry with
1775  * a new reply. This eats the reply.
1776  */
1777 void
1778 StoreEntry::replaceHttpReply(HttpReply *rep, bool andStartWriting)
1779 {
1780     debugs(20, 3, "StoreEntry::replaceHttpReply: " << url());
1781
1782     if (!mem_obj) {
1783         debugs(20, DBG_CRITICAL, "Attempt to replace object with no in-memory representation");
1784         return;
1785     }
1786
1787     mem_obj->replaceHttpReply(rep);
1788
1789     if (andStartWriting)
1790         startWriting();
1791 }
1792
1793 void
1794 StoreEntry::startWriting()
1795 {
1796     Packer p;
1797
1798     /* TODO: when we store headers serparately remove the header portion */
1799     /* TODO: mark the length of the headers ? */
1800     /* We ONLY want the headers */
1801     packerToStoreInit(&p, this);
1802
1803     assert (isEmpty());
1804     assert(mem_obj);
1805
1806     const HttpReply *rep = getReply();
1807     assert(rep);
1808
1809     rep->packHeadersInto(&p);
1810     mem_obj->markEndOfReplyHeaders();
1811     EBIT_CLR(flags, ENTRY_FWD_HDR_WAIT);
1812
1813     rep->body.packInto(&p);
1814
1815     packerClean(&p);
1816 }
1817
1818 char const *
1819 StoreEntry::getSerialisedMetaData()
1820 {
1821     StoreMeta *tlv_list = storeSwapMetaBuild(this);
1822     int swap_hdr_sz;
1823     char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
1824     storeSwapTLVFree(tlv_list);
1825     assert (swap_hdr_sz >= 0);
1826     mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz;
1827     return result;
1828 }
1829
1830 void
1831 StoreEntry::trimMemory(const bool preserveSwappable)
1832 {
1833     /*
1834      * DPW 2007-05-09
1835      * Bug #1943.  We must not let go any data for IN_MEMORY
1836      * objects.  We have to wait until the mem_status changes.
1837      */
1838     if (mem_status == IN_MEMORY)
1839         return;
1840
1841     if (EBIT_TEST(flags, ENTRY_SPECIAL))
1842         return; // cannot trim because we do not load them again
1843
1844     if (preserveSwappable)
1845         mem_obj->trimSwappable();
1846     else
1847         mem_obj->trimUnSwappable();
1848
1849     debugs(88, 7, *this << " inmem_lo=" << mem_obj->inmem_lo);
1850 }
1851
1852 bool
1853 StoreEntry::modifiedSince(HttpRequest * request) const
1854 {
1855     int object_length;
1856     time_t mod_time = lastmod;
1857
1858     if (mod_time < 0)
1859         mod_time = timestamp;
1860
1861     debugs(88, 3, "modifiedSince: '" << url() << "'");
1862
1863     debugs(88, 3, "modifiedSince: mod_time = " << mod_time);
1864
1865     if (mod_time < 0)
1866         return true;
1867
1868     /* Find size of the object */
1869     object_length = getReply()->content_length;
1870
1871     if (object_length < 0)
1872         object_length = contentLen();
1873
1874     if (mod_time > request->ims) {
1875         debugs(88, 3, "--> YES: entry newer than client");
1876         return true;
1877     } else if (mod_time < request->ims) {
1878         debugs(88, 3, "-->  NO: entry older than client");
1879         return false;
1880     } else if (request->imslen < 0) {
1881         debugs(88, 3, "-->  NO: same LMT, no client length");
1882         return false;
1883     } else if (request->imslen == object_length) {
1884         debugs(88, 3, "-->  NO: same LMT, same length");
1885         return false;
1886     } else {
1887         debugs(88, 3, "--> YES: same LMT, different length");
1888         return true;
1889     }
1890 }
1891
1892 bool
1893 StoreEntry::hasIfMatchEtag(const HttpRequest &request) const
1894 {
1895     const String reqETags = request.header.getList(HDR_IF_MATCH);
1896     return hasOneOfEtags(reqETags, false);
1897 }
1898
1899 bool
1900 StoreEntry::hasIfNoneMatchEtag(const HttpRequest &request) const
1901 {
1902     const String reqETags = request.header.getList(HDR_IF_NONE_MATCH);
1903     // weak comparison is allowed only for HEAD or full-body GET requests
1904     const bool allowWeakMatch = !request.flags.isRanged &&
1905                                 (request.method == Http::METHOD_GET || request.method == Http::METHOD_HEAD);
1906     return hasOneOfEtags(reqETags, allowWeakMatch);
1907 }
1908
1909 /// whether at least one of the request ETags matches entity ETag
1910 bool
1911 StoreEntry::hasOneOfEtags(const String &reqETags, const bool allowWeakMatch) const
1912 {
1913     const ETag repETag = getReply()->header.getETag(HDR_ETAG);
1914     if (!repETag.str)
1915         return strListIsMember(&reqETags, "*", ',');
1916
1917     bool matched = false;
1918     const char *pos = NULL;
1919     const char *item;
1920     int ilen;
1921     while (!matched && strListGetItem(&reqETags, ',', &item, &ilen, &pos)) {
1922         if (!strncmp(item, "*", ilen))
1923             matched = true;
1924         else {
1925             String str;
1926             str.append(item, ilen);
1927             ETag reqETag;
1928             if (etagParseInit(&reqETag, str.termedBuf())) {
1929                 matched = allowWeakMatch ? etagIsWeakEqual(repETag, reqETag) :
1930                           etagIsStrongEqual(repETag, reqETag);
1931             }
1932         }
1933     }
1934     return matched;
1935 }
1936
1937 SwapDir::Pointer
1938 StoreEntry::store() const
1939 {
1940     assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured);
1941     return INDEXSD(swap_dirn);
1942 }
1943
1944 void
1945 StoreEntry::unlink()
1946 {
1947     store()->unlink(*this); // implies disconnect()
1948     swap_filen = -1;
1949     swap_dirn = -1;
1950     swap_status = SWAPOUT_NONE;
1951 }
1952
1953 /*
1954  * return true if the entry is in a state where
1955  * it can accept more data (ie with write() method)
1956  */
1957 bool
1958 StoreEntry::isAccepting() const
1959 {
1960     if (STORE_PENDING != store_status)
1961         return false;
1962
1963     if (EBIT_TEST(flags, ENTRY_ABORTED))
1964         return false;
1965
1966     return true;
1967 }
1968
1969 std::ostream &operator <<(std::ostream &os, const StoreEntry &e)
1970 {
1971     os << "e:";
1972
1973     if (e.mem_obj) {
1974         if (e.mem_obj->xitTable.index > -1)
1975             os << 't' << e.mem_obj->xitTable.index;
1976         if (e.mem_obj->memCache.index > -1)
1977             os << 'm' << e.mem_obj->memCache.index;
1978     }
1979     if (e.swap_filen > -1 || e.swap_dirn > -1)
1980         os << 'd' << e.swap_filen << '@' << e.swap_dirn;
1981
1982     os << '=';
1983
1984     // print only non-default status values, using unique letters
1985     if (e.mem_status != NOT_IN_MEMORY ||
1986         e.store_status != STORE_PENDING ||
1987         e.swap_status != SWAPOUT_NONE ||
1988         e.ping_status != PING_NONE) {
1989         if (e.mem_status != NOT_IN_MEMORY) os << 'm';
1990         if (e.store_status != STORE_PENDING) os << 's';
1991         if (e.swap_status != SWAPOUT_NONE) os << 'w' << e.swap_status;
1992         if (e.ping_status != PING_NONE) os << 'p' << e.ping_status;
1993     }
1994
1995     // print only set flags, using unique letters
1996     if (e.flags) {
1997         if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) os << 'S';
1998         if (EBIT_TEST(e.flags, ENTRY_REVALIDATE)) os << 'R';
1999         if (EBIT_TEST(e.flags, DELAY_SENDING)) os << 'P';
2000         if (EBIT_TEST(e.flags, RELEASE_REQUEST)) os << 'X';
2001         if (EBIT_TEST(e.flags, REFRESH_REQUEST)) os << 'F';
2002         if (EBIT_TEST(e.flags, ENTRY_DISPATCHED)) os << 'D';
2003         if (EBIT_TEST(e.flags, KEY_PRIVATE)) os << 'I';
2004         if (EBIT_TEST(e.flags, ENTRY_FWD_HDR_WAIT)) os << 'W';
2005         if (EBIT_TEST(e.flags, ENTRY_NEGCACHED)) os << 'N';
2006         if (EBIT_TEST(e.flags, ENTRY_VALIDATED)) os << 'V';
2007         if (EBIT_TEST(e.flags, ENTRY_BAD_LENGTH)) os << 'L';
2008         if (EBIT_TEST(e.flags, ENTRY_ABORTED)) os << 'A';
2009     }
2010
2011     if (e.mem_obj && e.mem_obj->smpCollapsed)
2012         os << 'O';
2013
2014     return os << '/' << &e << '*' << e.locks();
2015 }
2016
2017 /* NullStoreEntry */
2018
2019 NullStoreEntry NullStoreEntry::_instance;
2020
2021 NullStoreEntry *
2022 NullStoreEntry::getInstance()
2023 {
2024     return &_instance;
2025 }
2026
2027 char const *
2028 NullStoreEntry::getMD5Text() const
2029 {
2030     return "N/A";
2031 }
2032
2033 void
2034 NullStoreEntry::operator delete(void*)
2035 {
2036     fatal ("Attempt to delete NullStoreEntry\n");
2037 }
2038
2039 char const *
2040 NullStoreEntry::getSerialisedMetaData()
2041 {
2042     return NULL;
2043 }
2044
2045 #if !_USE_INLINE_
2046 #include "Store.cci"
2047 #endif