src/store.cc

   1
   2 /*
   3  * $Id: store.cc,v 1.542 2001/10/17 20:25:03 hno Exp $
   4  *
   5  * DEBUG: section 20    Storage Manager
   6  * AUTHOR: Harvest Derived
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37
  38 #define REBUILD_TIMESTAMP_DELTA_MAX 2
  39
  40 #define STORE_IN_MEM_BUCKETS            (229)
  41
  42 const char *memStatusStr[] =
  43 {
  44     "NOT_IN_MEMORY",
  45     "IN_MEMORY"
  46 };
  47
  48 const char *pingStatusStr[] =
  49 {
  50     "PING_NONE",
  51     "PING_WAITING",
  52     "PING_DONE"
  53 };
  54
  55 const char *storeStatusStr[] =
  56 {
  57     "STORE_OK",
  58     "STORE_PENDING"
  59 };
  60
  61 const char *swapStatusStr[] =
  62 {
  63     "SWAPOUT_NONE",
  64     "SWAPOUT_WRITING",
  65     "SWAPOUT_DONE"
  66 };
  67
  68 typedef struct lock_ctrl_t {
  69     SIH *callback;
  70     void *callback_data;
  71     StoreEntry *e;
  72 } lock_ctrl_t;
  73
  74 extern OBJH storeIOStats;
  75
  76 /*
  77  * local function prototypes
  78  */
  79 static int storeEntryValidLength(const StoreEntry *);
  80 static void storeGetMemSpace(int);
  81 static void storeHashDelete(StoreEntry *);
  82 static MemObject *new_MemObject(const char *, const char *);
  83 static void destroy_MemObject(StoreEntry *);
  84 static FREE destroy_StoreEntry;
  85 static void storePurgeMem(StoreEntry *);
  86 static void storeEntryReferenced(StoreEntry *);
  87 static void storeEntryDereferenced(StoreEntry *);
  88 static int getKeyCounter(void);
  89 static int storeKeepInMemory(const StoreEntry *);
  90 static OBJH storeCheckCachableStats;
  91 static EVH storeLateRelease;
  92
  93 /*
  94  * local variables
  95  */
  96 static Stack LateReleaseStack;
  97
  98 #if URL_CHECKSUM_DEBUG
  99 unsigned int
 100 url_checksum(const char *url)
 101 {
 102     unsigned int ck;
 103     MD5_CTX M;
 104     static unsigned char digest[16];
 105     MD5Init(&M);
 106     MD5Update(&M, (unsigned char *) url, strlen(url));
 107     MD5Final(digest, &M);
 108     xmemcpy(&ck, digest, sizeof(ck));
 109     return ck;
 110 }
 111 #endif
 112
 113 static MemObject *
 114 new_MemObject(const char *url, const char *log_url)
 115 {
 116     MemObject *mem = memAllocate(MEM_MEMOBJECT);
 117     mem->reply = httpReplyCreate();
 118     mem->url = xstrdup(url);
 119 #if URL_CHECKSUM_DEBUG
 120     mem->chksum = url_checksum(mem->url);
 121 #endif
 122     mem->log_url = xstrdup(log_url);
 123     mem->object_sz = -1;
 124     mem->fd = -1;
 125     /* XXX account log_url */
 126     debug(20, 3) ("new_MemObject: returning %p\n", mem);
 127     return mem;
 128 }
 129
 130 StoreEntry *
 131 new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url)
 132 {
 133     StoreEntry *e = NULL;
 134     e = memAllocate(MEM_STOREENTRY);
 135     if (mem_obj_flag)
 136         e->mem_obj = new_MemObject(url, log_url);
 137     debug(20, 3) ("new_StoreEntry: returning %p\n", e);
 138     e->expires = e->lastmod = e->lastref = e->timestamp = -1;
 139     e->swap_filen = -1;
 140     e->swap_dirn = -1;
 141     return e;
 142 }
 143
 144 static void
 145 destroy_MemObject(StoreEntry * e)
 146 {
 147     MemObject *mem = e->mem_obj;
 148     const Ctx ctx = ctx_enter(mem->url);
 149     debug(20, 3) ("destroy_MemObject: destroying %p\n", mem);
 150 #if URL_CHECKSUM_DEBUG
 151     assert(mem->chksum == url_checksum(mem->url));
 152 #endif
 153     e->mem_obj = NULL;
 154     if (!shutting_down)
 155         assert(mem->swapout.sio == NULL);
 156     stmemFree(&mem->data_hdr);
 157     mem->inmem_hi = 0;
 158     /*
 159      * There is no way to abort FD-less clients, so they might
 160      * still have mem->clients set if mem->fd == -1
 161      */
 162     assert(mem->fd == -1 || mem->clients.head == NULL);
 163     httpReplyDestroy(mem->reply);
 164     requestUnlink(mem->request);
 165     mem->request = NULL;
 166     ctx_exit(ctx);              /* must exit before we free mem->url */
 167     safe_free(mem->url);
 168     safe_free(mem->log_url);    /* XXX account log_url */
 169     safe_free(mem->vary_headers);
 170     memFree(mem, MEM_MEMOBJECT);
 171 }
 172
 173 static void
 174 destroy_StoreEntry(void *data)
 175 {
 176     StoreEntry *e = data;
 177     debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e);
 178     assert(e != NULL);
 179     if (e->mem_obj)
 180         destroy_MemObject(e);
 181     storeHashDelete(e);
 182     assert(e->hash.key == NULL);
 183     memFree(e, MEM_STOREENTRY);
 184 }
 185
 186 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
 187
 188 void
 189 storeHashInsert(StoreEntry * e, const cache_key * key)
 190 {
 191     debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n",
 192         e, storeKeyText(key));
 193     e->hash.key = storeKeyDup(key);
 194     hash_join(store_table, &e->hash);
 195 }
 196
 197 static void
 198 storeHashDelete(StoreEntry * e)
 199 {
 200     hash_remove_link(store_table, &e->hash);
 201     storeKeyFree(e->hash.key);
 202     e->hash.key = NULL;
 203 }
 204
 205 /* -------------------------------------------------------------------------- */
 206
 207
 208 /* get rid of memory copy of the object */
 209 /* Only call this if storeCheckPurgeMem(e) returns 1 */
 210 static void
 211 storePurgeMem(StoreEntry * e)
 212 {
 213     if (e->mem_obj == NULL)
 214         return;
 215     debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n",
 216         storeKeyText(e->hash.key));
 217     storeSetMemStatus(e, NOT_IN_MEMORY);
 218     destroy_MemObject(e);
 219     if (e->swap_status != SWAPOUT_DONE)
 220         storeRelease(e);
 221 }
 222
 223 static void
 224 storeEntryReferenced(StoreEntry * e)
 225 {
 226     SwapDir *SD;
 227
 228     /* Notify the fs that we're referencing this object again */
 229     if (e->swap_dirn > -1) {
 230         SD = INDEXSD(e->swap_dirn);
 231         if (SD->refobj)
 232             SD->refobj(SD, e);
 233     }
 234     /* Notify the memory cache that we're referencing this object again */
 235     if (e->mem_obj) {
 236         if (mem_policy->Referenced)
 237             mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl);
 238     }
 239 }
 240
 241 static void
 242 storeEntryDereferenced(StoreEntry * e)
 243 {
 244     SwapDir *SD;
 245
 246     /* Notify the fs that we're not referencing this object any more */
 247     if (e->swap_filen > -1) {
 248         SD = INDEXSD(e->swap_dirn);
 249         if (SD->unrefobj != NULL)
 250             SD->unrefobj(SD, e);
 251     }
 252     /* Notify the memory cache that we're not referencing this object any more */
 253     if (e->mem_obj) {
 254         if (mem_policy->Dereferenced)
 255             mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl);
 256     }
 257 }
 258
 259 void
 260 storeLockObject(StoreEntry * e)
 261 {
 262     e->lock_count++;
 263     debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
 264         storeKeyText(e->hash.key), (int) e->lock_count);
 265     e->lastref = squid_curtime;
 266     storeEntryReferenced(e);
 267 }
 268
 269 void
 270 storeReleaseRequest(StoreEntry * e)
 271 {
 272     if (EBIT_TEST(e->flags, RELEASE_REQUEST))
 273         return;
 274     debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->hash.key));
 275     EBIT_SET(e->flags, RELEASE_REQUEST);
 276     /*
 277      * Clear cachable flag here because we might get called before
 278      * anyone else even looks at the cachability flag.  Also, this
 279      * prevents httpMakePublic from really setting a public key.
 280      */
 281     EBIT_CLR(e->flags, ENTRY_CACHABLE);
 282     storeSetPrivateKey(e);
 283 }
 284
 285 /* unlock object, return -1 if object get released after unlock
 286  * otherwise lock_count */
 287 int
 288 storeUnlockObject(StoreEntry * e)
 289 {
 290     e->lock_count--;
 291     debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
 292         storeKeyText(e->hash.key), e->lock_count);
 293     if (e->lock_count)
 294         return (int) e->lock_count;
 295     if (e->store_status == STORE_PENDING)
 296         EBIT_SET(e->flags, RELEASE_REQUEST);
 297     assert(storePendingNClients(e) == 0);
 298     if (EBIT_TEST(e->flags, RELEASE_REQUEST))
 299         storeRelease(e);
 300     else if (storeKeepInMemory(e)) {
 301         storeEntryDereferenced(e);
 302         storeSetMemStatus(e, IN_MEMORY);
 303         requestUnlink(e->mem_obj->request);
 304         e->mem_obj->request = NULL;
 305     } else {
 306         storePurgeMem(e);
 307         storeEntryDereferenced(e);
 308         if (EBIT_TEST(e->flags, KEY_PRIVATE))
 309             debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__);
 310     }
 311     return 0;
 312 }
 313
 314 /* Lookup an object in the cache.
 315  * return just a reference to object, don't start swapping in yet. */
 316 StoreEntry *
 317 storeGet(const cache_key * key)
 318 {
 319     debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key));
 320     return (StoreEntry *) hash_lookup(store_table, key);
 321 }
 322
 323 StoreEntry *
 324 storeGetPublic(const char *uri, const method_t method)
 325 {
 326     return storeGet(storeKeyPublic(uri, method));
 327 }
 328
 329 StoreEntry *
 330 storeGetPublicByRequestMethod(request_t * req, const method_t method)
 331 {
 332     return storeGet(storeKeyPublicByRequestMethod(req, method));
 333 }
 334
 335 StoreEntry *
 336 storeGetPublicByRequest(request_t * req)
 337 {
 338     StoreEntry *e = storeGetPublicByRequestMethod(req, req->method);
 339     if (e == NULL && req->method == METHOD_HEAD)
 340         /* We can generate a HEAD reply from a cached GET object */
 341         e = storeGetPublicByRequestMethod(req, METHOD_GET);
 342     return e;
 343 }
 344
 345 static int
 346 getKeyCounter(void)
 347 {
 348     static int key_counter = 0;
 349     if (++key_counter < 0)
 350         key_counter = 1;
 351     return key_counter;
 352 }
 353
 354 void
 355 storeSetPrivateKey(StoreEntry * e)
 356 {
 357     const cache_key *newkey;
 358     MemObject *mem = e->mem_obj;
 359     if (e->hash.key && EBIT_TEST(e->flags, KEY_PRIVATE))
 360         return;                 /* is already private */
 361     if (e->hash.key) {
 362         if (e->swap_filen > -1)
 363             storeDirSwapLog(e, SWAP_LOG_DEL);
 364         storeHashDelete(e);
 365     }
 366     if (mem != NULL) {
 367         mem->id = getKeyCounter();
 368         newkey = storeKeyPrivate(mem->url, mem->method, mem->id);
 369     } else {
 370         newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter());
 371     }
 372     assert(hash_lookup(store_table, newkey) == NULL);
 373     EBIT_SET(e->flags, KEY_PRIVATE);
 374     storeHashInsert(e, newkey);
 375 }
 376
 377 void
 378 storeSetPublicKey(StoreEntry * e)
 379 {
 380     StoreEntry *e2 = NULL;
 381     const cache_key *newkey;
 382     MemObject *mem = e->mem_obj;
 383     if (e->hash.key && !EBIT_TEST(e->flags, KEY_PRIVATE))
 384         return;                 /* is already public */
 385     assert(mem);
 386     /*
 387      * We can't make RELEASE_REQUEST objects public.  Depending on
 388      * when RELEASE_REQUEST gets set, we might not be swapping out
 389      * the object.  If we're not swapping out, then subsequent
 390      * store clients won't be able to access object data which has
 391      * been freed from memory.
 392      *
 393      * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
 394      * be set, and storeSetPublicKey() should not be called.
 395      */
 396 #if MORE_DEBUG_OUTPUT
 397     if (EBIT_TEST(e->flags, RELEASE_REQUEST))
 398         debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n",
 399             e->hash.key, mem->url);
 400 #endif
 401     assert(!EBIT_TEST(e->flags, RELEASE_REQUEST));
 402     if (mem->request) {
 403         StoreEntry *pe;
 404         request_t *request = mem->request;
 405         if (!mem->vary_headers) {
 406             /* First handle the case where the object no longer varies */
 407             safe_free(request->vary_headers);
 408         } else {
 409             if (request->vary_headers && strcmp(request->vary_headers, mem->vary_headers) != 0) {
 410                 /* Oops.. the variance has changed. Kill the base object
 411                  * to record the new variance key
 412                  */
 413                 safe_free(request->vary_headers);       /* free old "bad" variance key */
 414                 pe = storeGetPublic(mem->url, mem->method);
 415                 if (pe)
 416                     storeRelease(pe);
 417             }
 418             /* Make sure the request knows the variance status */
 419             if (!request->vary_headers)
 420                 request->vary_headers = xstrdup(httpMakeVaryMark(request, mem->reply));
 421         }
 422         if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) {
 423             /* Create "vary" base object */
 424             http_version_t version;
 425             String vary;
 426             pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method);
 427             httpBuildVersion(&version, 1, 0);
 428             httpReplySetHeaders(pe->mem_obj->reply, version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000);
 429             vary = httpHeaderGetList(&mem->reply->header, HDR_VARY);
 430             if (strBuf(vary)) {
 431                 httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_VARY, strBuf(vary));
 432                 stringClean(&vary);
 433             }
 434 #if X_ACCELERATOR_VARY
 435             vary = httpHeaderGetList(&mem->reply->header, HDR_X_ACCELERATOR_VARY);
 436             if (strBuf(vary)) {
 437                 httpHeaderPutStr(&pe->mem_obj->reply->header, HDR_X_ACCELERATOR_VARY, strBuf(vary));
 438                 stringClean(&vary);
 439             }
 440 #endif
 441             storeSetPublicKey(pe);
 442             httpReplySwapOut(pe->mem_obj->reply, pe);
 443             storeBufferFlush(pe);
 444             storeTimestampsSet(pe);
 445             storeComplete(pe);
 446             storeUnlockObject(pe);
 447         }
 448         newkey = storeKeyPublicByRequest(mem->request);
 449     } else
 450         newkey = storeKeyPublic(mem->url, mem->method);
 451     if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
 452         debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url);
 453         storeSetPrivateKey(e2);
 454         storeRelease(e2);
 455         if (mem->request)
 456             newkey = storeKeyPublicByRequest(mem->request);
 457         else
 458             newkey = storeKeyPublic(mem->url, mem->method);
 459     }
 460     if (e->hash.key)
 461         storeHashDelete(e);
 462     EBIT_CLR(e->flags, KEY_PRIVATE);
 463     storeHashInsert(e, newkey);
 464     if (e->swap_filen > -1)
 465         storeDirSwapLog(e, SWAP_LOG_ADD);
 466 }
 467
 468 StoreEntry *
 469 storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method)
 470 {
 471     StoreEntry *e = NULL;
 472     MemObject *mem = NULL;
 473     debug(20, 3) ("storeCreateEntry: '%s'\n", url);
 474
 475     e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url);
 476     e->lock_count = 1;          /* Note lock here w/o calling storeLock() */
 477     mem = e->mem_obj;
 478     mem->method = method;
 479     if (neighbors_do_private_keys || !flags.hierarchical)
 480         storeSetPrivateKey(e);
 481     else
 482         storeSetPublicKey(e);
 483     if (flags.cachable) {
 484         EBIT_SET(e->flags, ENTRY_CACHABLE);
 485         EBIT_CLR(e->flags, RELEASE_REQUEST);
 486     } else {
 487         EBIT_CLR(e->flags, ENTRY_CACHABLE);
 488         storeReleaseRequest(e);
 489     }
 490     e->store_status = STORE_PENDING;
 491     storeSetMemStatus(e, NOT_IN_MEMORY);
 492     e->swap_status = SWAPOUT_NONE;
 493     e->swap_filen = -1;
 494     e->swap_dirn = -1;
 495     e->refcount = 0;
 496     e->lastref = squid_curtime;
 497     e->timestamp = -1;          /* set in storeTimestampsSet() */
 498     e->ping_status = PING_NONE;
 499     EBIT_SET(e->flags, ENTRY_VALIDATED);
 500     return e;
 501 }
 502
 503 /* Mark object as expired */
 504 void
 505 storeExpireNow(StoreEntry * e)
 506 {
 507     debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->hash.key));
 508     e->expires = squid_curtime;
 509 }
 510
 511 /* Append incoming data from a primary server to an entry. */
 512 void
 513 storeAppend(StoreEntry * e, const char *buf, int len)
 514 {
 515     MemObject *mem = e->mem_obj;
 516     assert(mem != NULL);
 517     assert(len >= 0);
 518     assert(e->store_status == STORE_PENDING);
 519     if (len) {
 520         debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
 521             len,
 522             storeKeyText(e->hash.key));
 523         storeGetMemSpace(len);
 524         stmemAppend(&mem->data_hdr, buf, len);
 525         mem->inmem_hi += len;
 526     }
 527     if (EBIT_TEST(e->flags, DELAY_SENDING))
 528         return;
 529     InvokeHandlers(e);
 530     storeSwapOut(e);
 531 }
 532
 533 void
 534 #if STDC_HEADERS
 535 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
 536 #else
 537 storeAppendPrintf(va_alist)
 538      va_dcl
 539 #endif
 540 {
 541 #if STDC_HEADERS
 542     va_list args;
 543     va_start(args, fmt);
 544 #else
 545     va_list args;
 546     StoreEntry *e = NULL;
 547     const char *fmt = NULL;
 548     va_start(args);
 549     e = va_arg(args, StoreEntry *);
 550     fmt = va_arg(args, char *);
 551 #endif
 552     storeAppendVPrintf(e, fmt, args);
 553     va_end(args);
 554 }
 555
 556 /* used be storeAppendPrintf and Packer */
 557 void
 558 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
 559 {
 560     LOCAL_ARRAY(char, buf, 4096);
 561     buf[0] = '\0';
 562     vsnprintf(buf, 4096, fmt, vargs);
 563     storeAppend(e, buf, strlen(buf));
 564 }
 565
 566 struct _store_check_cachable_hist {
 567     struct {
 568         int non_get;
 569         int not_entry_cachable;
 570         int release_request;
 571         int wrong_content_length;
 572         int negative_cached;
 573         int too_big;
 574         int too_small;
 575         int private_key;
 576         int too_many_open_files;
 577         int too_many_open_fds;
 578     } no;
 579     struct {
 580         int Default;
 581     } yes;
 582 } store_check_cachable_hist;
 583
 584 int
 585 storeTooManyDiskFilesOpen(void)
 586 {
 587     if (Config.max_open_disk_fds == 0)
 588         return 0;
 589     if (store_open_disk_fd > Config.max_open_disk_fds)
 590         return 1;
 591     return 0;
 592 }
 593
 594 static int
 595 storeCheckTooSmall(StoreEntry * e)
 596 {
 597     MemObject *mem = e->mem_obj;
 598     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 599         return 0;
 600     if (STORE_OK == e->store_status)
 601         if (mem->object_sz < Config.Store.minObjectSize)
 602             return 1;
 603     if (mem->reply->content_length > -1)
 604         if (mem->reply->content_length < (int) Config.Store.minObjectSize)
 605             return 1;
 606     return 0;
 607 }
 608
 609 int
 610 storeCheckCachable(StoreEntry * e)
 611 {
 612 #if CACHE_ALL_METHODS
 613     if (e->mem_obj->method != METHOD_GET) {
 614         debug(20, 2) ("storeCheckCachable: NO: non-GET method\n");
 615         store_check_cachable_hist.no.non_get++;
 616     } else
 617 #endif
 618     if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) {
 619         debug(20, 2) ("storeCheckCachable: NO: not cachable\n");
 620         store_check_cachable_hist.no.not_entry_cachable++;
 621     } else if (EBIT_TEST(e->flags, RELEASE_REQUEST)) {
 622         debug(20, 2) ("storeCheckCachable: NO: release requested\n");
 623         store_check_cachable_hist.no.release_request++;
 624     } else if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) {
 625         debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n");
 626         store_check_cachable_hist.no.wrong_content_length++;
 627     } else if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) {
 628         debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
 629         store_check_cachable_hist.no.negative_cached++;
 630         return 0;               /* avoid release call below */
 631     } else if ((e->mem_obj->reply->content_length > 0 &&
 632                 e->mem_obj->reply->content_length > Config.Store.maxObjectSize) ||
 633         e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
 634         debug(20, 2) ("storeCheckCachable: NO: too big\n");
 635         store_check_cachable_hist.no.too_big++;
 636     } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) {
 637         debug(20, 2) ("storeCheckCachable: NO: too big\n");
 638         store_check_cachable_hist.no.too_big++;
 639     } else if (storeCheckTooSmall(e)) {
 640         debug(20, 2) ("storeCheckCachable: NO: too small\n");
 641         store_check_cachable_hist.no.too_small++;
 642     } else if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
 643         debug(20, 3) ("storeCheckCachable: NO: private key\n");
 644         store_check_cachable_hist.no.private_key++;
 645     } else if (e->swap_status != SWAPOUT_NONE) {
 646         /*
 647          * here we checked the swap_status because the remaining
 648          * cases are only relevant only if we haven't started swapping
 649          * out the object yet.
 650          */
 651         return 1;
 652     } else if (storeTooManyDiskFilesOpen()) {
 653         debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n");
 654         store_check_cachable_hist.no.too_many_open_files++;
 655     } else if (fdNFree() < RESERVED_FD) {
 656         debug(20, 2) ("storeCheckCachable: NO: too many FD's open\n");
 657         store_check_cachable_hist.no.too_many_open_fds++;
 658     } else {
 659         store_check_cachable_hist.yes.Default++;
 660         return 1;
 661     }
 662     storeReleaseRequest(e);
 663     EBIT_CLR(e->flags, ENTRY_CACHABLE);
 664     return 0;
 665 }
 666
 667 static void
 668 storeCheckCachableStats(StoreEntry * sentry)
 669 {
 670     storeAppendPrintf(sentry, "Category\t Count\n");
 671
 672     storeAppendPrintf(sentry, "no.non_get\t%d\n",
 673         store_check_cachable_hist.no.non_get);
 674     storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
 675         store_check_cachable_hist.no.not_entry_cachable);
 676     storeAppendPrintf(sentry, "no.release_request\t%d\n",
 677         store_check_cachable_hist.no.release_request);
 678     storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
 679         store_check_cachable_hist.no.wrong_content_length);
 680     storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
 681         store_check_cachable_hist.no.negative_cached);
 682     storeAppendPrintf(sentry, "no.too_big\t%d\n",
 683         store_check_cachable_hist.no.too_big);
 684     storeAppendPrintf(sentry, "no.too_small\t%d\n",
 685         store_check_cachable_hist.no.too_small);
 686     storeAppendPrintf(sentry, "no.private_key\t%d\n",
 687         store_check_cachable_hist.no.private_key);
 688     storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
 689         store_check_cachable_hist.no.too_many_open_files);
 690     storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
 691         store_check_cachable_hist.no.too_many_open_fds);
 692     storeAppendPrintf(sentry, "yes.default\t%d\n",
 693         store_check_cachable_hist.yes.Default);
 694 }
 695
 696 /* Complete transfer into the local cache.  */
 697 void
 698 storeComplete(StoreEntry * e)
 699 {
 700     debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->hash.key));
 701     if (e->store_status != STORE_PENDING) {
 702         /*
 703          * if we're not STORE_PENDING, then probably we got aborted
 704          * and there should be NO clients on this entry
 705          */
 706         assert(EBIT_TEST(e->flags, ENTRY_ABORTED));
 707         assert(e->mem_obj->nclients == 0);
 708         return;
 709     }
 710     e->mem_obj->object_sz = e->mem_obj->inmem_hi;
 711     e->store_status = STORE_OK;
 712     assert(e->mem_status == NOT_IN_MEMORY);
 713     if (!storeEntryValidLength(e)) {
 714         EBIT_SET(e->flags, ENTRY_BAD_LENGTH);
 715         storeReleaseRequest(e);
 716     }
 717 #if USE_CACHE_DIGESTS
 718     if (e->mem_obj->request)
 719         e->mem_obj->request->hier.store_complete_stop = current_time;
 720 #endif
 721     /*
 722      * We used to call InvokeHandlers, then storeSwapOut.  However,
 723      * Madhukar Reddy <myreddy@persistence.com> reported that
 724      * responses without content length would sometimes get released
 725      * in client_side, thinking that the response is incomplete.
 726      */
 727     storeSwapOut(e);
 728     InvokeHandlers(e);
 729 }
 730
 731 /*
 732  * Someone wants to abort this transfer.  Set the reason in the
 733  * request structure, call the server-side callback and mark the
 734  * entry for releasing
 735  */
 736 void
 737 storeAbort(StoreEntry * e)
 738 {
 739     MemObject *mem = e->mem_obj;
 740     assert(e->store_status == STORE_PENDING);
 741     assert(mem != NULL);
 742     debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->hash.key));
 743     storeLockObject(e);         /* lock while aborting */
 744     storeNegativeCache(e);
 745     storeReleaseRequest(e);
 746     EBIT_SET(e->flags, ENTRY_ABORTED);
 747     storeSetMemStatus(e, NOT_IN_MEMORY);
 748     e->store_status = STORE_OK;
 749     /*
 750      * We assign an object length here.  The only other place we assign
 751      * the object length is in storeComplete()
 752      */
 753     mem->object_sz = mem->inmem_hi;
 754     /* Notify the server side */
 755     if (mem->abort.callback) {
 756         eventAdd("mem->abort.callback",
 757             mem->abort.callback,
 758             mem->abort.data,
 759             0.0,
 760             0);
 761         mem->abort.callback = NULL;
 762         mem->abort.data = NULL;
 763     }
 764     /* Notify the client side */
 765     InvokeHandlers(e);
 766     /* Close any swapout file */
 767     storeSwapOutFileClose(e);
 768     storeUnlockObject(e);       /* unlock */
 769 }
 770
 771 /* Clear Memory storage to accommodate the given object len */
 772 static void
 773 storeGetMemSpace(int size)
 774 {
 775     StoreEntry *e = NULL;
 776     int released = 0;
 777     static time_t last_check = 0;
 778     int pages_needed;
 779     RemovalPurgeWalker *walker;
 780     if (squid_curtime == last_check)
 781         return;
 782     last_check = squid_curtime;
 783     pages_needed = (size / SM_PAGE_SIZE) + 1;
 784     if (memInUse(MEM_MEM_NODE) + pages_needed < store_pages_max)
 785         return;
 786     debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed);
 787     /* XXX what to set as max_scan here? */
 788     walker = mem_policy->PurgeInit(mem_policy, 100000);
 789     while ((e = walker->Next(walker))) {
 790         storePurgeMem(e);
 791         released++;
 792         if (memInUse(MEM_MEM_NODE) + pages_needed < store_pages_max)
 793             break;
 794     }
 795     walker->Done(walker);
 796     debug(20, 3) ("storeGetMemSpace stats:\n");
 797     debug(20, 3) ("  %6d HOT objects\n", hot_obj_count);
 798     debug(20, 3) ("  %6d were released\n", released);
 799 }
 800
 801 /* The maximum objects to scan for maintain storage space */
 802 #define MAINTAIN_MAX_SCAN       1024
 803 #define MAINTAIN_MAX_REMOVE     64
 804
 805 /*
 806  * This routine is to be called by main loop in main.c.
 807  * It removes expired objects on only one bucket for each time called.
 808  * returns the number of objects removed
 809  *
 810  * This should get called 1/s from main().
 811  */
 812 void
 813 storeMaintainSwapSpace(void *datanotused)
 814 {
 815     int i;
 816     SwapDir *SD;
 817     static time_t last_warn_time = 0;
 818
 819     /* walk each fs */
 820     for (i = 0; i < Config.cacheSwap.n_configured; i++) {
 821         /* call the maintain function .. */
 822         SD = INDEXSD(i);
 823         /* XXX FixMe: This should be done "in parallell" on the different
 824          * cache_dirs, not one at a time.
 825          */
 826         if (SD->maintainfs != NULL)
 827             SD->maintainfs(SD);
 828     }
 829     if (store_swap_size > Config.Swap.maxSize) {
 830         if (squid_curtime - last_warn_time > 10) {
 831             debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
 832                 store_swap_size, Config.Swap.maxSize);
 833             last_warn_time = squid_curtime;
 834         }
 835     }
 836     /* Reregister a maintain event .. */
 837     eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
 838 }
 839
 840
 841 /* release an object from a cache */
 842 void
 843 storeRelease(StoreEntry * e)
 844 {
 845     debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->hash.key));
 846     /* If, for any reason we can't discard this object because of an
 847      * outstanding request, mark it for pending release */
 848     if (storeEntryLocked(e)) {
 849         storeExpireNow(e);
 850         debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n");
 851         storeReleaseRequest(e);
 852         return;
 853     }
 854     if (store_dirs_rebuilding && e->swap_filen > -1) {
 855         storeSetPrivateKey(e);
 856         if (e->mem_obj) {
 857             storeSetMemStatus(e, NOT_IN_MEMORY);
 858             destroy_MemObject(e);
 859         }
 860         if (e->swap_filen > -1) {
 861             /*
 862              * Fake a call to storeLockObject().  When rebuilding is done,
 863              * we'll just call storeUnlockObject() on these.
 864              */
 865             e->lock_count++;
 866             EBIT_SET(e->flags, RELEASE_REQUEST);
 867             stackPush(&LateReleaseStack, e);
 868             return;
 869         } else {
 870             destroy_StoreEntry(e);
 871         }
 872     }
 873     storeLog(STORE_LOG_RELEASE, e);
 874     if (e->swap_filen > -1) {
 875         storeUnlink(e);
 876         if (e->swap_status == SWAPOUT_DONE)
 877             if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
 878                 storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1);
 879         if (!EBIT_TEST(e->flags, KEY_PRIVATE))
 880             storeDirSwapLog(e, SWAP_LOG_DEL);
 881 #if 0
 882         /* From 2.4. I think we do this in storeUnlink? */
 883         storeSwapFileNumberSet(e, -1);
 884 #endif
 885     }
 886     storeSetMemStatus(e, NOT_IN_MEMORY);
 887     destroy_StoreEntry(e);
 888 }
 889
 890 static void
 891 storeLateRelease(void *unused)
 892 {
 893     StoreEntry *e;
 894     int i;
 895     static int n = 0;
 896     if (store_dirs_rebuilding) {
 897         eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
 898         return;
 899     }
 900     for (i = 0; i < 10; i++) {
 901         e = stackPop(&LateReleaseStack);
 902         if (e == NULL) {
 903             /* done! */
 904             debug(20, 1) ("storeLateRelease: released %d objects\n", n);
 905             return;
 906         }
 907         storeUnlockObject(e);
 908         n++;
 909     }
 910     eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
 911 }
 912
 913 /* return 1 if a store entry is locked */
 914 int
 915 storeEntryLocked(const StoreEntry * e)
 916 {
 917     if (e->lock_count)
 918         return 1;
 919     if (e->swap_status == SWAPOUT_WRITING)
 920         return 1;
 921     if (e->store_status == STORE_PENDING)
 922         return 1;
 923     /*
 924      * SPECIAL, PUBLIC entries should be "locked"
 925      */
 926     if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
 927         if (!EBIT_TEST(e->flags, KEY_PRIVATE))
 928             return 1;
 929     return 0;
 930 }
 931
 932 static int
 933 storeEntryValidLength(const StoreEntry * e)
 934 {
 935     int diff;
 936     const HttpReply *reply;
 937     assert(e->mem_obj != NULL);
 938     reply = e->mem_obj->reply;
 939     debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->hash.key));
 940     debug(20, 5) ("storeEntryValidLength:     object_len = %d\n",
 941         objectLen(e));
 942     debug(20, 5) ("storeEntryValidLength:         hdr_sz = %d\n",
 943         reply->hdr_sz);
 944     debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
 945         reply->content_length);
 946     if (reply->content_length < 0) {
 947         debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
 948             storeKeyText(e->hash.key));
 949         return 1;
 950     }
 951     if (reply->hdr_sz == 0) {
 952         debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n",
 953             storeKeyText(e->hash.key));
 954         return 1;
 955     }
 956     if (e->mem_obj->method == METHOD_HEAD) {
 957         debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n",
 958             storeKeyText(e->hash.key));
 959         return 1;
 960     }
 961     if (reply->sline.status == HTTP_NOT_MODIFIED)
 962         return 1;
 963     if (reply->sline.status == HTTP_NO_CONTENT)
 964         return 1;
 965     diff = reply->hdr_sz + reply->content_length - objectLen(e);
 966     if (diff == 0)
 967         return 1;
 968     debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
 969         diff < 0 ? -diff : diff,
 970         diff < 0 ? "big" : "small",
 971         storeKeyText(e->hash.key));
 972     return 0;
 973 }
 974
 975 static void
 976 storeInitHashValues(void)
 977 {
 978     int i;
 979     /* Calculate size of hash table (maximum currently 64k buckets).  */
 980     i = Config.Swap.maxSize / Config.Store.avgObjectSize;
 981     debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n",
 982         Config.Swap.maxSize, i);
 983     i /= Config.Store.objectsPerBucket;
 984     debug(20, 1) ("Target number of buckets: %d\n", i);
 985     /* ideally the full scan period should be configurable, for the
 986      * moment it remains at approximately 24 hours.  */
 987     store_hash_buckets = storeKeyHashBuckets(i);
 988     debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets);
 989     debug(20, 1) ("Max Mem  size: %d KB\n", Config.memMaxSize >> 10);
 990     debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize);
 991 }
 992
 993 void
 994 storeInit(void)
 995 {
 996     storeKeyInit();
 997     storeInitHashValues();
 998     store_table = hash_create(storeKeyHashCmp,
 999         store_hash_buckets, storeKeyHashHash);
1000     mem_policy = createRemovalPolicy(Config.memPolicy);
1001     storeDigestInit();
1002     storeLogOpen();
1003     stackInit(&LateReleaseStack);
1004     eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
1005     storeDirInit();
1006     storeRebuildStart();
1007     cachemgrRegister("storedir",
1008         "Store Directory Stats",
1009         storeDirStats, 0, 1);
1010     cachemgrRegister("store_check_cachable_stats",
1011         "storeCheckCachable() Stats",
1012         storeCheckCachableStats, 0, 1);
1013     cachemgrRegister("store_io",
1014         "Store IO Interface Stats",
1015         storeIOStats, 0, 1);
1016 }
1017
1018 void
1019 storeConfigure(void)
1020 {
1021     store_swap_high = (long) (((float) Config.Swap.maxSize *
1022             (float) Config.Swap.highWaterMark) / (float) 100);
1023     store_swap_low = (long) (((float) Config.Swap.maxSize *
1024             (float) Config.Swap.lowWaterMark) / (float) 100);
1025     store_pages_max = Config.memMaxSize / SM_PAGE_SIZE;
1026 }
1027
1028 static int
1029 storeKeepInMemory(const StoreEntry * e)
1030 {
1031     MemObject *mem = e->mem_obj;
1032     if (mem == NULL)
1033         return 0;
1034     if (mem->data_hdr.head == NULL)
1035         return 0;
1036     return mem->inmem_lo == 0;
1037 }
1038
1039 void
1040 storeNegativeCache(StoreEntry * e)
1041 {
1042     e->expires = squid_curtime + Config.negativeTtl;
1043     EBIT_SET(e->flags, ENTRY_NEGCACHED);
1044 }
1045
1046 void
1047 storeFreeMemory(void)
1048 {
1049     hashFreeItems(store_table, destroy_StoreEntry);
1050     hashFreeMemory(store_table);
1051     store_table = NULL;
1052 #if USE_CACHE_DIGESTS
1053     if (store_digest)
1054         cacheDigestDestroy(store_digest);
1055 #endif
1056     store_digest = NULL;
1057 }
1058
1059 int
1060 expiresMoreThan(time_t expires, time_t when)
1061 {
1062     if (expires < 0)            /* No Expires given */
1063         return 1;
1064     return (expires > (squid_curtime + when));
1065 }
1066
1067 int
1068 storeEntryValidToSend(StoreEntry * e)
1069 {
1070     if (EBIT_TEST(e->flags, RELEASE_REQUEST))
1071         return 0;
1072     if (EBIT_TEST(e->flags, ENTRY_NEGCACHED))
1073         if (e->expires <= squid_curtime)
1074             return 0;
1075     if (EBIT_TEST(e->flags, ENTRY_ABORTED))
1076         return 0;
1077     return 1;
1078 }
1079
1080 void
1081 storeTimestampsSet(StoreEntry * entry)
1082 {
1083     const HttpReply *reply = entry->mem_obj->reply;
1084     time_t served_date = reply->date;
1085     int age = httpHeaderGetInt(&reply->header, HDR_AGE);
1086     /*
1087      * The timestamp calculations below tries to mimic the properties
1088      * of the age calculation in RFC2616 section 13.2.3. The implementaion
1089      * isn't complete, and the most notable exception from the RFC is that
1090      * this does not account for response_delay, but it probably does
1091      * not matter much as this is calculated immediately when the headers
1092      * are received, not when the whole response has been received.
1093      */
1094     /* make sure that 0 <= served_date <= squid_curtime */
1095     if (served_date < 0 || served_date > squid_curtime)
1096         served_date = squid_curtime;
1097     /*
1098      * Compensate with Age header if origin server clock is ahead
1099      * of us and there is a cache in between us and the origin
1100      * server.  But DONT compensate if the age value is larger than
1101      * squid_curtime because it results in a negative served_date.
1102      */
1103     if (age > squid_curtime - served_date)
1104         if (squid_curtime > age)
1105             served_date = squid_curtime - age;
1106     entry->expires = reply->expires;
1107     entry->lastmod = reply->last_modified;
1108     entry->timestamp = served_date;
1109 }
1110
1111 void
1112 storeRegisterAbort(StoreEntry * e, STABH * cb, void *data)
1113 {
1114     MemObject *mem = e->mem_obj;
1115     assert(mem);
1116     assert(mem->abort.callback == NULL);
1117     mem->abort.callback = cb;
1118     mem->abort.data = data;
1119 }
1120
1121 void
1122 storeUnregisterAbort(StoreEntry * e)
1123 {
1124     MemObject *mem = e->mem_obj;
1125     assert(mem);
1126     mem->abort.callback = NULL;
1127 }
1128
1129 void
1130 storeMemObjectDump(MemObject * mem)
1131 {
1132     debug(20, 1) ("MemObject->data.head: %p\n",
1133         mem->data_hdr.head);
1134     debug(20, 1) ("MemObject->data.tail: %p\n",
1135         mem->data_hdr.tail);
1136     debug(20, 1) ("MemObject->data.origin_offset: %d\n",
1137         mem->data_hdr.origin_offset);
1138     debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
1139         (int) mem->start_ping.tv_sec,
1140         (int) mem->start_ping.tv_usec);
1141     debug(20, 1) ("MemObject->inmem_hi: %d\n",
1142         (int) mem->inmem_hi);
1143     debug(20, 1) ("MemObject->inmem_lo: %d\n",
1144         (int) mem->inmem_lo);
1145     debug(20, 1) ("MemObject->clients: %p\n",
1146         mem->clients);
1147     debug(20, 1) ("MemObject->nclients: %d\n",
1148         mem->nclients);
1149     debug(20, 1) ("MemObject->reply: %p\n",
1150         mem->reply);
1151     debug(20, 1) ("MemObject->request: %p\n",
1152         mem->request);
1153     debug(20, 1) ("MemObject->log_url: %p %s\n",
1154         mem->log_url,
1155         checkNullString(mem->log_url));
1156 }
1157
1158 void
1159 storeEntryDump(const StoreEntry * e, int l)
1160 {
1161     debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->hash.key));
1162     debug(20, l) ("StoreEntry->next: %p\n", e->hash.next);
1163     debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj);
1164     debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp);
1165     debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref);
1166     debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires);
1167     debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod);
1168     debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
1169     debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
1170     debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
1171     debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn);
1172     debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen);
1173     debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count);
1174     debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status);
1175     debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status);
1176     debug(20, l) ("StoreEntry->store_status: %d\n", (int) e->store_status);
1177     debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status);
1178 }
1179
1180 /*
1181  * NOTE, this function assumes only two mem states
1182  */
1183 void
1184 storeSetMemStatus(StoreEntry * e, int new_status)
1185 {
1186     MemObject *mem = e->mem_obj;
1187     if (new_status == e->mem_status)
1188         return;
1189     assert(mem != NULL);
1190     if (new_status == IN_MEMORY) {
1191         assert(mem->inmem_lo == 0);
1192         if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1193             debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n",
1194                 mem->url);
1195         } else {
1196             mem_policy->Add(mem_policy, e, &mem->repl);
1197             debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n",
1198                 mem->url);
1199         }
1200         hot_obj_count++;
1201     } else {
1202         if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) {
1203             debug(20, 4) ("storeSetMemStatus: special entry %s\n",
1204                 mem->url);
1205         } else {
1206             mem_policy->Remove(mem_policy, e, &mem->repl);
1207             debug(20, 4) ("storeSetMemStatus: removed mem node %s\n",
1208                 mem->url);
1209         }
1210         hot_obj_count--;
1211     }
1212     e->mem_status = new_status;
1213 }
1214
1215 const char *
1216 storeUrl(const StoreEntry * e)
1217 {
1218     if (e == NULL)
1219         return "[null_entry]";
1220     else if (e->mem_obj == NULL)
1221         return "[null_mem_obj]";
1222     else
1223         return e->mem_obj->url;
1224 }
1225
1226 void
1227 storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url)
1228 {
1229     if (e->mem_obj)
1230         return;
1231     e->mem_obj = new_MemObject(url, log_url);
1232 }
1233
1234 /* this just sets DELAY_SENDING */
1235 void
1236 storeBuffer(StoreEntry * e)
1237 {
1238     EBIT_SET(e->flags, DELAY_SENDING);
1239 }
1240
1241 /* this just clears DELAY_SENDING and Invokes the handlers */
1242 void
1243 storeBufferFlush(StoreEntry * e)
1244 {
1245     EBIT_CLR(e->flags, DELAY_SENDING);
1246     InvokeHandlers(e);
1247     storeSwapOut(e);
1248 }
1249
1250 int
1251 objectLen(const StoreEntry * e)
1252 {
1253     assert(e->mem_obj != NULL);
1254     return e->mem_obj->object_sz;
1255 }
1256
1257 int
1258 contentLen(const StoreEntry * e)
1259 {
1260     assert(e->mem_obj != NULL);
1261     assert(e->mem_obj->reply != NULL);
1262     return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz;
1263 }
1264
1265 HttpReply *
1266 storeEntryReply(StoreEntry * e)
1267 {
1268     if (NULL == e)
1269         return NULL;
1270     if (NULL == e->mem_obj)
1271         return NULL;
1272     return e->mem_obj->reply;
1273 }
1274
1275 void
1276 storeEntryReset(StoreEntry * e)
1277 {
1278     MemObject *mem = e->mem_obj;
1279     debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e));
1280     assert(mem->swapout.sio == NULL);
1281     stmemFree(&mem->data_hdr);
1282     mem->inmem_hi = mem->inmem_lo = 0;
1283     httpReplyDestroy(mem->reply);
1284     mem->reply = httpReplyCreate();
1285     e->expires = e->lastmod = e->timestamp = -1;
1286 }
1287
1288 /*
1289  * storeFsInit
1290  *
1291  * This routine calls the SETUP routine for each fs type.
1292  * I don't know where the best place for this is, and I'm not going to shuffle
1293  * around large chunks of code right now (that can be done once its working.)
1294  */
1295 void
1296 storeFsInit(void)
1297 {
1298     storeReplSetup();
1299     storeFsSetup();
1300 }
1301
1302
1303 /*
1304  * similar to above, but is called when a graceful shutdown is to occur
1305  * of each fs module.
1306  */
1307 void
1308 storeFsDone(void)
1309 {
1310     int i = 0;
1311
1312     while (storefs_list[i].typestr != NULL) {
1313         storefs_list[i].donefunc();
1314         i++;
1315     }
1316 }
1317
1318 /*
1319  * called to add another store fs module
1320  */
1321 void
1322 storeFsAdd(const char *type, STSETUP * setup)
1323 {
1324     int i;
1325     /* find the number of currently known storefs types */
1326     for (i = 0; storefs_list && storefs_list[i].typestr; i++) {
1327         assert(strcmp(storefs_list[i].typestr, type) != 0);
1328     }
1329     /* add the new type */
1330     storefs_list = xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t));
1331     memset(&storefs_list[i + 1], 0, sizeof(storefs_entry_t));
1332     storefs_list[i].typestr = type;
1333     /* Call the FS to set up capabilities and initialize the FS driver */
1334     setup(&storefs_list[i]);
1335 }
1336
1337 /*
1338  * called to add another store removal policy module
1339  */
1340 void
1341 storeReplAdd(const char *type, REMOVALPOLICYCREATE * create)
1342 {
1343     int i;
1344     /* find the number of currently known repl types */
1345     for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) {
1346         assert(strcmp(storerepl_list[i].typestr, type) != 0);
1347     }
1348     /* add the new type */
1349     storerepl_list = xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t));
1350     memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t));
1351     storerepl_list[i].typestr = type;
1352     storerepl_list[i].create = create;
1353 }
1354
1355 /*
1356  * Create a removal policy instance
1357  */
1358 RemovalPolicy *
1359 createRemovalPolicy(RemovalPolicySettings * settings)
1360 {
1361     storerepl_entry_t *r;
1362     for (r = storerepl_list; r && r->typestr; r++) {
1363         if (strcmp(r->typestr, settings->type) == 0)
1364             return r->create(settings->args);
1365     }
1366     debug(20, 1) ("ERROR: Unknown policy %s\n", settings->type);
1367     debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n");
1368     debug(20, 1) ("ERROR:   and memory_replacement_policy in squid.conf!\n");
1369     fatalf("ERROR: Unknown policy %s\n", settings->type);
1370     return NULL;                /* NOTREACHED */
1371 }
1372
1373 #if 0
1374 void
1375 storeSwapFileNumberSet(StoreEntry * e, sfileno filn)
1376 {
1377     if (e->swap_file_number == filn)
1378         return;
1379     if (filn < 0) {
1380         assert(-1 == filn);
1381         storeDirMapBitReset(e->swap_file_number);
1382         storeDirLRUDelete(e);
1383         e->swap_file_number = -1;
1384     } else {
1385         assert(-1 == e->swap_file_number);
1386         storeDirMapBitSet(e->swap_file_number = filn);
1387         storeDirLRUAdd(e);
1388     }
1389 }
1390 #endif