3 * $Id: store.cc,v 1.520 2000/05/03 17:15:42 adrian Exp $
5 * DEBUG: section 20 Storage Manager
6 * AUTHOR: Harvest Derived
8 * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
9 * ----------------------------------------------------------
11 * Squid is the result of efforts by numerous individuals from the
12 * Internet community. Development is led by Duane Wessels of the
13 * National Laboratory for Applied Network Research and funded by the
14 * National Science Foundation. Squid is Copyrighted (C) 1998 by
15 * the Regents of the University of California. Please see the
16 * COPYRIGHT file for full details. Squid incorporates software
17 * developed and/or copyrighted by other sources. Please see the
18 * CREDITS file for full details.
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
38 #define REBUILD_TIMESTAMP_DELTA_MAX 2
40 #define STORE_IN_MEM_BUCKETS (229)
42 const char *memStatusStr
[] =
48 const char *pingStatusStr
[] =
55 const char *storeStatusStr
[] =
61 const char *swapStatusStr
[] =
68 typedef struct lock_ctrl_t
{
75 * local function prototypes
77 static int storeEntryValidLength(const StoreEntry
*);
78 static void storeGetMemSpace(int);
79 static void storeHashDelete(StoreEntry
*);
80 static MemObject
*new_MemObject(const char *, const char *);
81 static void destroy_MemObject(StoreEntry
*);
82 static FREE destroy_StoreEntry
;
83 static void storePurgeMem(StoreEntry
*);
84 static int getKeyCounter(void);
85 static int storeKeepInMemory(const StoreEntry
*);
86 static OBJH storeCheckCachableStats
;
87 static EVH storeLateRelease
;
94 * The heap equivalent of inmem_list, inmem_heap, is in globals.c so other
95 * modules can access it when updating object metadata (e.g., refcount)
98 static dlink_list inmem_list
;
100 int store_pages_max
= 0;
101 int store_swap_high
= 0;
102 int store_swap_low
= 0;
103 static Stack LateReleaseStack
;
105 #if URL_CHECKSUM_DEBUG
107 url_checksum(const char *url
)
111 static unsigned char digest
[16];
113 MD5Update(&M
, (unsigned char *) url
, strlen(url
));
114 MD5Final(digest
, &M
);
115 xmemcpy(&ck
, digest
, sizeof(ck
));
121 new_MemObject(const char *url
, const char *log_url
)
123 MemObject
*mem
= memAllocate(MEM_MEMOBJECT
);
124 mem
->reply
= httpReplyCreate();
125 mem
->url
= xstrdup(url
);
126 #if URL_CHECKSUM_DEBUG
127 mem
->chksum
= url_checksum(mem
->url
);
129 mem
->log_url
= xstrdup(log_url
);
132 /* XXX account log_url */
133 debug(20, 3) ("new_MemObject: returning %p\n", mem
);
138 new_StoreEntry(int mem_obj_flag
, const char *url
, const char *log_url
)
140 StoreEntry
*e
= NULL
;
141 e
= memAllocate(MEM_STOREENTRY
);
143 e
->mem_obj
= new_MemObject(url
, log_url
);
144 debug(20, 3) ("new_StoreEntry: returning %p\n", e
);
145 e
->expires
= e
->lastmod
= e
->lastref
= e
->timestamp
= -1;
152 destroy_MemObject(StoreEntry
* e
)
154 MemObject
*mem
= e
->mem_obj
;
155 const Ctx ctx
= ctx_enter(mem
->url
);
156 debug(20, 3) ("destroy_MemObject: destroying %p\n", mem
);
157 #if URL_CHECKSUM_DEBUG
158 assert(mem
->chksum
== url_checksum(mem
->url
));
162 assert(mem
->swapout
.sio
== NULL
);
163 stmemFree(&mem
->data_hdr
);
166 * There is no way to abort FD-less clients, so they might
167 * still have mem->clients set if mem->fd == -1
169 assert(mem
->fd
== -1 || mem
->clients
== NULL
);
170 httpReplyDestroy(mem
->reply
);
171 requestUnlink(mem
->request
);
173 ctx_exit(ctx
); /* must exit before we free mem->url */
175 safe_free(mem
->log_url
); /* XXX account log_url */
176 memFree(mem
, MEM_MEMOBJECT
);
180 destroy_StoreEntry(void *data
)
182 StoreEntry
*e
= data
;
183 debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e
);
186 destroy_MemObject(e
);
188 assert(e
->key
== NULL
);
189 memFree(e
, MEM_STOREENTRY
);
192 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
195 storeHashInsert(StoreEntry
* e
, const cache_key
* key
)
197 debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n",
198 e
, storeKeyText(key
));
199 e
->key
= storeKeyDup(key
);
200 hash_join(store_table
, (hash_link
*) e
);
204 storeHashDelete(StoreEntry
* e
)
206 hash_remove_link(store_table
, (hash_link
*) e
);
207 storeKeyFree(e
->key
);
211 /* -------------------------------------------------------------------------- */
214 /* get rid of memory copy of the object */
215 /* Only call this if storeCheckPurgeMem(e) returns 1 */
217 storePurgeMem(StoreEntry
* e
)
219 if (e
->mem_obj
== NULL
)
221 debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n",
222 storeKeyText(e
->key
));
223 storeSetMemStatus(e
, NOT_IN_MEMORY
);
224 destroy_MemObject(e
);
225 if (e
->swap_status
!= SWAPOUT_DONE
)
230 storeLockObject(StoreEntry
* e
)
234 if (e
->swap_dirn
> -1)
235 SD
= INDEXSD(e
->swap_dirn
);
240 debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
241 storeKeyText(e
->key
), (int) e
->lock_count
);
242 e
->lastref
= squid_curtime
;
243 /* Notify the fs that we're referencing this object again */
244 if (SD
!= NULL
&& SD
->refobj
!= NULL
)
249 storeReleaseRequest(StoreEntry
* e
)
251 if (EBIT_TEST(e
->flags
, RELEASE_REQUEST
))
253 debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e
->key
));
254 EBIT_SET(e
->flags
, RELEASE_REQUEST
);
256 * Clear cachable flag here because we might get called before
257 * anyone else even looks at the cachability flag. Also, this
258 * prevents httpMakePublic from really setting a public key.
260 EBIT_CLR(e
->flags
, ENTRY_CACHABLE
);
261 storeSetPrivateKey(e
);
264 /* unlock object, return -1 if object get released after unlock
265 * otherwise lock_count */
267 storeUnlockObject(StoreEntry
* e
)
271 debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
272 storeKeyText(e
->key
), e
->lock_count
);
274 return (int) e
->lock_count
;
275 if (e
->store_status
== STORE_PENDING
)
276 EBIT_SET(e
->flags
, RELEASE_REQUEST
);
277 assert(storePendingNClients(e
) == 0);
278 /* Notify the fs that we're not referencing this object any more */
279 if (e
->swap_filen
> -1)
280 SD
= INDEXSD(e
->swap_dirn
);
283 if (SD
!= NULL
&& SD
->unrefobj
!= NULL
)
286 storeHeapPositionUpdate(e
, SD
);
289 /* Note: From 2.4. Not sure how this relates to the unrefobj() call above */
290 storeDirLRUDelete(e
);
294 if (EBIT_TEST(e
->flags
, RELEASE_REQUEST
))
296 else if (storeKeepInMemory(e
)) {
297 storeSetMemStatus(e
, IN_MEMORY
);
298 requestUnlink(e
->mem_obj
->request
);
299 e
->mem_obj
->request
= NULL
;
302 if (EBIT_TEST(e
->flags
, KEY_PRIVATE
))
303 debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__
, __LINE__
);
308 /* Lookup an object in the cache.
309 * return just a reference to object, don't start swapping in yet. */
311 storeGet(const cache_key
* key
)
313 debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key
));
314 return (StoreEntry
*) hash_lookup(store_table
, key
);
318 storeGetPublic(const char *uri
, const method_t method
)
320 return storeGet(storeKeyPublic(uri
, method
));
326 static int key_counter
= 0;
327 if (++key_counter
< 0)
333 storeSetPrivateKey(StoreEntry
* e
)
335 const cache_key
*newkey
;
336 MemObject
*mem
= e
->mem_obj
;
337 if (e
->key
&& EBIT_TEST(e
->flags
, KEY_PRIVATE
))
338 return; /* is already private */
340 if (e
->swap_filen
> -1)
341 storeDirSwapLog(e
, SWAP_LOG_DEL
);
345 mem
->id
= getKeyCounter();
346 newkey
= storeKeyPrivate(mem
->url
, mem
->method
, mem
->id
);
348 newkey
= storeKeyPrivate("JUNK", METHOD_NONE
, getKeyCounter());
350 assert(hash_lookup(store_table
, newkey
) == NULL
);
351 EBIT_SET(e
->flags
, KEY_PRIVATE
);
352 storeHashInsert(e
, newkey
);
356 storeSetPublicKey(StoreEntry
* e
)
358 StoreEntry
*e2
= NULL
;
359 const cache_key
*newkey
;
360 MemObject
*mem
= e
->mem_obj
;
361 if (e
->key
&& !EBIT_TEST(e
->flags
, KEY_PRIVATE
))
362 return; /* is already public */
365 * We can't make RELEASE_REQUEST objects public. Depending on
366 * when RELEASE_REQUEST gets set, we might not be swapping out
367 * the object. If we're not swapping out, then subsequent
368 * store clients won't be able to access object data which has
369 * been freed from memory.
371 * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
372 * be set, and storeSetPublicKey() should not be called.
375 if (EBIT_TEST(e
->flags
, RELEASE_REQUEST
))
376 debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n",
379 assert(!EBIT_TEST(e
->flags
, RELEASE_REQUEST
));
380 newkey
= storeKeyPublic(mem
->url
, mem
->method
);
381 if ((e2
= (StoreEntry
*) hash_lookup(store_table
, newkey
))) {
382 debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem
->url
);
383 storeSetPrivateKey(e2
);
385 newkey
= storeKeyPublic(mem
->url
, mem
->method
);
389 EBIT_CLR(e
->flags
, KEY_PRIVATE
);
390 storeHashInsert(e
, newkey
);
391 if (e
->swap_filen
> -1)
392 storeDirSwapLog(e
, SWAP_LOG_ADD
);
396 storeCreateEntry(const char *url
, const char *log_url
, request_flags flags
, method_t method
)
398 StoreEntry
*e
= NULL
;
399 MemObject
*mem
= NULL
;
400 debug(20, 3) ("storeCreateEntry: '%s'\n", url
);
402 e
= new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ
, url
, log_url
);
403 e
->lock_count
= 1; /* Note lock here w/o calling storeLock() */
405 mem
->method
= method
;
406 if (neighbors_do_private_keys
|| !flags
.hierarchical
)
407 storeSetPrivateKey(e
);
409 storeSetPublicKey(e
);
410 if (flags
.cachable
) {
411 EBIT_SET(e
->flags
, ENTRY_CACHABLE
);
412 EBIT_CLR(e
->flags
, RELEASE_REQUEST
);
414 EBIT_CLR(e
->flags
, ENTRY_CACHABLE
);
415 storeReleaseRequest(e
);
417 e
->store_status
= STORE_PENDING
;
418 storeSetMemStatus(e
, NOT_IN_MEMORY
);
419 e
->swap_status
= SWAPOUT_NONE
;
423 e
->lastref
= squid_curtime
;
424 e
->timestamp
= 0; /* set in storeTimestampsSet() */
425 e
->ping_status
= PING_NONE
;
426 EBIT_SET(e
->flags
, ENTRY_VALIDATED
);
430 /* Mark object as expired */
432 storeExpireNow(StoreEntry
* e
)
434 debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e
->key
));
435 e
->expires
= squid_curtime
;
438 /* Append incoming data from a primary server to an entry. */
440 storeAppend(StoreEntry
* e
, const char *buf
, int len
)
442 MemObject
*mem
= e
->mem_obj
;
445 assert(e
->store_status
== STORE_PENDING
);
447 debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
449 storeKeyText(e
->key
));
450 storeGetMemSpace(len
);
451 stmemAppend(&mem
->data_hdr
, buf
, len
);
452 mem
->inmem_hi
+= len
;
454 if (EBIT_TEST(e
->flags
, DELAY_SENDING
))
462 storeAppendPrintf(StoreEntry
* e
, const char *fmt
,...)
464 storeAppendPrintf(va_alist
)
473 StoreEntry
*e
= NULL
;
474 const char *fmt
= NULL
;
476 e
= va_arg(args
, StoreEntry
*);
477 fmt
= va_arg(args
, char *);
479 storeAppendVPrintf(e
, fmt
, args
);
483 /* used be storeAppendPrintf and Packer */
485 storeAppendVPrintf(StoreEntry
* e
, const char *fmt
, va_list vargs
)
487 LOCAL_ARRAY(char, buf
, 4096);
489 vsnprintf(buf
, 4096, fmt
, vargs
);
490 storeAppend(e
, buf
, strlen(buf
));
493 struct _store_check_cachable_hist
{
496 int not_entry_cachable
;
498 int wrong_content_length
;
502 int too_many_open_files
;
503 int too_many_open_fds
;
508 } store_check_cachable_hist
;
511 storeTooManyDiskFilesOpen(void)
513 if (Config
.max_open_disk_fds
== 0)
515 if (store_open_disk_fd
> Config
.max_open_disk_fds
)
521 storeCheckCachable(StoreEntry
* e
)
523 #if CACHE_ALL_METHODS
524 if (e
->mem_obj
->method
!= METHOD_GET
) {
525 debug(20, 2) ("storeCheckCachable: NO: non-GET method\n");
526 store_check_cachable_hist
.no
.non_get
++;
529 if (!EBIT_TEST(e
->flags
, ENTRY_CACHABLE
)) {
530 debug(20, 2) ("storeCheckCachable: NO: not cachable\n");
531 store_check_cachable_hist
.no
.not_entry_cachable
++;
532 } else if (EBIT_TEST(e
->flags
, RELEASE_REQUEST
)) {
533 debug(20, 2) ("storeCheckCachable: NO: release requested\n");
534 store_check_cachable_hist
.no
.release_request
++;
535 } else if (e
->store_status
== STORE_OK
&& EBIT_TEST(e
->flags
, ENTRY_BAD_LENGTH
)) {
536 debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n");
537 store_check_cachable_hist
.no
.wrong_content_length
++;
538 } else if (EBIT_TEST(e
->flags
, ENTRY_NEGCACHED
)) {
539 debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
540 store_check_cachable_hist
.no
.negative_cached
++;
541 return 0; /* avoid release call below */
542 } else if ((e
->mem_obj
->reply
->content_length
> 0 &&
543 e
->mem_obj
->reply
->content_length
> Config
.Store
.maxObjectSize
) ||
544 e
->mem_obj
->inmem_hi
> Config
.Store
.maxObjectSize
) {
545 debug(20, 2) ("storeCheckCachable: NO: too big\n");
546 store_check_cachable_hist
.no
.too_big
++;
547 } else if (e
->mem_obj
->reply
->content_length
> (int) Config
.Store
.maxObjectSize
) {
548 debug(20, 2) ("storeCheckCachable: NO: too big\n");
549 store_check_cachable_hist
.no
.too_big
++;
550 } else if (EBIT_TEST(e
->flags
, KEY_PRIVATE
)) {
551 debug(20, 3) ("storeCheckCachable: NO: private key\n");
552 store_check_cachable_hist
.no
.private_key
++;
553 } else if (e
->swap_status
!= SWAPOUT_NONE
) {
555 * here we checked the swap_status because the remaining
556 * cases are only relevant only if we haven't started swapping
557 * out the object yet.
560 } else if (storeTooManyDiskFilesOpen()) {
561 debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n");
562 store_check_cachable_hist
.no
.too_many_open_files
++;
563 } else if (fdNFree() < RESERVED_FD
) {
564 debug(20, 2) ("storeCheckCachable: NO: too many FD's open\n");
565 store_check_cachable_hist
.no
.too_many_open_fds
++;
567 store_check_cachable_hist
.yes
.Default
++;
570 storeReleaseRequest(e
);
571 EBIT_CLR(e
->flags
, ENTRY_CACHABLE
);
576 storeCheckCachableStats(StoreEntry
* sentry
)
578 storeAppendPrintf(sentry
, "Category\t Count\n");
580 storeAppendPrintf(sentry
, "no.non_get\t%d\n",
581 store_check_cachable_hist
.no
.non_get
);
582 storeAppendPrintf(sentry
, "no.not_entry_cachable\t%d\n",
583 store_check_cachable_hist
.no
.not_entry_cachable
);
584 storeAppendPrintf(sentry
, "no.release_request\t%d\n",
585 store_check_cachable_hist
.no
.release_request
);
586 storeAppendPrintf(sentry
, "no.wrong_content_length\t%d\n",
587 store_check_cachable_hist
.no
.wrong_content_length
);
588 storeAppendPrintf(sentry
, "no.negative_cached\t%d\n",
589 store_check_cachable_hist
.no
.negative_cached
);
590 storeAppendPrintf(sentry
, "no.too_big\t%d\n",
591 store_check_cachable_hist
.no
.too_big
);
592 storeAppendPrintf(sentry
, "no.private_key\t%d\n",
593 store_check_cachable_hist
.no
.private_key
);
594 storeAppendPrintf(sentry
, "no.too_many_open_files\t%d\n",
595 store_check_cachable_hist
.no
.too_many_open_files
);
596 storeAppendPrintf(sentry
, "no.too_many_open_fds\t%d\n",
597 store_check_cachable_hist
.no
.too_many_open_fds
);
598 storeAppendPrintf(sentry
, "yes.default\t%d\n",
599 store_check_cachable_hist
.yes
.Default
);
602 /* Complete transfer into the local cache. */
604 storeComplete(StoreEntry
* e
)
606 debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e
->key
));
607 if (e
->store_status
!= STORE_PENDING
) {
609 * if we're not STORE_PENDING, then probably we got aborted
610 * and there should be NO clients on this entry
612 assert(EBIT_TEST(e
->flags
, ENTRY_ABORTED
));
613 assert(e
->mem_obj
->nclients
== 0);
616 e
->mem_obj
->object_sz
= e
->mem_obj
->inmem_hi
;
617 e
->store_status
= STORE_OK
;
618 assert(e
->mem_status
== NOT_IN_MEMORY
);
619 if (!storeEntryValidLength(e
)) {
620 EBIT_SET(e
->flags
, ENTRY_BAD_LENGTH
);
621 storeReleaseRequest(e
);
623 #if USE_CACHE_DIGESTS
624 if (e
->mem_obj
->request
)
625 e
->mem_obj
->request
->hier
.store_complete_stop
= current_time
;
632 * Someone wants to abort this transfer. Set the reason in the
633 * request structure, call the server-side callback and mark the
634 * entry for releasing
637 storeAbort(StoreEntry
* e
)
639 MemObject
*mem
= e
->mem_obj
;
640 assert(e
->store_status
== STORE_PENDING
);
642 debug(20, 6) ("storeAbort: %s\n", storeKeyText(e
->key
));
643 storeLockObject(e
); /* lock while aborting */
644 storeNegativeCache(e
);
645 storeReleaseRequest(e
);
646 EBIT_SET(e
->flags
, ENTRY_ABORTED
);
647 storeSetMemStatus(e
, NOT_IN_MEMORY
);
648 e
->store_status
= STORE_OK
;
650 * We assign an object length here. The only other place we assign
651 * the object length is in storeComplete()
653 mem
->object_sz
= mem
->inmem_hi
;
654 /* Notify the server side */
655 if (mem
->abort
.callback
) {
656 eventAdd("mem->abort.callback",
661 mem
->abort
.callback
= NULL
;
662 mem
->abort
.data
= NULL
;
664 /* Notify the client side */
666 /* Close any swapout file */
667 storeSwapOutFileClose(e
);
668 storeUnlockObject(e
); /* unlock */
671 /* Clear Memory storage to accommodate the given object len */
673 storeGetMemSpace(int size
)
675 StoreEntry
*e
= NULL
;
677 static time_t last_check
= 0;
680 #if !HEAP_REPLACEMENT
683 dlink_node
*prev
= NULL
;
686 heap_key min_age
= 0.0;
687 link_list
*locked_entries
= NULL
;
689 if (squid_curtime
== last_check
)
691 last_check
= squid_curtime
;
692 pages_needed
= (size
/ SM_PAGE_SIZE
) + 1;
693 if (memInUse(MEM_STMEM_BUF
) + pages_needed
< store_pages_max
)
695 debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed
);
697 while (heap_nodes(inmem_heap
) > 0) {
698 age
= heap_peepminkey(inmem_heap
);
699 e
= heap_extractmin(inmem_heap
);
700 e
->mem_obj
->node
= NULL
; /* no longer in the heap */
701 if (storeEntryLocked(e
)) {
703 debug(20, 5) ("storeGetMemSpace: locked key %s\n",
704 storeKeyText(e
->key
));
705 linklistPush(&locked_entries
, e
);
709 debug(20, 3) ("Released memory object with key %f size %d refs %d url %s\n",
710 age
, e
->swap_file_sz
, e
->refcount
, e
->mem_obj
->url
);
713 if (memInUse(MEM_STMEM_BUF
) + pages_needed
< store_pages_max
)
717 * Increase the heap age factor.
720 inmem_heap
->age
= min_age
;
722 * Reinsert all bumped locked entries back into heap...
724 while ((e
= linklistShift(&locked_entries
)))
725 e
->mem_obj
->node
= heap_insert(inmem_heap
, e
);
727 head
= inmem_list
.head
;
728 for (m
= inmem_list
.tail
; m
; m
= prev
) {
733 if (storeEntryLocked(e
)) {
735 dlinkDelete(m
, &inmem_list
);
736 dlinkAdd(e
, m
, &inmem_list
);
741 if (memInUse(MEM_STMEM_BUF
) + pages_needed
< store_pages_max
)
745 debug(20, 3) ("storeGetMemSpace: released %d/%d locked %d\n",
746 released
, hot_obj_count
, locked
);
747 debug(20, 3) ("storeGetMemSpace stats:\n");
748 debug(20, 3) (" %6d HOT objects\n", hot_obj_count
);
749 debug(20, 3) (" %6d were released\n", released
);
752 /* The maximum objects to scan for maintain storage space */
753 #define MAINTAIN_MAX_SCAN 1024
754 #define MAINTAIN_MAX_REMOVE 64
757 * This routine is to be called by main loop in main.c.
758 * It removes expired objects on only one bucket for each time called.
759 * returns the number of objects removed
761 * This should get called 1/s from main().
764 storeMaintainSwapSpace(void *datanotused
)
770 for (i
= 0; i
< Config
.cacheSwap
.n_configured
; i
++) {
771 /* call the maintain function .. */
773 if (SD
->maintainfs
!= NULL
)
777 /* Reregister a maintain event .. */
778 eventAdd("MaintainSwapSpace", storeMaintainSwapSpace
, NULL
, 1.0, 1);
782 /* release an object from a cache */
784 storeRelease(StoreEntry
* e
)
786 debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e
->key
));
787 /* If, for any reason we can't discard this object because of an
788 * outstanding request, mark it for pending release */
789 if (storeEntryLocked(e
)) {
791 debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n");
792 storeReleaseRequest(e
);
795 if (store_dirs_rebuilding
&& e
->swap_filen
> -1) {
796 storeSetPrivateKey(e
);
798 storeSetMemStatus(e
, NOT_IN_MEMORY
);
799 destroy_MemObject(e
);
801 if (e
->swap_filen
> -1) {
803 * Fake a call to storeLockObject(). When rebuilding is done,
804 * we'll just call storeUnlockObject() on these.
807 EBIT_SET(e
->flags
, RELEASE_REQUEST
);
808 stackPush(&LateReleaseStack
, e
);
811 destroy_StoreEntry(e
);
814 storeLog(STORE_LOG_RELEASE
, e
);
815 if (e
->swap_filen
> -1) {
817 if (e
->swap_status
== SWAPOUT_DONE
)
818 if (EBIT_TEST(e
->flags
, ENTRY_VALIDATED
))
819 storeDirUpdateSwapSize(&Config
.cacheSwap
.swapDirs
[e
->swap_dirn
], e
->swap_file_sz
, -1);
820 if (!EBIT_TEST(e
->flags
, KEY_PRIVATE
))
821 storeDirSwapLog(e
, SWAP_LOG_DEL
);
823 /* From 2.4. I think we do this in storeUnlink? */
824 storeSwapFileNumberSet(e
, -1);
827 storeSetMemStatus(e
, NOT_IN_MEMORY
);
828 destroy_StoreEntry(e
);
832 storeLateRelease(void *unused
)
837 if (store_dirs_rebuilding
) {
838 eventAdd("storeLateRelease", storeLateRelease
, NULL
, 1.0, 1);
841 for (i
= 0; i
< 10; i
++) {
842 e
= stackPop(&LateReleaseStack
);
845 debug(20, 1) ("storeLateRelease: released %d objects\n", n
);
848 storeUnlockObject(e
);
851 eventAdd("storeLateRelease", storeLateRelease
, NULL
, 0.0, 1);
854 /* return 1 if a store entry is locked */
856 storeEntryLocked(const StoreEntry
* e
)
860 if (e
->swap_status
== SWAPOUT_WRITING
)
862 if (e
->store_status
== STORE_PENDING
)
865 * SPECIAL, PUBLIC entries should be "locked"
867 if (EBIT_TEST(e
->flags
, ENTRY_SPECIAL
))
868 if (!EBIT_TEST(e
->flags
, KEY_PRIVATE
))
874 storeEntryValidLength(const StoreEntry
* e
)
877 const HttpReply
*reply
;
878 assert(e
->mem_obj
!= NULL
);
879 reply
= e
->mem_obj
->reply
;
880 debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e
->key
));
881 debug(20, 5) ("storeEntryValidLength: object_len = %d\n",
883 debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n",
885 debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
886 reply
->content_length
);
887 if (reply
->content_length
< 0) {
888 debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
889 storeKeyText(e
->key
));
892 if (reply
->hdr_sz
== 0) {
893 debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n",
894 storeKeyText(e
->key
));
897 if (e
->mem_obj
->method
== METHOD_HEAD
) {
898 debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n",
899 storeKeyText(e
->key
));
902 if (reply
->sline
.status
== HTTP_NOT_MODIFIED
)
904 if (reply
->sline
.status
== HTTP_NO_CONTENT
)
906 diff
= reply
->hdr_sz
+ reply
->content_length
- objectLen(e
);
909 debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
910 diff
< 0 ? -diff
: diff
,
911 diff
< 0 ? "big" : "small",
912 storeKeyText(e
->key
));
917 storeInitHashValues(void)
920 /* Calculate size of hash table (maximum currently 64k buckets). */
921 i
= Config
.Swap
.maxSize
/ Config
.Store
.avgObjectSize
;
922 debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n",
923 Config
.Swap
.maxSize
, i
);
924 i
/= Config
.Store
.objectsPerBucket
;
925 debug(20, 1) ("Target number of buckets: %d\n", i
);
926 /* ideally the full scan period should be configurable, for the
927 * moment it remains at approximately 24 hours. */
928 store_hash_buckets
= storeKeyHashBuckets(i
);
929 debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets
);
930 debug(20, 1) ("Max Mem size: %d KB\n", Config
.memMaxSize
>> 10);
931 debug(20, 1) ("Max Swap size: %d KB\n", Config
.Swap
.maxSize
);
938 storeInitHashValues();
939 store_table
= hash_create(storeKeyHashCmp
,
940 store_hash_buckets
, storeKeyHashHash
);
944 inmem_heap
= new_heap(1000, HeapKeyGen_StoreEntry_GDSF
);
946 stackInit(&LateReleaseStack
);
947 eventAdd("storeLateRelease", storeLateRelease
, NULL
, 1.0, 1);
950 cachemgrRegister("storedir",
951 "Store Directory Stats",
952 storeDirStats
, 0, 1);
953 cachemgrRegister("store_check_cachable_stats",
954 "storeCheckCachable() Stats",
955 storeCheckCachableStats
, 0, 1);
961 store_swap_high
= (long) (((float) Config
.Swap
.maxSize
*
962 (float) Config
.Swap
.highWaterMark
) / (float) 100);
963 store_swap_low
= (long) (((float) Config
.Swap
.maxSize
*
964 (float) Config
.Swap
.lowWaterMark
) / (float) 100);
965 store_pages_max
= Config
.memMaxSize
/ SM_PAGE_SIZE
;
969 storeKeepInMemory(const StoreEntry
* e
)
971 MemObject
*mem
= e
->mem_obj
;
974 if (mem
->data_hdr
.head
== NULL
)
976 return mem
->inmem_lo
== 0;
980 storeNegativeCache(StoreEntry
* e
)
982 e
->expires
= squid_curtime
+ Config
.negativeTtl
;
983 EBIT_SET(e
->flags
, ENTRY_NEGCACHED
);
987 storeFreeMemory(void)
989 hashFreeItems(store_table
, destroy_StoreEntry
);
990 hashFreeMemory(store_table
);
992 #if USE_CACHE_DIGESTS
994 cacheDigestDestroy(store_digest
);
1000 expiresMoreThan(time_t expires
, time_t when
)
1002 if (expires
< 0) /* No Expires given */
1004 return (expires
> (squid_curtime
+ when
));
1008 storeEntryValidToSend(StoreEntry
* e
)
1010 if (EBIT_TEST(e
->flags
, RELEASE_REQUEST
))
1012 if (EBIT_TEST(e
->flags
, ENTRY_NEGCACHED
))
1013 if (e
->expires
<= squid_curtime
)
1015 if (EBIT_TEST(e
->flags
, ENTRY_ABORTED
))
1021 storeTimestampsSet(StoreEntry
* entry
)
1023 const HttpReply
*reply
= entry
->mem_obj
->reply
;
1024 time_t served_date
= reply
->date
;
1025 int age
= httpHeaderGetInt(&reply
->header
, HDR_AGE
);
1027 * The timestamp calculations below tries to mimic the properties
1028 * of the age calculation in RFC2616 section 13.2.3. The implementaion
1029 * isn't complete, and the most notable exception from the RFC is that
1030 * this does not account for response_delay, but it probably does
1031 * not matter much as this is calculated immediately when the headers
1032 * are received, not when the whole response has been received.
1034 /* make sure that 0 <= served_date <= squid_curtime */
1035 if (served_date
< 0 || served_date
> squid_curtime
)
1036 served_date
= squid_curtime
;
1038 * Compensate with Age header if origin server clock is ahead of us
1039 * and there is a cache in between us and the origin server
1041 if (age
> squid_curtime
- served_date
)
1042 served_date
= squid_curtime
- age
;
1043 entry
->expires
= reply
->expires
;
1044 entry
->lastmod
= reply
->last_modified
;
1045 entry
->timestamp
= served_date
;
1049 storeRegisterAbort(StoreEntry
* e
, STABH
* cb
, void *data
)
1051 MemObject
*mem
= e
->mem_obj
;
1053 assert(mem
->abort
.callback
== NULL
);
1054 mem
->abort
.callback
= cb
;
1055 mem
->abort
.data
= data
;
1059 storeUnregisterAbort(StoreEntry
* e
)
1061 MemObject
*mem
= e
->mem_obj
;
1063 mem
->abort
.callback
= NULL
;
1067 storeMemObjectDump(MemObject
* mem
)
1069 debug(20, 1) ("MemObject->data.head: %p\n",
1070 mem
->data_hdr
.head
);
1071 debug(20, 1) ("MemObject->data.tail: %p\n",
1072 mem
->data_hdr
.tail
);
1073 debug(20, 1) ("MemObject->data.origin_offset: %d\n",
1074 mem
->data_hdr
.origin_offset
);
1075 debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
1076 (int) mem
->start_ping
.tv_sec
,
1077 (int) mem
->start_ping
.tv_usec
);
1078 debug(20, 1) ("MemObject->inmem_hi: %d\n",
1079 (int) mem
->inmem_hi
);
1080 debug(20, 1) ("MemObject->inmem_lo: %d\n",
1081 (int) mem
->inmem_lo
);
1082 debug(20, 1) ("MemObject->clients: %p\n",
1084 debug(20, 1) ("MemObject->nclients: %d\n",
1086 debug(20, 1) ("MemObject->reply: %p\n",
1088 debug(20, 1) ("MemObject->request: %p\n",
1090 debug(20, 1) ("MemObject->log_url: %p %s\n",
1092 checkNullString(mem
->log_url
));
1096 storeEntryDump(const StoreEntry
* e
, int l
)
1098 debug(20, l
) ("StoreEntry->key: %s\n", storeKeyText(e
->key
));
1099 debug(20, l
) ("StoreEntry->next: %p\n", e
->next
);
1100 debug(20, l
) ("StoreEntry->mem_obj: %p\n", e
->mem_obj
);
1101 debug(20, l
) ("StoreEntry->timestamp: %d\n", (int) e
->timestamp
);
1102 debug(20, l
) ("StoreEntry->lastref: %d\n", (int) e
->lastref
);
1103 debug(20, l
) ("StoreEntry->expires: %d\n", (int) e
->expires
);
1104 debug(20, l
) ("StoreEntry->lastmod: %d\n", (int) e
->lastmod
);
1105 debug(20, l
) ("StoreEntry->swap_file_sz: %d\n", (int) e
->swap_file_sz
);
1106 debug(20, l
) ("StoreEntry->refcount: %d\n", e
->refcount
);
1107 debug(20, l
) ("StoreEntry->flags: %s\n", storeEntryFlags(e
));
1108 debug(20, l
) ("StoreEntry->swap_dirn: %d\n", (int) e
->swap_dirn
);
1109 debug(20, l
) ("StoreEntry->swap_filen: %d\n", (int) e
->swap_filen
);
1110 debug(20, l
) ("StoreEntry->lock_count: %d\n", (int) e
->lock_count
);
1111 debug(20, l
) ("StoreEntry->mem_status: %d\n", (int) e
->mem_status
);
1112 debug(20, l
) ("StoreEntry->ping_status: %d\n", (int) e
->ping_status
);
1113 debug(20, l
) ("StoreEntry->store_status: %d\n", (int) e
->store_status
);
1114 debug(20, l
) ("StoreEntry->swap_status: %d\n", (int) e
->swap_status
);
1118 * NOTE, this function assumes only two mem states
1121 storeSetMemStatus(StoreEntry
* e
, int new_status
)
1123 MemObject
*mem
= e
->mem_obj
;
1124 if (new_status
== e
->mem_status
)
1126 assert(mem
!= NULL
);
1127 if (new_status
== IN_MEMORY
) {
1128 assert(mem
->inmem_lo
== 0);
1129 #if HEAP_REPLACEMENT
1130 if (mem
->node
== NULL
) {
1131 if (EBIT_TEST(e
->flags
, ENTRY_SPECIAL
)) {
1132 debug(20, 4) ("storeSetMemStatus: not inserting special %s\n",
1135 mem
->node
= heap_insert(inmem_heap
, e
);
1136 debug(20, 4) ("storeSetMemStatus: inserted mem node %p\n",
1141 dlinkAdd(e
, &mem
->lru
, &inmem_list
);
1145 #if HEAP_REPLACEMENT
1147 * It's being removed from the memory heap; is it already gone?
1150 heap_delete(inmem_heap
, mem
->node
);
1151 debug(20, 4) ("storeSetMemStatus: deleted mem node %p\n",
1156 dlinkDelete(&mem
->lru
, &inmem_list
);
1160 e
->mem_status
= new_status
;
1164 storeUrl(const StoreEntry
* e
)
1167 return "[null_entry]";
1168 else if (e
->mem_obj
== NULL
)
1169 return "[null_mem_obj]";
1171 return e
->mem_obj
->url
;
1175 storeCreateMemObject(StoreEntry
* e
, const char *url
, const char *log_url
)
1179 e
->mem_obj
= new_MemObject(url
, log_url
);
1182 /* this just sets DELAY_SENDING */
1184 storeBuffer(StoreEntry
* e
)
1186 EBIT_SET(e
->flags
, DELAY_SENDING
);
1189 /* this just clears DELAY_SENDING and Invokes the handlers */
1191 storeBufferFlush(StoreEntry
* e
)
1193 EBIT_CLR(e
->flags
, DELAY_SENDING
);
1199 objectLen(const StoreEntry
* e
)
1201 assert(e
->mem_obj
!= NULL
);
1202 return e
->mem_obj
->object_sz
;
1206 contentLen(const StoreEntry
* e
)
1208 assert(e
->mem_obj
!= NULL
);
1209 assert(e
->mem_obj
->reply
!= NULL
);
1210 return e
->mem_obj
->object_sz
- e
->mem_obj
->reply
->hdr_sz
;
1214 storeEntryReply(StoreEntry
* e
)
1218 if (NULL
== e
->mem_obj
)
1220 return e
->mem_obj
->reply
;
1224 storeEntryReset(StoreEntry
* e
)
1226 MemObject
*mem
= e
->mem_obj
;
1227 debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e
));
1228 assert(mem
->swapout
.sio
== NULL
);
1229 stmemFree(&mem
->data_hdr
);
1230 mem
->inmem_hi
= mem
->inmem_lo
= 0;
1231 httpReplyDestroy(mem
->reply
);
1232 mem
->reply
= httpReplyCreate();
1233 e
->expires
= e
->lastmod
= e
->timestamp
= -1;
1236 #if HEAP_REPLACEMENT
1238 * This routine only handles memory updates these days
1241 storeHeapPositionUpdate(StoreEntry
* e
, SwapDir
* SD
)
1243 if (e
->mem_obj
&& e
->mem_obj
->node
)
1244 heap_update(inmem_heap
, e
->mem_obj
->node
, e
);
1251 * This routine calls the SETUP routine for each fs type.
1252 * I don't know where the best place for this is, and I'm not going to shuffle
1253 * around large chunks of code right now (that can be done once its working.)
1263 * similar to above, but is called when a graceful shutdown is to occur
1264 * of each fs module.
1271 while (storefs_list
[i
].typestr
!= NULL
) {
1272 storefs_list
[i
].donefunc();
1278 * called to add another store fs module
1280 void storeFsAdd(char *type
, STSETUP
*setup
)
1283 /* find the number of currently known storefs types */
1284 for (i
= 0; storefs_list
&& storefs_list
[i
].typestr
; i
++) {
1285 assert(strcmp(storefs_list
[i
].typestr
, type
)!=0);
1287 /* add the new type */
1288 storefs_list
= xrealloc(storefs_list
, (i
+ 2) * sizeof(storefs_entry_t
));
1289 memset(&storefs_list
[i
+1],0,sizeof(storefs_entry_t
));
1290 storefs_list
[i
].typestr
= type
;
1291 /* Call the FS to set up capabilities and initialize the FS driver */
1292 setup(&storefs_list
[i
]);
1297 storeSwapFileNumberSet(StoreEntry
* e
, sfileno filn
)
1299 if (e
->swap_file_number
== filn
)
1303 storeDirMapBitReset(e
->swap_file_number
);
1304 storeDirLRUDelete(e
);
1305 e
->swap_file_number
= -1;
1307 assert(-1 == e
->swap_file_number
);
1308 storeDirMapBitSet(e
->swap_file_number
= filn
);