]> git.ipfire.org Git - thirdparty/squid.git/blob - src/store.cc
projects / thirdparty / squid.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
From: Henrik Nordstrom <hno@hem.passagen.se>
[thirdparty/squid.git] / src / store.cc
1
2 /*
3 * $Id: store.cc,v 1.486 1999/01/21 23:15:40 wessels Exp $
4 *
5 * DEBUG: section 20 Storage Manager
6 * AUTHOR: Harvest Derived
7 *
8 * SQUID Internet Object Cache http://squid.nlanr.net/Squid/
9 * ----------------------------------------------------------
10 *
11 * Squid is the result of efforts by numerous individuals from the
12 * Internet community. Development is led by Duane Wessels of the
13 * National Laboratory for Applied Network Research and funded by the
14 * National Science Foundation. Squid is Copyrighted (C) 1998 by
15 * Duane Wessels and the University of California San Diego. Please
16 * see the COPYRIGHT file for full details. Squid incorporates
17 * software developed and/or copyrighted by other sources. Please see
18 * the CREDITS file for full details.
19 *
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
24 *
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
29 *
30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
33 *
34 */
35
36 #include "squid.h"
37
38 #define REBUILD_TIMESTAMP_DELTA_MAX 2
39
40 #define STORE_IN_MEM_BUCKETS (229)
41
42 const char *memStatusStr[] =
43 {
44 "NOT_IN_MEMORY",
45 "IN_MEMORY"
46 };
47
48 const char *pingStatusStr[] =
49 {
50 "PING_NONE",
51 "PING_WAITING",
52 "PING_TIMEOUT",
53 "PING_DONE"
54 };
55
56 const char *storeStatusStr[] =
57 {
58 "STORE_OK",
59 "STORE_PENDING"
60 };
61
62 const char *swapStatusStr[] =
63 {
64 "SWAPOUT_NONE",
65 "SWAPOUT_OPENING",
66 "SWAPOUT_WRITING",
67 "SWAPOUT_DONE"
68 };
69
70 typedef struct lock_ctrl_t {
71 SIH *callback;
72 void *callback_data;
73 StoreEntry *e;
74 } lock_ctrl_t;
75
76 /*
77 * local function prototypes
78 */
79 static int storeCheckExpired(const StoreEntry *);
80 static int storeEntryLocked(const StoreEntry *);
81 static int storeEntryValidLength(const StoreEntry *);
82 static void storeGetMemSpace(int);
83 static void storeHashDelete(StoreEntry *);
84 static MemObject *new_MemObject(const char *, const char *);
85 static void destroy_MemObject(StoreEntry *);
86 static FREE destroy_StoreEntry;
87 static void storePurgeMem(StoreEntry *);
88 static int getKeyCounter(void);
89 static int storeKeepInMemory(const StoreEntry *);
90 static OBJH storeCheckCachableStats;
91 static EVH storeLateRelease;
92
93 /*
94 * local variables
95 */
96 static dlink_list inmem_list;
97 static int store_pages_max = 0;
98 static int store_swap_high = 0;
99 static int store_swap_low = 0;
100 static int store_swap_mid = 0;
101 static int store_maintain_rate;
102 static Stack LateReleaseStack;
103
104 static MemObject *
105 new_MemObject(const char *url, const char *log_url)
106 {
107 MemObject *mem = memAllocate(MEM_MEMOBJECT);
108 mem->reply = httpReplyCreate();
109 mem->url = xstrdup(url);
110 mem->log_url = xstrdup(log_url);
111 mem->swapout.fd = -1;
112 mem->object_sz = -1;
113 mem->fd = -1;
114 /* XXX account log_url */
115 debug(20, 3) ("new_MemObject: returning %p\n", mem);
116 return mem;
117 }
118
119 StoreEntry *
120 new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url)
121 {
122 StoreEntry *e = NULL;
123 e = memAllocate(MEM_STOREENTRY);
124 if (mem_obj_flag)
125 e->mem_obj = new_MemObject(url, log_url);
126 debug(20, 3) ("new_StoreEntry: returning %p\n", e);
127 e->expires = e->lastmod = e->lastref = e->timestamp = -1;
128 return e;
129 }
130
131 static void
132 destroy_MemObject(StoreEntry * e)
133 {
134 MemObject *mem = e->mem_obj;
135 const Ctx ctx = ctx_enter(mem->url);
136 debug(20, 3) ("destroy_MemObject: destroying %p\n", mem);
137 e->mem_obj = NULL;
138 if (!shutting_down)
139 assert(mem->swapout.fd == -1);
140 stmemFree(&mem->data_hdr);
141 mem->inmem_hi = 0;
142 /* XXX account log_url */
143 #if USE_ASYNC_IO
144 while (mem->clients != NULL)
145 storeUnregister(e, mem->clients->callback_data);
146 #endif
147 /*
148 * There is no way to abort FD-less clients, so they might
149 * still have mem->clients set if mem->fd == -1
150 */
151 assert(mem->fd == -1 || mem->clients == NULL);
152 httpReplyDestroy(mem->reply);
153 requestUnlink(mem->request);
154 mem->request = NULL;
155 ctx_exit(ctx); /* must exit before we free mem->url */
156 safe_free(mem->url);
157 safe_free(mem->log_url);
158 memFree(mem, MEM_MEMOBJECT);
159 }
160
161 static void
162 destroy_StoreEntry(void *data)
163 {
164 StoreEntry *e = data;
165 debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e);
166 assert(e != NULL);
167 if (e->mem_obj)
168 destroy_MemObject(e);
169 storeHashDelete(e);
170 assert(e->key == NULL);
171 memFree(e, MEM_STOREENTRY);
172 }
173
174 /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
175
176 void
177 storeHashInsert(StoreEntry * e, const cache_key * key)
178 {
179 debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n",
180 e, storeKeyText(key));
181 e->key = storeKeyDup(key);
182 hash_join(store_table, (hash_link *) e);
183 dlinkAdd(e, &e->lru, &store_list);
184 }
185
186 static void
187 storeHashDelete(StoreEntry * e)
188 {
189 hash_remove_link(store_table, (hash_link *) e);
190 dlinkDelete(&e->lru, &store_list);
191 storeKeyFree(e->key);
192 e->key = NULL;
193 }
194
195 /* -------------------------------------------------------------------------- */
196
197
198 /* get rid of memory copy of the object */
199 /* Only call this if storeCheckPurgeMem(e) returns 1 */
200 static void
201 storePurgeMem(StoreEntry * e)
202 {
203 if (e->mem_obj == NULL)
204 return;
205 debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n",
206 storeKeyText(e->key));
207 storeSetMemStatus(e, NOT_IN_MEMORY);
208 destroy_MemObject(e);
209 if (e->swap_status != SWAPOUT_DONE)
210 storeRelease(e);
211 }
212
213 void
214 storeLockObject(StoreEntry * e)
215 {
216 if (e->lock_count++ == 0) {
217 dlinkDelete(&e->lru, &store_list);
218 dlinkAdd(e, &e->lru, &store_list);
219 }
220 debug(20, 3) ("storeLockObject: key '%s' count=%d\n",
221 storeKeyText(e->key), (int) e->lock_count);
222 e->lastref = squid_curtime;
223 }
224
225 void
226 storeReleaseRequest(StoreEntry * e)
227 {
228 if (EBIT_TEST(e->flags, RELEASE_REQUEST))
229 return;
230 debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->key));
231 EBIT_SET(e->flags, RELEASE_REQUEST);
232 /*
233 * Clear cachable flag here because we might get called before
234 * anyone else even looks at the cachability flag. Also, this
235 * prevents httpMakePublic from really setting a public key.
236 */
237 EBIT_CLR(e->flags, ENTRY_CACHABLE);
238 storeSetPrivateKey(e);
239 }
240
241 /* unlock object, return -1 if object get released after unlock
242 * otherwise lock_count */
243 int
244 storeUnlockObject(StoreEntry * e)
245 {
246 e->lock_count--;
247 debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n",
248 storeKeyText(e->key), e->lock_count);
249 if (e->lock_count)
250 return (int) e->lock_count;
251 if (e->store_status == STORE_PENDING)
252 EBIT_SET(e->flags, RELEASE_REQUEST);
253 assert(storePendingNClients(e) == 0);
254 if (EBIT_TEST(e->flags, RELEASE_REQUEST))
255 storeRelease(e);
256 else if (storeKeepInMemory(e)) {
257 storeSetMemStatus(e, IN_MEMORY);
258 requestUnlink(e->mem_obj->request);
259 e->mem_obj->request = NULL;
260 } else {
261 storePurgeMem(e);
262 if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
263 dlinkDelete(&e->lru, &store_list);
264 dlinkAddTail(e, &e->lru, &store_list);
265 }
266 }
267 return 0;
268 }
269
270 /* Lookup an object in the cache.
271 * return just a reference to object, don't start swapping in yet. */
272 StoreEntry *
273 storeGet(const cache_key * key)
274 {
275 debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key));
276 return (StoreEntry *) hash_lookup(store_table, key);
277 }
278
279 StoreEntry *
280 storeGetPublic(const char *uri, const method_t method)
281 {
282 StoreEntry *e = storeGet(storeKeyPublic(uri, method));
283 if (e == NULL && squid_curtime < 922500000)
284 e = storeGet(storeKeyPublicOld(uri, method));
285 return e;
286 }
287
288 static int
289 getKeyCounter(void)
290 {
291 static int key_counter = 0;
292 if (++key_counter < 0)
293 key_counter = 1;
294 return key_counter;
295 }
296
297 void
298 storeSetPrivateKey(StoreEntry * e)
299 {
300 const cache_key *newkey;
301 MemObject *mem = e->mem_obj;
302 if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE))
303 return; /* is already private */
304 if (e->key) {
305 if (e->swap_file_number > -1)
306 storeDirSwapLog(e, SWAP_LOG_DEL);
307 storeHashDelete(e);
308 }
309 if (mem != NULL) {
310 mem->id = getKeyCounter();
311 newkey = storeKeyPrivate(mem->url, mem->method, mem->id);
312 } else {
313 newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter());
314 }
315 assert(hash_lookup(store_table, newkey) == NULL);
316 EBIT_SET(e->flags, KEY_PRIVATE);
317 storeHashInsert(e, newkey);
318 }
319
320 void
321 storeSetPublicKey(StoreEntry * e)
322 {
323 StoreEntry *e2 = NULL;
324 const cache_key *newkey;
325 MemObject *mem = e->mem_obj;
326 if (e->key && !EBIT_TEST(e->flags, KEY_PRIVATE))
327 return; /* is already public */
328 assert(mem);
329 /*
330 * We can't make RELEASE_REQUEST objects public. Depending on
331 * when RELEASE_REQUEST gets set, we might not be swapping out
332 * the object. If we're not swapping out, then subsequent
333 * store clients won't be able to access object data which has
334 * been freed from memory.
335 *
336 * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not
337 * be set, and storeSetPublicKey() should not be called.
338 */
339 assert(!EBIT_TEST(e->flags, RELEASE_REQUEST));
340 newkey = storeKeyPublic(mem->url, mem->method);
341 if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) {
342 debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url);
343 storeSetPrivateKey(e2);
344 storeRelease(e2);
345 newkey = storeKeyPublic(mem->url, mem->method);
346 }
347 if (e->key)
348 storeHashDelete(e);
349 EBIT_CLR(e->flags, KEY_PRIVATE);
350 storeHashInsert(e, newkey);
351 if (e->swap_file_number > -1)
352 storeDirSwapLog(e, SWAP_LOG_ADD);
353 }
354
355 StoreEntry *
356 storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method)
357 {
358 StoreEntry *e = NULL;
359 MemObject *mem = NULL;
360 debug(20, 3) ("storeCreateEntry: '%s'\n", url);
361
362 e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url);
363 e->lock_count = 1; /* Note lock here w/o calling storeLock() */
364 mem = e->mem_obj;
365 mem->method = method;
366 if (neighbors_do_private_keys || !flags.hierarchical)
367 storeSetPrivateKey(e);
368 else
369 storeSetPublicKey(e);
370 if (flags.cachable) {
371 EBIT_SET(e->flags, ENTRY_CACHABLE);
372 EBIT_CLR(e->flags, RELEASE_REQUEST);
373 } else {
374 EBIT_CLR(e->flags, ENTRY_CACHABLE);
375 storeReleaseRequest(e);
376 }
377 e->store_status = STORE_PENDING;
378 storeSetMemStatus(e, NOT_IN_MEMORY);
379 e->swap_status = SWAPOUT_NONE;
380 e->swap_file_number = -1;
381 e->refcount = 0;
382 e->lastref = squid_curtime;
383 e->timestamp = 0; /* set in storeTimestampsSet() */
384 e->ping_status = PING_NONE;
385 EBIT_SET(e->flags, ENTRY_VALIDATED);
386 return e;
387 }
388
389 /* Mark object as expired */
390 void
391 storeExpireNow(StoreEntry * e)
392 {
393 debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->key));
394 e->expires = squid_curtime;
395 }
396
397 /* Append incoming data from a primary server to an entry. */
398 void
399 storeAppend(StoreEntry * e, const char *buf, int len)
400 {
401 MemObject *mem = e->mem_obj;
402 assert(mem != NULL);
403 assert(len >= 0);
404 assert(e->store_status == STORE_PENDING);
405 if (len) {
406 debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n",
407 len,
408 storeKeyText(e->key));
409 storeGetMemSpace(len);
410 stmemAppend(&mem->data_hdr, buf, len);
411 mem->inmem_hi += len;
412 }
413 if (EBIT_TEST(e->flags, DELAY_SENDING))
414 return;
415 #ifdef OPTIMISTIC_IO
416 storeLockObject(e);
417 #endif
418 InvokeHandlers(e);
419 storeCheckSwapOut(e);
420 #ifdef OPTIMISTIC_IO
421 storeUnlockObject(e);
422 #endif
423 }
424
425 #if STDC_HEADERS
426 void
427 storeAppendPrintf(StoreEntry * e, const char *fmt,...)
428 {
429 va_list args;
430 va_start(args, fmt);
431 #else
432 void
433 storeAppendPrintf(va_alist)
434 va_dcl
435 {
436 va_list args;
437 StoreEntry *e = NULL;
438 const char *fmt = NULL;
439 va_start(args);
440 e = va_arg(args, StoreEntry *);
441 fmt = va_arg(args, char *);
442 #endif
443 storeAppendVPrintf(e, fmt, args);
444 va_end(args);
445 }
446
447 /* used be storeAppendPrintf and Packer */
448 void
449 storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs)
450 {
451 LOCAL_ARRAY(char, buf, 4096);
452 buf[0] = '\0';
453 vsnprintf(buf, 4096, fmt, vargs);
454 storeAppend(e, buf, strlen(buf));
455 }
456
457 struct _store_check_cachable_hist {
458 struct {
459 int non_get;
460 int not_entry_cachable;
461 int release_request;
462 int wrong_content_length;
463 int negative_cached;
464 int too_big;
465 int private_key;
466 int too_many_open_files;
467 int too_many_open_fds;
468 int lru_age_too_low;
469 } no;
470 struct {
471 int Default;
472 } yes;
473 } store_check_cachable_hist;
474
475 int
476 storeTooManyDiskFilesOpen(void)
477 {
478 if (Config.max_open_disk_fds == 0)
479 return 0;
480 if (store_open_disk_fd > Config.max_open_disk_fds)
481 return 1;
482 return 0;
483 }
484
485 int
486 storeCheckCachable(StoreEntry * e)
487 {
488 #if CACHE_ALL_METHODS
489 if (e->mem_obj->method != METHOD_GET) {
490 debug(20, 2) ("storeCheckCachable: NO: non-GET method\n");
491 store_check_cachable_hist.no.non_get++;
492 } else
493 #endif
494 if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) {
495 debug(20, 2) ("storeCheckCachable: NO: not cachable\n");
496 store_check_cachable_hist.no.not_entry_cachable++;
497 } else if (EBIT_TEST(e->flags, RELEASE_REQUEST)) {
498 debug(20, 2) ("storeCheckCachable: NO: release requested\n");
499 store_check_cachable_hist.no.release_request++;
500 } else if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) {
501 debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n");
502 store_check_cachable_hist.no.wrong_content_length++;
503 } else if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) {
504 debug(20, 3) ("storeCheckCachable: NO: negative cached\n");
505 store_check_cachable_hist.no.negative_cached++;
506 return 0; /* avoid release call below */
507 } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) {
508 debug(20, 2) ("storeCheckCachable: NO: too big\n");
509 store_check_cachable_hist.no.too_big++;
510 } else if (EBIT_TEST(e->flags, KEY_PRIVATE)) {
511 debug(20, 3) ("storeCheckCachable: NO: private key\n");
512 store_check_cachable_hist.no.private_key++;
513 } else if (e->swap_status != SWAPOUT_NONE) {
514 /*
515 * here we checked the swap_status because the remaining
516 * cases are only relevant only if we haven't started swapping
517 * out the object yet.
518 */
519 return 1;
520 } else if (storeTooManyDiskFilesOpen()) {
521 debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n");
522 store_check_cachable_hist.no.too_many_open_files++;
523 } else if (fdNFree() < RESERVED_FD) {
524 debug(20, 2) ("storeCheckCachable: NO: too many FD's open\n");
525 store_check_cachable_hist.no.too_many_open_fds++;
526 } else if (storeExpiredReferenceAge() < 300) {
527 debug(20, 2) ("storeCheckCachable: NO: LRU Age = %d\n",
528 storeExpiredReferenceAge());
529 store_check_cachable_hist.no.lru_age_too_low++;
530 } else {
531 store_check_cachable_hist.yes.Default++;
532 return 1;
533 }
534 storeReleaseRequest(e);
535 EBIT_CLR(e->flags, ENTRY_CACHABLE);
536 return 0;
537 }
538
539 static void
540 storeCheckCachableStats(StoreEntry * sentry)
541 {
542 storeAppendPrintf(sentry, "Category\t Count\n");
543
544 storeAppendPrintf(sentry, "no.non_get\t%d\n",
545 store_check_cachable_hist.no.non_get);
546 storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n",
547 store_check_cachable_hist.no.not_entry_cachable);
548 storeAppendPrintf(sentry, "no.release_request\t%d\n",
549 store_check_cachable_hist.no.release_request);
550 storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n",
551 store_check_cachable_hist.no.wrong_content_length);
552 storeAppendPrintf(sentry, "no.negative_cached\t%d\n",
553 store_check_cachable_hist.no.negative_cached);
554 storeAppendPrintf(sentry, "no.too_big\t%d\n",
555 store_check_cachable_hist.no.too_big);
556 storeAppendPrintf(sentry, "no.private_key\t%d\n",
557 store_check_cachable_hist.no.private_key);
558 storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n",
559 store_check_cachable_hist.no.too_many_open_files);
560 storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n",
561 store_check_cachable_hist.no.too_many_open_fds);
562 storeAppendPrintf(sentry, "no.lru_age_too_low\t%d\n",
563 store_check_cachable_hist.no.lru_age_too_low);
564 storeAppendPrintf(sentry, "yes.default\t%d\n",
565 store_check_cachable_hist.yes.Default);
566 }
567
568 /* Complete transfer into the local cache. */
569 void
570 storeComplete(StoreEntry * e)
571 {
572 debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key));
573 if (e->store_status != STORE_PENDING) {
574 /*
575 * if we're not STORE_PENDING, then probably we got aborted
576 * and there should be NO clients on this entry
577 */
578 assert(EBIT_TEST(e->flags, ENTRY_ABORTED));
579 assert(e->mem_obj->nclients == 0);
580 return;
581 }
582 e->mem_obj->object_sz = e->mem_obj->inmem_hi;
583 e->store_status = STORE_OK;
584 assert(e->mem_status == NOT_IN_MEMORY);
585 if (!storeEntryValidLength(e)) {
586 EBIT_SET(e->flags, ENTRY_BAD_LENGTH);
587 storeReleaseRequest(e);
588 }
589 #if USE_CACHE_DIGESTS
590 if (e->mem_obj->request)
591 e->mem_obj->request->hier.store_complete_stop = current_time;
592 #endif
593 InvokeHandlers(e);
594 storeCheckSwapOut(e);
595 }
596
597 /*
598 * Someone wants to abort this transfer. Set the reason in the
599 * request structure, call the server-side callback and mark the
600 * entry for releasing
601 */
602 void
603 storeAbort(StoreEntry * e)
604 {
605 MemObject *mem = e->mem_obj;
606 assert(e->store_status == STORE_PENDING);
607 assert(mem != NULL);
608 debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->key));
609 storeLockObject(e); /* lock while aborting */
610 storeNegativeCache(e);
611 storeReleaseRequest(e);
612 EBIT_SET(e->flags, ENTRY_ABORTED);
613 storeSetMemStatus(e, NOT_IN_MEMORY);
614 /* No DISK swap for negative cached object */
615 e->swap_status = SWAPOUT_NONE;
616 e->store_status = STORE_OK;
617 /*
618 * We assign an object length here. The only other place we assign
619 * the object length is in storeComplete()
620 */
621 mem->object_sz = mem->inmem_hi;
622 /* Notify the server side */
623 if (mem->abort.callback) {
624 eventAdd("mem->abort.callback",
625 mem->abort.callback,
626 mem->abort.data,
627 0.0,
628 0);
629 mem->abort.callback = NULL;
630 mem->abort.data = NULL;
631 }
632 /* Notify the client side */
633 InvokeHandlers(e);
634 /* Do we need to close the swapout file? */
635 /* Not if we never started swapping out */
636 /* But we may need to cancel an open/stat in progress if using ASYNC */
637 #if USE_ASYNC_IO
638 aioCancel(-1, e);
639 #endif
640 if (e->swap_file_number > -1) {
641 #if USE_ASYNC_IO
642 /* Need to cancel any pending ASYNC writes right now */
643 if (mem->swapout.fd >= 0)
644 aioCancel(mem->swapout.fd, NULL);
645 #endif
646 /* we have to close the disk file if there is no write pending */
647 if (!storeSwapOutWriteQueued(mem))
648 storeSwapOutFileClose(e);
649 }
650 storeUnlockObject(e); /* unlock */
651 }
652
653 /* Clear Memory storage to accommodate the given object len */
654 static void
655 storeGetMemSpace(int size)
656 {
657 StoreEntry *e = NULL;
658 int released = 0;
659 static time_t last_check = 0;
660 int pages_needed;
661 dlink_node *m;
662 dlink_node *head;
663 dlink_node *prev = NULL;
664 if (squid_curtime == last_check)
665 return;
666 last_check = squid_curtime;
667 pages_needed = (size / SM_PAGE_SIZE) + 1;
668 if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
669 return;
670 if (store_rebuilding)
671 return;
672 debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed);
673 head = inmem_list.head;
674 for (m = inmem_list.tail; m; m = prev) {
675 if (m == head)
676 break;
677 prev = m->prev;
678 e = m->data;
679 if (storeEntryLocked(e)) {
680 dlinkDelete(m, &inmem_list);
681 dlinkAdd(e, m, &inmem_list);
682 continue;
683 }
684 released++;
685 storePurgeMem(e);
686 if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max)
687 break;
688 }
689 debug(20, 3) ("storeGetMemSpace stats:\n");
690 debug(20, 3) (" %6d HOT objects\n", hot_obj_count);
691 debug(20, 3) (" %6d were released\n", released);
692 }
693
694 /* The maximum objects to scan for maintain storage space */
695 #define MAINTAIN_MAX_SCAN 1024
696 #define MAINTAIN_MAX_REMOVE 64
697
698 /*
699 * This routine is to be called by main loop in main.c.
700 * It removes expired objects on only one bucket for each time called.
701 * returns the number of objects removed
702 *
703 * This should get called 1/s from main().
704 */
705 void
706 storeMaintainSwapSpace(void *datanotused)
707 {
708 dlink_node *m;
709 dlink_node *prev = NULL;
710 StoreEntry *e = NULL;
711 int scanned = 0;
712 int locked = 0;
713 int expired = 0;
714 int max_scan;
715 int max_remove;
716 static time_t last_warn_time = 0;
717 /* We can't delete objects while rebuilding swap */
718 if (store_rebuilding) {
719 eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
720 return;
721 } else if (store_swap_size < store_swap_mid) {
722 max_scan = 100;
723 max_remove = 8;
724 eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1);
725 } else if (store_swap_size < store_swap_high) {
726 max_scan = 200;
727 max_remove = 8;
728 eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.1, 1);
729 } else {
730 max_scan = 500;
731 max_remove = 32;
732 eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.0, 1);
733 }
734 debug(20, 3) ("storeMaintainSwapSpace\n");
735 for (m = store_list.tail; m; m = prev) {
736 prev = m->prev;
737 e = m->data;
738 scanned++;
739 if (storeEntryLocked(e)) {
740 /*
741 * If there is a locked entry at the tail of the LRU list,
742 * move it to the beginning to get it out of the way.
743 * Theoretically, we might have all locked objects at the
744 * tail, and then we'll never remove anything here and the
745 * LRU age will go to zero.
746 */
747 if (memInUse(MEM_STOREENTRY) > max_scan) {
748 dlinkDelete(&e->lru, &store_list);
749 dlinkAdd(e, &e->lru, &store_list);
750 }
751 locked++;
752 } else if (storeCheckExpired(e)) {
753 expired++;
754 storeRelease(e);
755 }
756 if (expired >= max_remove)
757 break;
758 if (scanned >= max_scan)
759 break;
760 }
761 debug(20, 3) ("storeMaintainSwapSpace stats:\n");
762 debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY));
763 debug(20, 3) (" %6d were scanned\n", scanned);
764 debug(20, 3) (" %6d were locked\n", locked);
765 debug(20, 3) (" %6d were expired\n", expired);
766 if (store_swap_size < Config.Swap.maxSize)
767 return;
768 if (squid_curtime - last_warn_time < 10)
769 return;
770 debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n",
771 store_swap_size, Config.Swap.maxSize);
772 last_warn_time = squid_curtime;
773 }
774
775
776 /* release an object from a cache */
777 /* return number of objects released. */
778 void
779 storeRelease(StoreEntry * e)
780 {
781 debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->key));
782 /* If, for any reason we can't discard this object because of an
783 * outstanding request, mark it for pending release */
784 if (storeEntryLocked(e)) {
785 storeExpireNow(e);
786 debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n");
787 storeReleaseRequest(e);
788 return;
789 }
790 #if USE_ASYNC_IO
791 /*
792 * Make sure all forgotten async ops are cancelled
793 */
794 aioCancel(-1, e);
795 #endif
796 if (store_rebuilding) {
797 storeSetPrivateKey(e);
798 if (e->mem_obj) {
799 storeSetMemStatus(e, NOT_IN_MEMORY);
800 destroy_MemObject(e);
801 }
802 /*
803 * Fake a call to storeLockObject(). When rebuilding is done,
804 * we'll just call storeUnlockObject() on these.
805 */
806 e->lock_count++;
807 stackPush(&LateReleaseStack, e);
808 return;
809 }
810 storeLog(STORE_LOG_RELEASE, e);
811 if (e->swap_file_number > -1) {
812 storeUnlinkFileno(e->swap_file_number);
813 storeDirMapBitReset(e->swap_file_number);
814 if (e->swap_status == SWAPOUT_DONE)
815 if (EBIT_TEST(e->flags, ENTRY_VALIDATED))
816 storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1);
817 if (!EBIT_TEST(e->flags, KEY_PRIVATE))
818 storeDirSwapLog(e, SWAP_LOG_DEL);
819 }
820 storeSetMemStatus(e, NOT_IN_MEMORY);
821 destroy_StoreEntry(e);
822 }
823
824 static void
825 storeLateRelease(void *unused)
826 {
827 StoreEntry *e;
828 int i;
829 static int n = 0;
830 if (store_rebuilding) {
831 eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
832 return;
833 }
834 for (i = 0; i < 10; i++) {
835 e = stackPop(&LateReleaseStack);
836 if (e == NULL) {
837 /* done! */
838 debug(20, 1) ("storeLateRelease: released %d objects\n", n);
839 return;
840 }
841 storeUnlockObject(e);
842 n++;
843 }
844 eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1);
845 }
846
847 /* return 1 if a store entry is locked */
848 static int
849 storeEntryLocked(const StoreEntry * e)
850 {
851 if (e->lock_count)
852 return 1;
853 if (e->swap_status == SWAPOUT_OPENING)
854 return 1;
855 if (e->swap_status == SWAPOUT_WRITING)
856 return 1;
857 if (e->store_status == STORE_PENDING)
858 return 1;
859 /*
860 * SPECIAL, PUBLIC entries should be "locked"
861 */
862 if (EBIT_TEST(e->flags, ENTRY_SPECIAL))
863 if (!EBIT_TEST(e->flags, KEY_PRIVATE))
864 return 1;
865 return 0;
866 }
867
868 static int
869 storeEntryValidLength(const StoreEntry * e)
870 {
871 int diff;
872 const HttpReply *reply;
873 assert(e->mem_obj != NULL);
874 reply = e->mem_obj->reply;
875 debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key));
876 debug(20, 5) ("storeEntryValidLength: object_len = %d\n",
877 objectLen(e));
878 debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n",
879 reply->hdr_sz);
880 debug(20, 5) ("storeEntryValidLength: content_length = %d\n",
881 reply->content_length);
882 if (reply->content_length < 0) {
883 debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n",
884 storeKeyText(e->key));
885 return 1;
886 }
887 if (reply->hdr_sz == 0) {
888 debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n",
889 storeKeyText(e->key));
890 return 1;
891 }
892 if (e->mem_obj->method == METHOD_HEAD) {
893 debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n",
894 storeKeyText(e->key));
895 return 1;
896 }
897 if (reply->sline.status == HTTP_NOT_MODIFIED)
898 return 1;
899 if (reply->sline.status == HTTP_NO_CONTENT)
900 return 1;
901 diff = reply->hdr_sz + reply->content_length - objectLen(e);
902 if (diff == 0)
903 return 1;
904 debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n",
905 diff < 0 ? -diff : diff,
906 diff < 0 ? "big" : "small",
907 storeKeyText(e->key));
908 return 0;
909 }
910
911 static void
912 storeInitHashValues(void)
913 {
914 int i;
915 /* Calculate size of hash table (maximum currently 64k buckets). */
916 i = Config.Swap.maxSize / Config.Store.avgObjectSize;
917 debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n",
918 Config.Swap.maxSize, i);
919 i /= Config.Store.objectsPerBucket;
920 debug(20, 1) ("Target number of buckets: %d\n", i);
921 /* ideally the full scan period should be configurable, for the
922 * moment it remains at approximately 24 hours. */
923 store_hash_buckets = storeKeyHashBuckets(i);
924 store_maintain_rate = 86400 / store_hash_buckets;
925 assert(store_maintain_rate > 0);
926 debug(20, 1) ("Using %d Store buckets, replacement runs every %d second%s\n",
927 store_hash_buckets,
928 store_maintain_rate,
929 store_maintain_rate == 1 ? null_string : "s");
930 debug(20, 1) ("Max Mem size: %d KB\n", Config.memMaxSize >> 10);
931 debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize);
932 }
933
934 void
935 storeInit(void)
936 {
937 storeKeyInit();
938 storeInitHashValues();
939 store_table = hash_create(storeKeyHashCmp,
940 store_hash_buckets, storeKeyHashHash);
941 storeDigestInit();
942 storeLogOpen();
943 if (storeVerifyCacheDirs() < 0) {
944 xstrncpy(tmp_error_buf,
945 "\tFailed to verify one of the swap directories, Check cache.log\n"
946 "\tfor details. Run 'squid -z' to create swap directories\n"
947 "\tif needed, or if running Squid for the first time.",
948 ERROR_BUF_SZ);
949 fatal(tmp_error_buf);
950 }
951 storeDirOpenSwapLogs();
952 store_list.head = store_list.tail = NULL;
953 inmem_list.head = inmem_list.tail = NULL;
954 stackInit(&LateReleaseStack);
955 eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1);
956 storeRebuildStart();
957 cachemgrRegister("storedir",
958 "Store Directory Stats",
959 storeDirStats, 0, 1);
960 cachemgrRegister("store_check_cachable_stats",
961 "storeCheckCachable() Stats",
962 storeCheckCachableStats, 0, 1);
963 }
964
965 void
966 storeConfigure(void)
967 {
968 store_swap_high = (long) (((float) Config.Swap.maxSize *
969 (float) Config.Swap.highWaterMark) / (float) 100);
970 store_swap_low = (long) (((float) Config.Swap.maxSize *
971 (float) Config.Swap.lowWaterMark) / (float) 100);
972 store_swap_mid = (store_swap_high >> 1) + (store_swap_low >> 1);
973 store_pages_max = Config.memMaxSize / SM_PAGE_SIZE;
974 }
975
976 static int
977 storeKeepInMemory(const StoreEntry * e)
978 {
979 MemObject *mem = e->mem_obj;
980 if (mem == NULL)
981 return 0;
982 if (mem->data_hdr.head == NULL)
983 return 0;
984 return mem->inmem_lo == 0;
985 }
986
987 static int
988 storeCheckExpired(const StoreEntry * e)
989 {
990 if (storeEntryLocked(e))
991 return 0;
992 if (EBIT_TEST(e->flags, RELEASE_REQUEST))
993 return 1;
994 if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires)
995 return 1;
996 if (squid_curtime - e->lastref > storeExpiredReferenceAge())
997 return 1;
998 return 0;
999 }
1000
1001 /*
1002 * storeExpiredReferenceAge
1003 *
1004 * The LRU age is scaled exponentially between 1 minute and
1005 * Config.referenceAge , when store_swap_low < store_swap_size <
1006 * store_swap_high. This keeps store_swap_size within the low and high
1007 * water marks. If the cache is very busy then store_swap_size stays
1008 * closer to the low water mark, if it is not busy, then it will stay
1009 * near the high water mark. The LRU age value can be examined on the
1010 * cachemgr 'info' page.
1011 */
1012 time_t
1013 storeExpiredReferenceAge(void)
1014 {
1015 double x;
1016 double z;
1017 time_t age;
1018 x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low);
1019 x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x;
1020 z = pow((double) (Config.referenceAge / 60), x);
1021 age = (time_t) (z * 60.0);
1022 if (age < 60)
1023 age = 60;
1024 else if (age > 31536000)
1025 age = 31536000;
1026 return age;
1027 }
1028
1029 void
1030 storeNegativeCache(StoreEntry * e)
1031 {
1032 e->expires = squid_curtime + Config.negativeTtl;
1033 EBIT_SET(e->flags, ENTRY_NEGCACHED);
1034 }
1035
1036 void
1037 storeFreeMemory(void)
1038 {
1039 hashFreeItems(store_table, destroy_StoreEntry);
1040 hashFreeMemory(store_table);
1041 store_table = NULL;
1042 #if USE_CACHE_DIGEST
1043 if (store_digest)
1044 cacheDigestDestroy(store_digest);
1045 #endif
1046 store_digest = NULL;
1047 }
1048
1049 int
1050 expiresMoreThan(time_t expires, time_t when)
1051 {
1052 if (expires < 0) /* No Expires given */
1053 return 1;
1054 return (expires > (squid_curtime + when));
1055 }
1056
1057 int
1058 storeEntryValidToSend(StoreEntry * e)
1059 {
1060 if (EBIT_TEST(e->flags, RELEASE_REQUEST))
1061 return 0;
1062 if (EBIT_TEST(e->flags, ENTRY_NEGCACHED))
1063 if (e->expires <= squid_curtime)
1064 return 0;
1065 if (EBIT_TEST(e->flags, ENTRY_ABORTED))
1066 return 0;
1067 return 1;
1068 }
1069
1070 void
1071 storeTimestampsSet(StoreEntry * entry)
1072 {
1073 time_t served_date = -1;
1074 const HttpReply *reply = entry->mem_obj->reply;
1075 served_date = reply->date;
1076 if (served_date < 0)
1077 served_date = squid_curtime;
1078 entry->expires = reply->expires;
1079 entry->lastmod = reply->last_modified;
1080 entry->timestamp = served_date;
1081 }
1082
1083 void
1084 storeRegisterAbort(StoreEntry * e, STABH * cb, void *data)
1085 {
1086 MemObject *mem = e->mem_obj;
1087 assert(mem);
1088 assert(mem->abort.callback == NULL);
1089 mem->abort.callback = cb;
1090 mem->abort.data = data;
1091 }
1092
1093 void
1094 storeUnregisterAbort(StoreEntry * e)
1095 {
1096 MemObject *mem = e->mem_obj;
1097 assert(mem);
1098 mem->abort.callback = NULL;
1099 }
1100
1101 void
1102 storeMemObjectDump(MemObject * mem)
1103 {
1104 debug(20, 1) ("MemObject->data.head: %p\n",
1105 mem->data_hdr.head);
1106 debug(20, 1) ("MemObject->data.tail: %p\n",
1107 mem->data_hdr.tail);
1108 debug(20, 1) ("MemObject->data.origin_offset: %d\n",
1109 mem->data_hdr.origin_offset);
1110 debug(20, 1) ("MemObject->start_ping: %d.%06d\n",
1111 (int) mem->start_ping.tv_sec,
1112 (int) mem->start_ping.tv_usec);
1113 debug(20, 1) ("MemObject->inmem_hi: %d\n",
1114 (int) mem->inmem_hi);
1115 debug(20, 1) ("MemObject->inmem_lo: %d\n",
1116 (int) mem->inmem_lo);
1117 debug(20, 1) ("MemObject->clients: %p\n",
1118 mem->clients);
1119 debug(20, 1) ("MemObject->nclients: %d\n",
1120 mem->nclients);
1121 debug(20, 1) ("MemObject->swapout.fd: %d\n",
1122 mem->swapout.fd);
1123 debug(20, 1) ("MemObject->reply: %p\n",
1124 mem->reply);
1125 debug(20, 1) ("MemObject->request: %p\n",
1126 mem->request);
1127 debug(20, 1) ("MemObject->log_url: %p %s\n",
1128 mem->log_url,
1129 checkNullString(mem->log_url));
1130 }
1131
1132 void
1133 storeEntryDump(StoreEntry * e, int l)
1134 {
1135 debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->key));
1136 debug(20, l) ("StoreEntry->next: %p\n", e->next);
1137 debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj);
1138 debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp);
1139 debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref);
1140 debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires);
1141 debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod);
1142 debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz);
1143 debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount);
1144 debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e));
1145 debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number);
1146 debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count);
1147 debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status);
1148 debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status);
1149 debug(20, l) ("StoreEntry->store_status: %d\n", (int) e->store_status);
1150 debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status);
1151 }
1152
1153 /* NOTE, this function assumes only two mem states */
1154 void
1155 storeSetMemStatus(StoreEntry * e, int new_status)
1156 {
1157 MemObject *mem = e->mem_obj;
1158 if (new_status == e->mem_status)
1159 return;
1160 assert(mem != NULL);
1161 if (new_status == IN_MEMORY) {
1162 assert(mem->inmem_lo == 0);
1163 dlinkAdd(e, &mem->lru, &inmem_list);
1164 hot_obj_count++;
1165 } else {
1166 dlinkDelete(&mem->lru, &inmem_list);
1167 hot_obj_count--;
1168 }
1169 e->mem_status = new_status;
1170 }
1171
1172 const char *
1173 storeUrl(const StoreEntry * e)
1174 {
1175 if (e == NULL)
1176 return "[null_entry]";
1177 else if (e->mem_obj == NULL)
1178 return "[null_mem_obj]";
1179 else
1180 return e->mem_obj->url;
1181 }
1182
1183 void
1184 storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url)
1185 {
1186 if (e->mem_obj)
1187 return;
1188 e->mem_obj = new_MemObject(url, log_url);
1189 }
1190
1191 /* this just sets DELAY_SENDING */
1192 void
1193 storeBuffer(StoreEntry * e)
1194 {
1195 EBIT_SET(e->flags, DELAY_SENDING);
1196 }
1197
1198 /* this just clears DELAY_SENDING and Invokes the handlers */
1199 void
1200 storeBufferFlush(StoreEntry * e)
1201 {
1202 EBIT_CLR(e->flags, DELAY_SENDING);
1203 InvokeHandlers(e);
1204 storeCheckSwapOut(e);
1205 }
1206
1207 void
1208 storeUnlinkFileno(int fileno)
1209 {
1210 debug(20, 5) ("storeUnlinkFileno: %08X\n", fileno);
1211 #if USE_ASYNC_IO
1212 safeunlink(storeSwapFullPath(fileno, NULL), 1);
1213 #else
1214 unlinkdUnlink(storeSwapFullPath(fileno, NULL));
1215 #endif
1216 }
1217
1218 int
1219 objectLen(const StoreEntry * e)
1220 {
1221 assert(e->mem_obj != NULL);
1222 return e->mem_obj->object_sz;
1223 }
1224
1225 int
1226 contentLen(const StoreEntry * e)
1227 {
1228 assert(e->mem_obj != NULL);
1229 assert(e->mem_obj->reply != NULL);
1230 return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz;
1231 }
1232
1233 HttpReply *
1234 storeEntryReply(StoreEntry * e)
1235 {
1236 if (NULL == e)
1237 return NULL;
1238 if (NULL == e->mem_obj)
1239 return NULL;
1240 return e->mem_obj->reply;
1241 }
1242
1243 void
1244 storeEntryReset(StoreEntry * e)
1245 {
1246 MemObject *mem = e->mem_obj;
1247 debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e));
1248 assert(mem->swapout.fd == -1);
1249 stmemFree(&mem->data_hdr);
1250 mem->inmem_hi = mem->inmem_lo = 0;
1251 httpReplyDestroy(mem->reply);
1252 mem->reply = httpReplyCreate();
1253 }
Mirror of https://github.com/squid-cache/squid.git