]>
Commit | Line | Data |
---|---|---|
164f7660 | 1 | |
30a4f2a8 | 2 | /* |
789f4e81 | 3 | * $Id: store.cc,v 1.478 1999/01/12 15:47:56 wessels Exp $ |
30a4f2a8 | 4 | * |
8638fc66 | 5 | * DEBUG: section 20 Storage Manager |
30a4f2a8 | 6 | * AUTHOR: Harvest Derived |
7 | * | |
42c04c16 | 8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
e25c139f | 9 | * ---------------------------------------------------------- |
30a4f2a8 | 10 | * |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
e25c139f | 13 | * National Laboratory for Applied Network Research and funded by the |
14 | * National Science Foundation. Squid is Copyrighted (C) 1998 by | |
15 | * Duane Wessels and the University of California San Diego. Please | |
16 | * see the COPYRIGHT file for full details. Squid incorporates | |
17 | * software developed and/or copyrighted by other sources. Please see | |
18 | * the CREDITS file for full details. | |
30a4f2a8 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
cbdec147 | 32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. |
e25c139f | 33 | * |
c943f331 | 34 | */ |
090089c4 | 35 | |
f09f5b26 | 36 | #include "squid.h" |
090089c4 | 37 | |
38 | #define REBUILD_TIMESTAMP_DELTA_MAX 2 | |
227fbb74 | 39 | |
fcf5283d | 40 | #define STORE_IN_MEM_BUCKETS (229) |
090089c4 | 41 | |
0ee4272b | 42 | const char *memStatusStr[] = |
e62d2dea | 43 | { |
9dfb6c1c | 44 | "NOT_IN_MEMORY", |
9dfb6c1c | 45 | "IN_MEMORY" |
46 | }; | |
47 | ||
0ee4272b | 48 | const char *pingStatusStr[] = |
e62d2dea | 49 | { |
f17936ab | 50 | "PING_NONE", |
9dfb6c1c | 51 | "PING_WAITING", |
52 | "PING_TIMEOUT", | |
f17936ab | 53 | "PING_DONE" |
9dfb6c1c | 54 | }; |
55 | ||
0ee4272b | 56 | const char *storeStatusStr[] = |
e62d2dea | 57 | { |
9dfb6c1c | 58 | "STORE_OK", |
b7fe0ab0 | 59 | "STORE_PENDING" |
9dfb6c1c | 60 | }; |
61 | ||
0ee4272b | 62 | const char *swapStatusStr[] = |
e62d2dea | 63 | { |
8350fe9b | 64 | "SWAPOUT_NONE", |
65 | "SWAPOUT_OPENING", | |
66 | "SWAPOUT_WRITING", | |
67 | "SWAPOUT_DONE" | |
9dfb6c1c | 68 | }; |
69 | ||
0a0bf5db | 70 | typedef struct lock_ctrl_t { |
582b6456 | 71 | SIH *callback; |
0a0bf5db | 72 | void *callback_data; |
73 | StoreEntry *e; | |
74 | } lock_ctrl_t; | |
75 | ||
e3ef2b09 | 76 | /* |
77 | * local function prototypes | |
78 | */ | |
01fe9bf4 | 79 | static int storeCheckExpired(const StoreEntry *); |
f5b8bbc4 | 80 | static int storeEntryLocked(const StoreEntry *); |
81 | static int storeEntryValidLength(const StoreEntry *); | |
82 | static void storeGetMemSpace(int); | |
b93bcace | 83 | static void storeHashDelete(StoreEntry *); |
9fb13bb6 | 84 | static MemObject *new_MemObject(const char *, const char *); |
6cf028ab | 85 | static void destroy_MemObject(StoreEntry *); |
ec878047 | 86 | static FREE destroy_StoreEntry; |
f5b8bbc4 | 87 | static void storePurgeMem(StoreEntry *); |
007b8be4 | 88 | static int getKeyCounter(void); |
8350fe9b | 89 | static int storeKeepInMemory(const StoreEntry *); |
8423ff74 | 90 | static OBJH storeCheckCachableStats; |
e42d5181 | 91 | static EVH storeLateRelease; |
a21fbb54 | 92 | |
e3ef2b09 | 93 | /* |
94 | * local variables | |
95 | */ | |
b93bcace | 96 | static dlink_list inmem_list; |
e954773d | 97 | static int store_pages_high = 0; |
98 | static int store_pages_low = 0; | |
58104eab | 99 | static int store_swap_high = 0; |
100 | static int store_swap_low = 0; | |
d1497906 | 101 | static int store_swap_mid = 0; |
e924600d | 102 | static int store_maintain_rate; |
e42d5181 | 103 | static Stack LateReleaseStack; |
66cedb85 | 104 | |
b8d8561b | 105 | static MemObject * |
9fb13bb6 | 106 | new_MemObject(const char *url, const char *log_url) |
227fbb74 | 107 | { |
7021844c | 108 | MemObject *mem = memAllocate(MEM_MEMOBJECT); |
cb69b4c7 | 109 | mem->reply = httpReplyCreate(); |
9fb13bb6 | 110 | mem->url = xstrdup(url); |
88738790 | 111 | mem->log_url = xstrdup(log_url); |
8350fe9b | 112 | mem->swapout.fd = -1; |
07304bf9 | 113 | mem->object_sz = -1; |
ed03a839 | 114 | mem->fd = -1; |
59c4d35b | 115 | /* XXX account log_url */ |
a3d5953d | 116 | debug(20, 3) ("new_MemObject: returning %p\n", mem); |
30a4f2a8 | 117 | return mem; |
227fbb74 | 118 | } |
119 | ||
f09f5b26 | 120 | StoreEntry * |
9fb13bb6 | 121 | new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) |
090089c4 | 122 | { |
123 | StoreEntry *e = NULL; | |
7021844c | 124 | e = memAllocate(MEM_STOREENTRY); |
227fbb74 | 125 | if (mem_obj_flag) |
9fb13bb6 | 126 | e->mem_obj = new_MemObject(url, log_url); |
a3d5953d | 127 | debug(20, 3) ("new_StoreEntry: returning %p\n", e); |
e17dc75c | 128 | e->expires = e->lastmod = e->lastref = e->timestamp = -1; |
227fbb74 | 129 | return e; |
090089c4 | 130 | } |
131 | ||
b8d8561b | 132 | static void |
0e473d70 | 133 | destroy_MemObject(StoreEntry * e) |
090089c4 | 134 | { |
6cf028ab | 135 | MemObject *mem = e->mem_obj; |
123abbe1 | 136 | const Ctx ctx = ctx_enter(mem->url); |
a3d5953d | 137 | debug(20, 3) ("destroy_MemObject: destroying %p\n", mem); |
41f7e38d | 138 | e->mem_obj = NULL; |
9e665466 | 139 | if (!shutting_down) |
e82d6d21 | 140 | assert(mem->swapout.fd == -1); |
18fe65d0 | 141 | stmemFree(&mem->data_hdr); |
142 | mem->inmem_hi = 0; | |
59c4d35b | 143 | /* XXX account log_url */ |
6cf028ab | 144 | #if USE_ASYNC_IO |
0e473d70 | 145 | while (mem->clients != NULL) |
6cf028ab | 146 | storeUnregister(e, mem->clients->callback_data); |
147 | #endif | |
6982a226 | 148 | /* |
149 | * There is no way to abort FD-less clients, so they might | |
150 | * still have mem->clients set if mem->fd == -1 | |
151 | */ | |
152 | assert(mem->fd == -1 || mem->clients == NULL); | |
cb69b4c7 | 153 | httpReplyDestroy(mem->reply); |
30a4f2a8 | 154 | requestUnlink(mem->request); |
155 | mem->request = NULL; | |
2ac76861 | 156 | ctx_exit(ctx); /* must exit before we free mem->url */ |
48959832 | 157 | safe_free(mem->url); |
158 | safe_free(mem->log_url); | |
db1cd23c | 159 | memFree(mem, MEM_MEMOBJECT); |
090089c4 | 160 | } |
161 | ||
b8d8561b | 162 | static void |
ec878047 | 163 | destroy_StoreEntry(void *data) |
090089c4 | 164 | { |
ec878047 | 165 | StoreEntry *e = data; |
a3d5953d | 166 | debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); |
9e975e4e | 167 | assert(e != NULL); |
227fbb74 | 168 | if (e->mem_obj) |
6cf028ab | 169 | destroy_MemObject(e); |
043b055b | 170 | storeHashDelete(e); |
9fb13bb6 | 171 | assert(e->key == NULL); |
db1cd23c | 172 | memFree(e, MEM_STOREENTRY); |
227fbb74 | 173 | } |
090089c4 | 174 | |
090089c4 | 175 | /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ |
176 | ||
f09f5b26 | 177 | void |
9fb13bb6 | 178 | storeHashInsert(StoreEntry * e, const cache_key * key) |
090089c4 | 179 | { |
a3d5953d | 180 | debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", |
9fb13bb6 | 181 | e, storeKeyText(key)); |
182 | e->key = storeKeyDup(key); | |
b93bcace | 183 | hash_join(store_table, (hash_link *) e); |
66f07209 | 184 | dlinkAdd(e, &e->lru, &store_list); |
090089c4 | 185 | } |
186 | ||
b93bcace | 187 | static void |
b8d8561b | 188 | storeHashDelete(StoreEntry * e) |
090089c4 | 189 | { |
b93bcace | 190 | hash_remove_link(store_table, (hash_link *) e); |
e3ef2b09 | 191 | dlinkDelete(&e->lru, &store_list); |
9fb13bb6 | 192 | storeKeyFree(e->key); |
193 | e->key = NULL; | |
090089c4 | 194 | } |
195 | ||
090089c4 | 196 | /* -------------------------------------------------------------------------- */ |
197 | ||
090089c4 | 198 | |
199 | /* get rid of memory copy of the object */ | |
620da955 | 200 | /* Only call this if storeCheckPurgeMem(e) returns 1 */ |
24382924 | 201 | static void |
b8d8561b | 202 | storePurgeMem(StoreEntry * e) |
090089c4 | 203 | { |
2aca8433 | 204 | if (e->mem_obj == NULL) |
090089c4 | 205 | return; |
9fb13bb6 | 206 | debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", |
207 | storeKeyText(e->key)); | |
090089c4 | 208 | storeSetMemStatus(e, NOT_IN_MEMORY); |
6cf028ab | 209 | destroy_MemObject(e); |
06e8899b | 210 | if (e->swap_status != SWAPOUT_DONE) |
8272aded | 211 | storeRelease(e); |
090089c4 | 212 | } |
213 | ||
0a0bf5db | 214 | void |
95c4b18f | 215 | storeLockObject(StoreEntry * e) |
090089c4 | 216 | { |
b93bcace | 217 | if (e->lock_count++ == 0) { |
e3ef2b09 | 218 | dlinkDelete(&e->lru, &store_list); |
219 | dlinkAdd(e, &e->lru, &store_list); | |
b93bcace | 220 | } |
a3d5953d | 221 | debug(20, 3) ("storeLockObject: key '%s' count=%d\n", |
9fb13bb6 | 222 | storeKeyText(e->key), (int) e->lock_count); |
b8de7ebe | 223 | e->lastref = squid_curtime; |
090089c4 | 224 | } |
225 | ||
b8d8561b | 226 | void |
227 | storeReleaseRequest(StoreEntry * e) | |
2285407f | 228 | { |
d46a87a8 | 229 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
58bcd31b | 230 | return; |
9fb13bb6 | 231 | debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->key)); |
d46a87a8 | 232 | EBIT_SET(e->flags, RELEASE_REQUEST); |
f3e570e9 | 233 | /* |
234 | * Clear cachable flag here because we might get called before | |
235 | * anyone else even looks at the cachability flag. Also, this | |
236 | * prevents httpMakePublic from really setting a public key. | |
237 | */ | |
d46a87a8 | 238 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
fe54d06d | 239 | storeSetPrivateKey(e); |
2285407f | 240 | } |
241 | ||
090089c4 | 242 | /* unlock object, return -1 if object get released after unlock |
243 | * otherwise lock_count */ | |
b8d8561b | 244 | int |
245 | storeUnlockObject(StoreEntry * e) | |
090089c4 | 246 | { |
6c895381 | 247 | e->lock_count--; |
a3d5953d | 248 | debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", |
9fb13bb6 | 249 | storeKeyText(e->key), e->lock_count); |
30a4f2a8 | 250 | if (e->lock_count) |
a1e47288 | 251 | return (int) e->lock_count; |
b7fe0ab0 | 252 | if (e->store_status == STORE_PENDING) |
d46a87a8 | 253 | EBIT_SET(e->flags, RELEASE_REQUEST); |
9e975e4e | 254 | assert(storePendingNClients(e) == 0); |
d46a87a8 | 255 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
30a4f2a8 | 256 | storeRelease(e); |
8350fe9b | 257 | else if (storeKeepInMemory(e)) { |
258 | storeSetMemStatus(e, IN_MEMORY); | |
259 | requestUnlink(e->mem_obj->request); | |
260 | e->mem_obj->request = NULL; | |
66f07209 | 261 | } else { |
30a4f2a8 | 262 | storePurgeMem(e); |
d46a87a8 | 263 | if (EBIT_TEST(e->flags, KEY_PRIVATE)) { |
66f07209 | 264 | dlinkDelete(&e->lru, &store_list); |
265 | dlinkAddTail(e, &e->lru, &store_list); | |
266 | } | |
267 | } | |
6c895381 | 268 | return 0; |
090089c4 | 269 | } |
270 | ||
271 | /* Lookup an object in the cache. | |
272 | * return just a reference to object, don't start swapping in yet. */ | |
b8d8561b | 273 | StoreEntry * |
9fb13bb6 | 274 | storeGet(const cache_key * key) |
090089c4 | 275 | { |
9fb13bb6 | 276 | debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); |
277 | return (StoreEntry *) hash_lookup(store_table, key); | |
090089c4 | 278 | } |
279 | ||
08e5d64f | 280 | StoreEntry * |
281 | storeGetPublic(const char *uri, const method_t method) | |
282 | { | |
674ac814 | 283 | return storeGet(storeKeyPublic(uri, method)); |
08e5d64f | 284 | } |
285 | ||
007b8be4 | 286 | static int |
287 | getKeyCounter(void) | |
04e8dbaa | 288 | { |
007b8be4 | 289 | static int key_counter = 0; |
290 | if (++key_counter < 0) | |
2cc3f720 | 291 | key_counter = 1; |
007b8be4 | 292 | return key_counter; |
04e8dbaa | 293 | } |
294 | ||
6c57e268 | 295 | void |
b8d8561b | 296 | storeSetPrivateKey(StoreEntry * e) |
227fbb74 | 297 | { |
9fb13bb6 | 298 | const cache_key *newkey; |
299 | MemObject *mem = e->mem_obj; | |
d46a87a8 | 300 | if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE)) |
6eb42cae | 301 | return; /* is already private */ |
b109de6b | 302 | if (e->key) { |
303 | if (e->swap_file_number > -1) | |
304 | storeDirSwapLog(e, SWAP_LOG_DEL); | |
bc0bce21 | 305 | storeHashDelete(e); |
b109de6b | 306 | } |
9fb13bb6 | 307 | if (mem != NULL) { |
007b8be4 | 308 | mem->id = getKeyCounter(); |
309 | newkey = storeKeyPrivate(mem->url, mem->method, mem->id); | |
9fb13bb6 | 310 | } else { |
007b8be4 | 311 | newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); |
9fb13bb6 | 312 | } |
313 | assert(hash_lookup(store_table, newkey) == NULL); | |
d46a87a8 | 314 | EBIT_SET(e->flags, KEY_PRIVATE); |
8638fc66 | 315 | storeHashInsert(e, newkey); |
227fbb74 | 316 | } |
317 | ||
b8d8561b | 318 | void |
319 | storeSetPublicKey(StoreEntry * e) | |
227fbb74 | 320 | { |
6eb42cae | 321 | StoreEntry *e2 = NULL; |
9fb13bb6 | 322 | const cache_key *newkey; |
323 | MemObject *mem = e->mem_obj; | |
d46a87a8 | 324 | if (e->key && !EBIT_TEST(e->flags, KEY_PRIVATE)) |
6eb42cae | 325 | return; /* is already public */ |
9fb13bb6 | 326 | assert(mem); |
f3e570e9 | 327 | /* |
328 | * We can't make RELEASE_REQUEST objects public. Depending on | |
329 | * when RELEASE_REQUEST gets set, we might not be swapping out | |
330 | * the object. If we're not swapping out, then subsequent | |
331 | * store clients won't be able to access object data which has | |
332 | * been freed from memory. | |
d87ebd78 | 333 | * |
334 | * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not | |
335 | * be set, and storeSetPublicKey() should not be called. | |
f3e570e9 | 336 | */ |
d46a87a8 | 337 | assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); |
2ac237e2 | 338 | newkey = storeKeyPublic(mem->url, mem->method); |
9fb13bb6 | 339 | if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { |
340 | debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); | |
07622ccd | 341 | storeSetPrivateKey(e2); |
30a4f2a8 | 342 | storeRelease(e2); |
2ac237e2 | 343 | newkey = storeKeyPublic(mem->url, mem->method); |
6eb42cae | 344 | } |
bc0bce21 | 345 | if (e->key) |
346 | storeHashDelete(e); | |
d46a87a8 | 347 | EBIT_CLR(e->flags, KEY_PRIVATE); |
8638fc66 | 348 | storeHashInsert(e, newkey); |
b109de6b | 349 | if (e->swap_file_number > -1) |
350 | storeDirSwapLog(e, SWAP_LOG_ADD); | |
227fbb74 | 351 | } |
352 | ||
b8d8561b | 353 | StoreEntry * |
92695e5e | 354 | storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method) |
090089c4 | 355 | { |
090089c4 | 356 | StoreEntry *e = NULL; |
30a4f2a8 | 357 | MemObject *mem = NULL; |
92695e5e | 358 | debug(20, 3) ("storeCreateEntry: '%s'\n", url); |
090089c4 | 359 | |
f09f5b26 | 360 | e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url); |
30a4f2a8 | 361 | e->lock_count = 1; /* Note lock here w/o calling storeLock() */ |
362 | mem = e->mem_obj; | |
2ac237e2 | 363 | mem->method = method; |
92695e5e | 364 | if (neighbors_do_private_keys || !flags.hierarchical) |
86101e40 | 365 | storeSetPrivateKey(e); |
366 | else | |
367 | storeSetPublicKey(e); | |
92695e5e | 368 | if (flags.cachable) { |
d46a87a8 | 369 | EBIT_SET(e->flags, ENTRY_CACHABLE); |
370 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
090089c4 | 371 | } else { |
d46a87a8 | 372 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
2daae136 | 373 | storeReleaseRequest(e); |
090089c4 | 374 | } |
234967c9 | 375 | e->store_status = STORE_PENDING; |
090089c4 | 376 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 377 | e->swap_status = SWAPOUT_NONE; |
090089c4 | 378 | e->swap_file_number = -1; |
090089c4 | 379 | e->refcount = 0; |
b8de7ebe | 380 | e->lastref = squid_curtime; |
ca98227c | 381 | e->timestamp = 0; /* set in storeTimestampsSet() */ |
30a4f2a8 | 382 | e->ping_status = PING_NONE; |
d46a87a8 | 383 | EBIT_SET(e->flags, ENTRY_VALIDATED); |
090089c4 | 384 | return e; |
385 | } | |
386 | ||
6eb42cae | 387 | /* Mark object as expired */ |
b8d8561b | 388 | void |
389 | storeExpireNow(StoreEntry * e) | |
9174e204 | 390 | { |
9fb13bb6 | 391 | debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->key)); |
b8de7ebe | 392 | e->expires = squid_curtime; |
9174e204 | 393 | } |
394 | ||
090089c4 | 395 | /* Append incoming data from a primary server to an entry. */ |
b8d8561b | 396 | void |
b8014333 | 397 | storeAppend(StoreEntry * e, const char *buf, int len) |
090089c4 | 398 | { |
3a1c3e2f | 399 | MemObject *mem = e->mem_obj; |
400 | assert(mem != NULL); | |
401 | assert(len >= 0); | |
789f4e81 | 402 | assert(e->store_status == STORE_PENDING); |
090089c4 | 403 | if (len) { |
d0e2935f | 404 | debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", |
405 | len, | |
406 | storeKeyText(e->key)); | |
38792624 | 407 | storeGetMemSpace(len); |
18fe65d0 | 408 | stmemAppend(&mem->data_hdr, buf, len); |
8350fe9b | 409 | mem->inmem_hi += len; |
090089c4 | 410 | } |
d46a87a8 | 411 | if (EBIT_TEST(e->flags, DELAY_SENDING)) |
d0e2935f | 412 | return; |
9d66d521 | 413 | #ifdef OPTIMISTIC_IO |
414 | storeLockObject(e); | |
415 | #endif | |
d0e2935f | 416 | InvokeHandlers(e); |
8350fe9b | 417 | storeCheckSwapOut(e); |
9d66d521 | 418 | #ifdef OPTIMISTIC_IO |
419 | storeUnlockObject(e); | |
420 | #endif | |
090089c4 | 421 | } |
422 | ||
6de2df60 | 423 | #if STDC_HEADERS |
b8d8561b | 424 | void |
fe4e214f | 425 | storeAppendPrintf(StoreEntry * e, const char *fmt,...) |
c30c5a73 | 426 | { |
15c05bb0 | 427 | va_list args; |
15c05bb0 | 428 | va_start(args, fmt); |
c30c5a73 | 429 | #else |
b8d8561b | 430 | void |
431 | storeAppendPrintf(va_alist) | |
15c05bb0 | 432 | va_dcl |
433 | { | |
434 | va_list args; | |
435 | StoreEntry *e = NULL; | |
0ee4272b | 436 | const char *fmt = NULL; |
15c05bb0 | 437 | va_start(args); |
438 | e = va_arg(args, StoreEntry *); | |
439 | fmt = va_arg(args, char *); | |
c30c5a73 | 440 | #endif |
cb69b4c7 | 441 | storeAppendVPrintf(e, fmt, args); |
442 | va_end(args); | |
443 | } | |
444 | ||
445 | /* used be storeAppendPrintf and Packer */ | |
446 | void | |
447 | storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs) | |
448 | { | |
449 | LOCAL_ARRAY(char, buf, 4096); | |
15c05bb0 | 450 | buf[0] = '\0'; |
cb69b4c7 | 451 | vsnprintf(buf, 4096, fmt, vargs); |
15c05bb0 | 452 | storeAppend(e, buf, strlen(buf)); |
c30c5a73 | 453 | } |
454 | ||
8423ff74 | 455 | struct _store_check_cachable_hist { |
456 | struct { | |
457 | int non_get; | |
458 | int not_entry_cachable; | |
459 | int release_request; | |
460 | int wrong_content_length; | |
461 | int negative_cached; | |
462 | int too_big; | |
463 | int private_key; | |
c5f627c2 | 464 | int too_many_open_files; |
59ffcdf8 | 465 | int too_many_open_fds; |
8423ff74 | 466 | int lru_age_too_low; |
467 | } no; | |
468 | struct { | |
469 | int Default; | |
470 | } yes; | |
471 | } store_check_cachable_hist; | |
472 | ||
c47511fd | 473 | int |
474 | storeTooManyDiskFilesOpen(void) | |
475 | { | |
476 | if (Config.max_open_disk_fds == 0) | |
477 | return 0; | |
478 | if (open_disk_fd > Config.max_open_disk_fds) | |
479 | return 1; | |
480 | return 0; | |
481 | } | |
482 | ||
f09f5b26 | 483 | int |
8350fe9b | 484 | storeCheckCachable(StoreEntry * e) |
6602e70e | 485 | { |
2ac237e2 | 486 | #if CACHE_ALL_METHODS |
487 | if (e->mem_obj->method != METHOD_GET) { | |
8350fe9b | 488 | debug(20, 2) ("storeCheckCachable: NO: non-GET method\n"); |
8423ff74 | 489 | store_check_cachable_hist.no.non_get++; |
2ac237e2 | 490 | } else |
491 | #endif | |
d46a87a8 | 492 | if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) { |
8350fe9b | 493 | debug(20, 2) ("storeCheckCachable: NO: not cachable\n"); |
8423ff74 | 494 | store_check_cachable_hist.no.not_entry_cachable++; |
d46a87a8 | 495 | } else if (EBIT_TEST(e->flags, RELEASE_REQUEST)) { |
8350fe9b | 496 | debug(20, 2) ("storeCheckCachable: NO: release requested\n"); |
8423ff74 | 497 | store_check_cachable_hist.no.release_request++; |
d46a87a8 | 498 | } else if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) { |
8350fe9b | 499 | debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n"); |
8423ff74 | 500 | store_check_cachable_hist.no.wrong_content_length++; |
d46a87a8 | 501 | } else if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) { |
c0dee43f | 502 | debug(20, 3) ("storeCheckCachable: NO: negative cached\n"); |
8423ff74 | 503 | store_check_cachable_hist.no.negative_cached++; |
3e98df20 | 504 | return 0; /* avoid release call below */ |
8350fe9b | 505 | } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { |
506 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); | |
8423ff74 | 507 | store_check_cachable_hist.no.too_big++; |
d46a87a8 | 508 | } else if (EBIT_TEST(e->flags, KEY_PRIVATE)) { |
8350fe9b | 509 | debug(20, 3) ("storeCheckCachable: NO: private key\n"); |
8423ff74 | 510 | store_check_cachable_hist.no.private_key++; |
c47511fd | 511 | } else if (storeTooManyDiskFilesOpen()) { |
c5f627c2 | 512 | debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n"); |
513 | store_check_cachable_hist.no.too_many_open_files++; | |
59ffcdf8 | 514 | } else if (fdNFree() < RESERVED_FD) { |
515 | debug(20, 2) ("storeCheckCachable: NO: too FD's open\n"); | |
516 | store_check_cachable_hist.no.too_many_open_fds++; | |
734b57eb | 517 | } else if (storeExpiredReferenceAge() < 300) { |
c8b09b26 | 518 | debug(20, 2) ("storeCheckCachable: NO: LRU Age = %d\n", |
519 | storeExpiredReferenceAge()); | |
8423ff74 | 520 | store_check_cachable_hist.no.lru_age_too_low++; |
d4432957 | 521 | } else { |
8423ff74 | 522 | store_check_cachable_hist.yes.Default++; |
6602e70e | 523 | return 1; |
d4432957 | 524 | } |
2daae136 | 525 | storeReleaseRequest(e); |
d46a87a8 | 526 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
6602e70e | 527 | return 0; |
528 | } | |
529 | ||
8423ff74 | 530 | static void |
531 | storeCheckCachableStats(StoreEntry * sentry) | |
532 | { | |
c40acff3 | 533 | storeAppendPrintf(sentry, "Category\t Count\n"); |
534 | ||
8423ff74 | 535 | storeAppendPrintf(sentry, "no.non_get\t%d\n", |
536 | store_check_cachable_hist.no.non_get); | |
537 | storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n", | |
538 | store_check_cachable_hist.no.not_entry_cachable); | |
539 | storeAppendPrintf(sentry, "no.release_request\t%d\n", | |
540 | store_check_cachable_hist.no.release_request); | |
541 | storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n", | |
542 | store_check_cachable_hist.no.wrong_content_length); | |
543 | storeAppendPrintf(sentry, "no.negative_cached\t%d\n", | |
544 | store_check_cachable_hist.no.negative_cached); | |
545 | storeAppendPrintf(sentry, "no.too_big\t%d\n", | |
546 | store_check_cachable_hist.no.too_big); | |
547 | storeAppendPrintf(sentry, "no.private_key\t%d\n", | |
548 | store_check_cachable_hist.no.private_key); | |
c5f627c2 | 549 | storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n", |
550 | store_check_cachable_hist.no.too_many_open_files); | |
59ffcdf8 | 551 | storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n", |
552 | store_check_cachable_hist.no.too_many_open_fds); | |
8423ff74 | 553 | storeAppendPrintf(sentry, "no.lru_age_too_low\t%d\n", |
554 | store_check_cachable_hist.no.lru_age_too_low); | |
555 | storeAppendPrintf(sentry, "yes.default\t%d\n", | |
556 | store_check_cachable_hist.yes.Default); | |
557 | } | |
558 | ||
090089c4 | 559 | /* Complete transfer into the local cache. */ |
b8d8561b | 560 | void |
561 | storeComplete(StoreEntry * e) | |
090089c4 | 562 | { |
9fb13bb6 | 563 | debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key)); |
22955fba | 564 | assert(e->store_status == STORE_PENDING); |
07304bf9 | 565 | e->mem_obj->object_sz = e->mem_obj->inmem_hi; |
234967c9 | 566 | e->store_status = STORE_OK; |
8350fe9b | 567 | assert(e->mem_status == NOT_IN_MEMORY); |
41587298 | 568 | if (!storeEntryValidLength(e)) { |
d46a87a8 | 569 | EBIT_SET(e->flags, ENTRY_BAD_LENGTH); |
41587298 | 570 | storeReleaseRequest(e); |
571 | } | |
6cfa8966 | 572 | #if USE_CACHE_DIGESTS |
544b1fd4 | 573 | if (e->mem_obj->request) |
574 | e->mem_obj->request->hier.store_complete_stop = current_time; | |
39edba21 | 575 | #endif |
3a1c3e2f | 576 | InvokeHandlers(e); |
8c123b71 | 577 | storeCheckSwapOut(e); |
7e3e1d01 | 578 | } |
579 | ||
090089c4 | 580 | /* |
474cac1b | 581 | * Someone wants to abort this transfer. Set the reason in the |
582 | * request structure, call the server-side callback and mark the | |
583 | * entry for releasing | |
090089c4 | 584 | */ |
b8d8561b | 585 | void |
9b312a19 | 586 | storeAbort(StoreEntry * e, int cbflag) |
090089c4 | 587 | { |
3e98df20 | 588 | MemObject *mem = e->mem_obj; |
6801f8a8 | 589 | STABH *callback; |
590 | void *data; | |
8f39b81d | 591 | assert(e->store_status == STORE_PENDING); |
592 | assert(mem != NULL); | |
9fb13bb6 | 593 | debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->key)); |
3d02186d | 594 | storeLockObject(e); /* lock while aborting */ |
79b5cc5f | 595 | storeNegativeCache(e); |
474cac1b | 596 | storeReleaseRequest(e); |
b7fe0ab0 | 597 | EBIT_SET(e->flags, ENTRY_ABORTED); |
8350fe9b | 598 | storeSetMemStatus(e, NOT_IN_MEMORY); |
090089c4 | 599 | /* No DISK swap for negative cached object */ |
8350fe9b | 600 | e->swap_status = SWAPOUT_NONE; |
789f4e81 | 601 | e->store_status = STORE_OK; |
496e1d76 | 602 | /* |
603 | * We assign an object length here. The only other place we assign | |
604 | * the object length is in storeComplete() | |
605 | */ | |
07304bf9 | 606 | mem->object_sz = mem->inmem_hi; |
474cac1b | 607 | /* Notify the server side */ |
608 | if (cbflag && mem->abort.callback) { | |
6801f8a8 | 609 | callback = mem->abort.callback; |
610 | data = mem->abort.data; | |
bfcaf585 | 611 | mem->abort.callback = NULL; |
6801f8a8 | 612 | mem->abort.data = NULL; |
613 | callback(data); | |
bfcaf585 | 614 | } |
474cac1b | 615 | /* Notify the client side */ |
090089c4 | 616 | InvokeHandlers(e); |
6e86c3e8 | 617 | /* Do we need to close the swapout file? */ |
618 | /* Not if we never started swapping out */ | |
6cf028ab | 619 | /* But we may need to cancel an open/stat in progress if using ASYNC */ |
620 | #if USE_ASYNC_IO | |
621 | aioCancel(-1, e); | |
622 | #endif | |
3d02186d | 623 | if (e->swap_file_number > -1) { |
6cf028ab | 624 | #if USE_ASYNC_IO |
3d02186d | 625 | /* Need to cancel any pending ASYNC writes right now */ |
626 | if (mem->swapout.fd >= 0) | |
627 | aioCancel(mem->swapout.fd, NULL); | |
6cf028ab | 628 | #endif |
3d02186d | 629 | /* we have to close the disk file if there is no write pending */ |
61038223 | 630 | if (!storeSwapOutWriteQueued(mem)) |
3d02186d | 631 | storeSwapOutFileClose(e); |
632 | } | |
633 | storeUnlockObject(e); /* unlock */ | |
090089c4 | 634 | } |
635 | ||
090089c4 | 636 | /* Clear Memory storage to accommodate the given object len */ |
38792624 | 637 | static void |
d4432957 | 638 | storeGetMemSpace(int size) |
090089c4 | 639 | { |
b32508fb | 640 | StoreEntry *e = NULL; |
20cba4b4 | 641 | int released = 0; |
b32508fb | 642 | static time_t last_check = 0; |
e954773d | 643 | int pages_needed; |
b93bcace | 644 | dlink_node *m; |
d6ea85ce | 645 | dlink_node *head; |
79d39a72 | 646 | dlink_node *prev = NULL; |
b32508fb | 647 | if (squid_curtime == last_check) |
38792624 | 648 | return; |
b32508fb | 649 | last_check = squid_curtime; |
e954773d | 650 | pages_needed = (size / SM_PAGE_SIZE) + 1; |
3f6c0fb2 | 651 | if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_high) |
38792624 | 652 | return; |
311ea387 | 653 | if (store_rebuilding) |
38792624 | 654 | return; |
a3d5953d | 655 | debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed); |
d6ea85ce | 656 | head = inmem_list.head; |
b93bcace | 657 | for (m = inmem_list.tail; m; m = prev) { |
d6ea85ce | 658 | if (m == head) |
659 | break; | |
8350fe9b | 660 | prev = m->prev; |
b93bcace | 661 | e = m->data; |
d6ea85ce | 662 | if (storeEntryLocked(e)) { |
5999b776 | 663 | dlinkDelete(m, &inmem_list); |
664 | dlinkAdd(e, m, &inmem_list); | |
8350fe9b | 665 | continue; |
d6ea85ce | 666 | } |
20cba4b4 | 667 | released++; |
2d51a60e | 668 | storePurgeMem(e); |
d6ea85ce | 669 | if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_high) |
8350fe9b | 670 | break; |
671 | } | |
5e3d4fef | 672 | debug(20, 3) ("storeGetMemSpace stats:\n"); |
59c4d35b | 673 | debug(20, 3) (" %6d HOT objects\n", hot_obj_count); |
20cba4b4 | 674 | debug(20, 3) (" %6d were released\n", released); |
090089c4 | 675 | } |
676 | ||
090089c4 | 677 | /* The maximum objects to scan for maintain storage space */ |
fcefe642 | 678 | #define MAINTAIN_MAX_SCAN 1024 |
679 | #define MAINTAIN_MAX_REMOVE 64 | |
090089c4 | 680 | |
fcefe642 | 681 | /* |
682 | * This routine is to be called by main loop in main.c. | |
683 | * It removes expired objects on only one bucket for each time called. | |
684 | * returns the number of objects removed | |
685 | * | |
686 | * This should get called 1/s from main(). | |
687 | */ | |
679ac4f0 | 688 | void |
79d39a72 | 689 | storeMaintainSwapSpace(void *datanotused) |
090089c4 | 690 | { |
b93bcace | 691 | dlink_node *m; |
79d39a72 | 692 | dlink_node *prev = NULL; |
a1b0d7cf | 693 | StoreEntry *e = NULL; |
090089c4 | 694 | int scanned = 0; |
090089c4 | 695 | int locked = 0; |
679ac4f0 | 696 | int expired = 0; |
20cba4b4 | 697 | int max_scan; |
698 | int max_remove; | |
ec14f197 | 699 | static time_t last_warn_time = 0; |
fcefe642 | 700 | /* We can't delete objects while rebuilding swap */ |
c8b09b26 | 701 | if (store_rebuilding) { |
702 | eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); | |
679ac4f0 | 703 | return; |
d1497906 | 704 | } else if (store_swap_size < store_swap_mid) { |
705 | max_scan = 100; | |
706 | max_remove = 8; | |
707 | eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); | |
c8b09b26 | 708 | } else if (store_swap_size < store_swap_high) { |
709 | max_scan = 200; | |
710 | max_remove = 8; | |
711 | eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.1, 1); | |
20cba4b4 | 712 | } else { |
4cb56589 | 713 | max_scan = 500; |
c8b09b26 | 714 | max_remove = 32; |
715 | eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 0.0, 1); | |
20cba4b4 | 716 | } |
fcefe642 | 717 | debug(20, 3) ("storeMaintainSwapSpace\n"); |
e3ef2b09 | 718 | for (m = store_list.tail; m; m = prev) { |
b93bcace | 719 | prev = m->prev; |
720 | e = m->data; | |
d150898a | 721 | scanned++; |
b93bcace | 722 | if (storeEntryLocked(e)) { |
ac2197dc | 723 | /* |
724 | * If there is a locked entry at the tail of the LRU list, | |
725 | * move it to the beginning to get it out of the way. | |
726 | * Theoretically, we might have all locked objects at the | |
727 | * tail, and then we'll never remove anything here and the | |
728 | * LRU age will go to zero. | |
729 | */ | |
730 | if (memInUse(MEM_STOREENTRY) > max_scan) { | |
4b4cd312 | 731 | dlinkDelete(&e->lru, &store_list); |
732 | dlinkAdd(e, &e->lru, &store_list); | |
ac2197dc | 733 | } |
b93bcace | 734 | locked++; |
01fe9bf4 | 735 | } else if (storeCheckExpired(e)) { |
b109de6b | 736 | expired++; |
737 | storeRelease(e); | |
090089c4 | 738 | } |
d150898a | 739 | if (expired >= max_remove) |
b93bcace | 740 | break; |
d150898a | 741 | if (scanned >= max_scan) |
b93bcace | 742 | break; |
090089c4 | 743 | } |
4cb56589 | 744 | debug(20, 3) ("storeMaintainSwapSpace stats:\n"); |
745 | debug(20, 3) (" %6d objects\n", memInUse(MEM_STOREENTRY)); | |
746 | debug(20, 3) (" %6d were scanned\n", scanned); | |
747 | debug(20, 3) (" %6d were locked\n", locked); | |
748 | debug(20, 3) (" %6d were expired\n", expired); | |
fcefe642 | 749 | if (store_swap_size < Config.Swap.maxSize) |
b93bcace | 750 | return; |
751 | if (squid_curtime - last_warn_time < 10) | |
752 | return; | |
753 | debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", | |
754 | store_swap_size, Config.Swap.maxSize); | |
755 | last_warn_time = squid_curtime; | |
090089c4 | 756 | } |
757 | ||
758 | ||
759 | /* release an object from a cache */ | |
cc61958c | 760 | /* return number of objects released. */ |
6c78a099 | 761 | void |
b8d8561b | 762 | storeRelease(StoreEntry * e) |
090089c4 | 763 | { |
9fb13bb6 | 764 | debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->key)); |
090089c4 | 765 | /* If, for any reason we can't discard this object because of an |
766 | * outstanding request, mark it for pending release */ | |
767 | if (storeEntryLocked(e)) { | |
9174e204 | 768 | storeExpireNow(e); |
a3d5953d | 769 | debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n"); |
2daae136 | 770 | storeReleaseRequest(e); |
6c78a099 | 771 | return; |
090089c4 | 772 | } |
77059ddf | 773 | #if USE_ASYNC_IO |
774 | /* | |
775 | * Make sure all forgotten async ops are cancelled | |
776 | */ | |
777 | aioCancel(-1, e); | |
6cf028ab | 778 | #endif |
e42d5181 | 779 | if (store_rebuilding) { |
780 | storeSetPrivateKey(e); | |
781 | if (e->mem_obj) { | |
782 | storeSetMemStatus(e, NOT_IN_MEMORY); | |
783 | destroy_MemObject(e); | |
784 | } | |
785 | /* | |
786 | * Fake a call to storeLockObject(). When rebuilding is done, | |
787 | * we'll just call storeUnlockObject() on these. | |
788 | */ | |
789 | e->lock_count++; | |
790 | stackPush(&LateReleaseStack, e); | |
791 | return; | |
792 | } | |
a65ff22e | 793 | storeLog(STORE_LOG_RELEASE, e); |
43fee0b8 | 794 | if (e->swap_file_number > -1) { |
25535cbe | 795 | storeUnlinkFileno(e->swap_file_number); |
d961be0a | 796 | storeDirMapBitReset(e->swap_file_number); |
a65ff22e | 797 | if (e->swap_status == SWAPOUT_DONE) |
d46a87a8 | 798 | if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) |
7b3d6868 | 799 | storeDirUpdateSwapSize(e->swap_file_number, e->swap_file_sz, -1); |
d46a87a8 | 800 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) |
b109de6b | 801 | storeDirSwapLog(e, SWAP_LOG_DEL); |
090089c4 | 802 | } |
8350fe9b | 803 | storeSetMemStatus(e, NOT_IN_MEMORY); |
30a4f2a8 | 804 | destroy_StoreEntry(e); |
090089c4 | 805 | } |
806 | ||
e42d5181 | 807 | static void |
808 | storeLateRelease(void *unused) | |
809 | { | |
810 | StoreEntry *e; | |
811 | int i; | |
812 | static int n = 0; | |
813 | if (store_rebuilding) { | |
814 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); | |
815 | return; | |
816 | } | |
817 | for (i = 0; i < 10; i++) { | |
818 | e = stackPop(&LateReleaseStack); | |
819 | if (e == NULL) { | |
820 | /* done! */ | |
821 | debug(20, 0) ("storeLateRelease: released %d objects\n", n); | |
822 | return; | |
823 | } | |
824 | storeUnlockObject(e); | |
825 | n++; | |
826 | } | |
827 | eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1); | |
828 | } | |
829 | ||
090089c4 | 830 | /* return 1 if a store entry is locked */ |
24382924 | 831 | static int |
fe4e214f | 832 | storeEntryLocked(const StoreEntry * e) |
090089c4 | 833 | { |
30a4f2a8 | 834 | if (e->lock_count) |
835 | return 1; | |
6cf028ab | 836 | if (e->swap_status == SWAPOUT_OPENING) |
837 | return 1; | |
8350fe9b | 838 | if (e->swap_status == SWAPOUT_WRITING) |
30a4f2a8 | 839 | return 1; |
a1e47288 | 840 | if (e->store_status == STORE_PENDING) |
841 | return 1; | |
d46a87a8 | 842 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) |
365cb147 | 843 | return 1; |
30a4f2a8 | 844 | return 0; |
090089c4 | 845 | } |
846 | ||
24382924 | 847 | static int |
fe4e214f | 848 | storeEntryValidLength(const StoreEntry * e) |
6602e70e | 849 | { |
ffe4a367 | 850 | int diff; |
d8b249ef | 851 | const HttpReply *reply; |
8a5e92ce | 852 | assert(e->mem_obj != NULL); |
07304bf9 | 853 | reply = e->mem_obj->reply; |
9fb13bb6 | 854 | debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key)); |
07304bf9 | 855 | debug(20, 5) ("storeEntryValidLength: object_len = %d\n", |
856 | objectLen(e)); | |
857 | debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", | |
858 | reply->hdr_sz); | |
859 | debug(20, 5) ("storeEntryValidLength: content_length = %d\n", | |
d8b249ef | 860 | reply->content_length); |
861 | if (reply->content_length < 0) { | |
1790d392 | 862 | debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", |
9fb13bb6 | 863 | storeKeyText(e->key)); |
ffe4a367 | 864 | return 1; |
865 | } | |
07304bf9 | 866 | if (reply->hdr_sz == 0) { |
1790d392 | 867 | debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", |
9fb13bb6 | 868 | storeKeyText(e->key)); |
ffe4a367 | 869 | return 1; |
870 | } | |
ebf4efff | 871 | if (e->mem_obj->method == METHOD_HEAD) { |
872 | debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n", | |
9fb13bb6 | 873 | storeKeyText(e->key)); |
ebf4efff | 874 | return 1; |
ffe4a367 | 875 | } |
cb69b4c7 | 876 | if (reply->sline.status == HTTP_NOT_MODIFIED) |
ebf4efff | 877 | return 1; |
cb69b4c7 | 878 | if (reply->sline.status == HTTP_NO_CONTENT) |
ebf4efff | 879 | return 1; |
d8b249ef | 880 | diff = reply->hdr_sz + reply->content_length - objectLen(e); |
ebf4efff | 881 | if (diff == 0) |
882 | return 1; | |
883 | debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", | |
884 | diff < 0 ? -diff : diff, | |
e69324cd | 885 | diff < 0 ? "big" : "small", |
ebf4efff | 886 | storeKeyText(e->key)); |
887 | return 0; | |
ffe4a367 | 888 | } |
6602e70e | 889 | |
66cedb85 | 890 | static void |
891 | storeInitHashValues(void) | |
892 | { | |
893 | int i; | |
a7e59001 | 894 | /* Calculate size of hash table (maximum currently 64k buckets). */ |
38792624 | 895 | i = Config.Swap.maxSize / Config.Store.avgObjectSize; |
86101e40 | 896 | debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n", |
66cedb85 | 897 | Config.Swap.maxSize, i); |
38792624 | 898 | i /= Config.Store.objectsPerBucket; |
a3d5953d | 899 | debug(20, 1) ("Target number of buckets: %d\n", i); |
66cedb85 | 900 | /* ideally the full scan period should be configurable, for the |
901 | * moment it remains at approximately 24 hours. */ | |
9fb13bb6 | 902 | store_hash_buckets = storeKeyHashBuckets(i); |
24ffafb4 | 903 | store_maintain_rate = 86400 / store_hash_buckets; |
9fb13bb6 | 904 | assert(store_maintain_rate > 0); |
7454a5c2 | 905 | debug(20, 1) ("Using %d Store buckets, replacement runs every %d second%s\n", |
9fb13bb6 | 906 | store_hash_buckets, |
66cedb85 | 907 | store_maintain_rate, |
908 | store_maintain_rate == 1 ? null_string : "s"); | |
a47b9029 | 909 | debug(20, 1) ("Max Mem size: %d KB\n", Config.Mem.maxSize >> 10); |
910 | debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize); | |
66cedb85 | 911 | } |
912 | ||
b8d8561b | 913 | void |
914 | storeInit(void) | |
c943f331 | 915 | { |
25535cbe | 916 | storeKeyInit(); |
66cedb85 | 917 | storeInitHashValues(); |
9fb13bb6 | 918 | store_table = hash_create(storeKeyHashCmp, |
919 | store_hash_buckets, storeKeyHashHash); | |
8638fc66 | 920 | storeDigestInit(); |
e3ef2b09 | 921 | storeLogOpen(); |
85407535 | 922 | if (storeVerifyCacheDirs() < 0) { |
d377699f | 923 | xstrncpy(tmp_error_buf, |
924 | "\tFailed to verify one of the swap directories, Check cache.log\n" | |
925 | "\tfor details. Run 'squid -z' to create swap directories\n" | |
926 | "\tif needed, or if running Squid for the first time.", | |
927 | ERROR_BUF_SZ); | |
928 | fatal(tmp_error_buf); | |
929 | } | |
e3ef2b09 | 930 | storeDirOpenSwapLogs(); |
e3ef2b09 | 931 | store_list.head = store_list.tail = NULL; |
b93bcace | 932 | inmem_list.head = inmem_list.tail = NULL; |
e42d5181 | 933 | stackInit(&LateReleaseStack); |
934 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); | |
5830cdb3 | 935 | storeRebuildStart(); |
4cb56589 | 936 | cachemgrRegister("storedir", |
22f3fd98 | 937 | "Store Directory Stats", |
1da3b90b | 938 | storeDirStats, 0, 1); |
8423ff74 | 939 | cachemgrRegister("store_check_cachable_stats", |
940 | "storeCheckCachable() Stats", | |
941 | storeCheckCachableStats, 0, 1); | |
b1c0cc67 | 942 | } |
c943f331 | 943 | |
b8d8561b | 944 | void |
945 | storeConfigure(void) | |
b1c0cc67 | 946 | { |
e954773d | 947 | int store_mem_high = 0; |
948 | int store_mem_low = 0; | |
b6f794d6 | 949 | store_mem_high = (long) (Config.Mem.maxSize / 100) * |
950 | Config.Mem.highWaterMark; | |
951 | store_mem_low = (long) (Config.Mem.maxSize / 100) * | |
952 | Config.Mem.lowWaterMark; | |
090089c4 | 953 | |
b1c0cc67 | 954 | store_swap_high = (long) (((float) Config.Swap.maxSize * |
955 | (float) Config.Swap.highWaterMark) / (float) 100); | |
956 | store_swap_low = (long) (((float) Config.Swap.maxSize * | |
957 | (float) Config.Swap.lowWaterMark) / (float) 100); | |
d1497906 | 958 | store_swap_mid = (store_swap_high >> 1) + (store_swap_low >> 1); |
e954773d | 959 | |
960 | store_pages_high = store_mem_high / SM_PAGE_SIZE; | |
961 | store_pages_low = store_mem_low / SM_PAGE_SIZE; | |
090089c4 | 962 | } |
963 | ||
56f29785 | 964 | static int |
8350fe9b | 965 | storeKeepInMemory(const StoreEntry * e) |
56f29785 | 966 | { |
8350fe9b | 967 | MemObject *mem = e->mem_obj; |
968 | if (mem == NULL) | |
56f29785 | 969 | return 0; |
18fe65d0 | 970 | if (mem->data_hdr.head == NULL) |
8350fe9b | 971 | return 0; |
972 | return mem->inmem_lo == 0; | |
56f29785 | 973 | } |
974 | ||
b8d8561b | 975 | static int |
01fe9bf4 | 976 | storeCheckExpired(const StoreEntry * e) |
620da955 | 977 | { |
978 | if (storeEntryLocked(e)) | |
979 | return 0; | |
d46a87a8 | 980 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
1148b77c | 981 | return 1; |
d46a87a8 | 982 | if (EBIT_TEST(e->flags, ENTRY_NEGCACHED) && squid_curtime >= e->expires) |
7cc1830f | 983 | return 1; |
fcefe642 | 984 | if (squid_curtime - e->lastref > storeExpiredReferenceAge()) |
66cedb85 | 985 | return 1; |
66cedb85 | 986 | return 0; |
620da955 | 987 | } |
988 | ||
db9ccd5a | 989 | /* |
990 | * storeExpiredReferenceAge | |
991 | * | |
429fdbec | 992 | * The LRU age is scaled exponentially between 1 minute and |
993 | * Config.referenceAge , when store_swap_low < store_swap_size < | |
994 | * store_swap_high. This keeps store_swap_size within the low and high | |
995 | * water marks. If the cache is very busy then store_swap_size stays | |
996 | * closer to the low water mark, if it is not busy, then it will stay | |
997 | * near the high water mark. The LRU age value can be examined on the | |
998 | * cachemgr 'info' page. | |
fbdfb3cf | 999 | */ |
35feb4aa | 1000 | time_t |
58104eab | 1001 | storeExpiredReferenceAge(void) |
1002 | { | |
58104eab | 1003 | double x; |
1004 | double z; | |
1005 | time_t age; | |
429fdbec | 1006 | x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low); |
1007 | x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; | |
8ac49cdc | 1008 | z = pow((double) (Config.referenceAge / 60), x); |
58104eab | 1009 | age = (time_t) (z * 60.0); |
1010 | if (age < 60) | |
1011 | age = 60; | |
1012 | else if (age > 31536000) | |
1013 | age = 31536000; | |
1014 | return age; | |
0a362c34 | 1015 | } |
2edc2504 | 1016 | |
b8d8561b | 1017 | void |
1018 | storeNegativeCache(StoreEntry * e) | |
79b5cc5f | 1019 | { |
ff5731b1 | 1020 | e->expires = squid_curtime + Config.negativeTtl; |
d46a87a8 | 1021 | EBIT_SET(e->flags, ENTRY_NEGCACHED); |
79b5cc5f | 1022 | } |
0a21bd84 | 1023 | |
1024 | void | |
1025 | storeFreeMemory(void) | |
1026 | { | |
ec878047 | 1027 | hashFreeItems(store_table, destroy_StoreEntry); |
0a21bd84 | 1028 | hashFreeMemory(store_table); |
afe95a7e | 1029 | store_table = NULL; |
c68e9c6b | 1030 | #if USE_CACHE_DIGEST |
8638fc66 | 1031 | if (store_digest) |
1032 | cacheDigestDestroy(store_digest); | |
c68e9c6b | 1033 | #endif |
8638fc66 | 1034 | store_digest = NULL; |
0a21bd84 | 1035 | } |
a7e59001 | 1036 | |
1037 | int | |
1038 | expiresMoreThan(time_t expires, time_t when) | |
1039 | { | |
48f44632 | 1040 | if (expires < 0) /* No Expires given */ |
1041 | return 1; | |
1042 | return (expires > (squid_curtime + when)); | |
a7e59001 | 1043 | } |
fe54d06d | 1044 | |
1045 | int | |
1046 | storeEntryValidToSend(StoreEntry * e) | |
1047 | { | |
d46a87a8 | 1048 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
fe54d06d | 1049 | return 0; |
d46a87a8 | 1050 | if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) |
44f78c24 | 1051 | if (e->expires <= squid_curtime) |
fe54d06d | 1052 | return 0; |
b7fe0ab0 | 1053 | if (EBIT_TEST(e->flags, ENTRY_ABORTED)) |
fe54d06d | 1054 | return 0; |
1055 | return 1; | |
1056 | } | |
62663274 | 1057 | |
ca98227c | 1058 | void |
1059 | storeTimestampsSet(StoreEntry * entry) | |
1060 | { | |
1061 | time_t served_date = -1; | |
2246b732 | 1062 | const HttpReply *reply = entry->mem_obj->reply; |
d8b249ef | 1063 | served_date = reply->date; |
cb69b4c7 | 1064 | if (served_date < 0) |
1065 | served_date = squid_curtime; | |
d8b249ef | 1066 | entry->expires = reply->expires; |
1c3e77cd | 1067 | entry->lastmod = reply->last_modified; |
ca98227c | 1068 | entry->timestamp = served_date; |
1069 | } | |
429fdbec | 1070 | |
bfcaf585 | 1071 | void |
1072 | storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) | |
1073 | { | |
1074 | MemObject *mem = e->mem_obj; | |
1075 | assert(mem); | |
1076 | assert(mem->abort.callback == NULL); | |
1077 | mem->abort.callback = cb; | |
1078 | mem->abort.data = data; | |
1079 | } | |
1080 | ||
1081 | void | |
1082 | storeUnregisterAbort(StoreEntry * e) | |
1083 | { | |
1084 | MemObject *mem = e->mem_obj; | |
1085 | assert(mem); | |
1086 | mem->abort.callback = NULL; | |
1087 | } | |
88738790 | 1088 | |
1089 | void | |
1090 | storeMemObjectDump(MemObject * mem) | |
1091 | { | |
18fe65d0 | 1092 | debug(20, 1) ("MemObject->data.head: %p\n", |
1093 | mem->data_hdr.head); | |
1094 | debug(20, 1) ("MemObject->data.tail: %p\n", | |
1095 | mem->data_hdr.tail); | |
1096 | debug(20, 1) ("MemObject->data.origin_offset: %d\n", | |
1097 | mem->data_hdr.origin_offset); | |
88738790 | 1098 | debug(20, 1) ("MemObject->start_ping: %d.%06d\n", |
5f6ac48b | 1099 | (int) mem->start_ping.tv_sec, |
1100 | (int) mem->start_ping.tv_usec); | |
8350fe9b | 1101 | debug(20, 1) ("MemObject->inmem_hi: %d\n", |
5f6ac48b | 1102 | (int) mem->inmem_hi); |
8350fe9b | 1103 | debug(20, 1) ("MemObject->inmem_lo: %d\n", |
5f6ac48b | 1104 | (int) mem->inmem_lo); |
88738790 | 1105 | debug(20, 1) ("MemObject->clients: %p\n", |
1106 | mem->clients); | |
1107 | debug(20, 1) ("MemObject->nclients: %d\n", | |
1108 | mem->nclients); | |
8350fe9b | 1109 | debug(20, 1) ("MemObject->swapout.fd: %d\n", |
1110 | mem->swapout.fd); | |
88738790 | 1111 | debug(20, 1) ("MemObject->reply: %p\n", |
1112 | mem->reply); | |
1113 | debug(20, 1) ("MemObject->request: %p\n", | |
1114 | mem->request); | |
1115 | debug(20, 1) ("MemObject->log_url: %p %s\n", | |
1116 | mem->log_url, | |
1117 | checkNullString(mem->log_url)); | |
1118 | } | |
8350fe9b | 1119 | |
f09f5b26 | 1120 | void |
e3ef2b09 | 1121 | storeEntryDump(StoreEntry * e, int l) |
d377699f | 1122 | { |
e3ef2b09 | 1123 | debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->key)); |
1124 | debug(20, l) ("StoreEntry->next: %p\n", e->next); | |
1125 | debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj); | |
1126 | debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp); | |
1127 | debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); | |
1128 | debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires); | |
1129 | debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod); | |
07304bf9 | 1130 | debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz); |
e3ef2b09 | 1131 | debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount); |
415e0dd2 | 1132 | debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e)); |
e3ef2b09 | 1133 | debug(20, l) ("StoreEntry->swap_file_number: %d\n", (int) e->swap_file_number); |
1134 | debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count); | |
1135 | debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status); | |
1136 | debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status); | |
1137 | debug(20, l) ("StoreEntry->store_status: %d\n", (int) e->store_status); | |
1138 | debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status); | |
d377699f | 1139 | } |
1140 | ||
8350fe9b | 1141 | /* NOTE, this function assumes only two mem states */ |
f09f5b26 | 1142 | void |
8350fe9b | 1143 | storeSetMemStatus(StoreEntry * e, int new_status) |
1144 | { | |
b93bcace | 1145 | MemObject *mem = e->mem_obj; |
8350fe9b | 1146 | if (new_status == e->mem_status) |
1147 | return; | |
b93bcace | 1148 | assert(mem != NULL); |
1149 | if (new_status == IN_MEMORY) { | |
1150 | assert(mem->inmem_lo == 0); | |
2ac237e2 | 1151 | dlinkAdd(e, &mem->lru, &inmem_list); |
59c4d35b | 1152 | hot_obj_count++; |
b93bcace | 1153 | } else { |
2ac237e2 | 1154 | dlinkDelete(&mem->lru, &inmem_list); |
59c4d35b | 1155 | hot_obj_count--; |
b93bcace | 1156 | } |
8350fe9b | 1157 | e->mem_status = new_status; |
1158 | } | |
6e86c3e8 | 1159 | |
9fb13bb6 | 1160 | const char * |
1161 | storeUrl(const StoreEntry * e) | |
1162 | { | |
1163 | if (e == NULL) | |
24ffafb4 | 1164 | return "[null_entry]"; |
9fb13bb6 | 1165 | else if (e->mem_obj == NULL) |
24ffafb4 | 1166 | return "[null_mem_obj]"; |
9fb13bb6 | 1167 | else |
1168 | return e->mem_obj->url; | |
1169 | } | |
24ffafb4 | 1170 | |
1171 | void | |
1172 | storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url) | |
1173 | { | |
1174 | if (e->mem_obj) | |
1175 | return; | |
1176 | e->mem_obj = new_MemObject(url, log_url); | |
1177 | } | |
64763c37 | 1178 | |
438fc1e3 | 1179 | /* this just sets DELAY_SENDING */ |
1180 | void | |
8daca701 | 1181 | storeBuffer(StoreEntry * e) |
438fc1e3 | 1182 | { |
d46a87a8 | 1183 | EBIT_SET(e->flags, DELAY_SENDING); |
438fc1e3 | 1184 | } |
1185 | ||
1186 | /* this just clears DELAY_SENDING and Invokes the handlers */ | |
1187 | void | |
8daca701 | 1188 | storeBufferFlush(StoreEntry * e) |
438fc1e3 | 1189 | { |
d46a87a8 | 1190 | EBIT_CLR(e->flags, DELAY_SENDING); |
8daca701 | 1191 | InvokeHandlers(e); |
d5bd7c41 | 1192 | storeCheckSwapOut(e); |
438fc1e3 | 1193 | } |
25535cbe | 1194 | |
1195 | void | |
1196 | storeUnlinkFileno(int fileno) | |
1197 | { | |
07304bf9 | 1198 | debug(20, 5) ("storeUnlinkFileno: %08X\n", fileno); |
25535cbe | 1199 | #if USE_ASYNC_IO |
1200 | safeunlink(storeSwapFullPath(fileno, NULL), 1); | |
1201 | #else | |
1202 | unlinkdUnlink(storeSwapFullPath(fileno, NULL)); | |
1203 | #endif | |
1204 | } | |
07304bf9 | 1205 | |
1206 | int | |
1207 | objectLen(const StoreEntry * e) | |
1208 | { | |
1209 | assert(e->mem_obj != NULL); | |
1210 | return e->mem_obj->object_sz; | |
1211 | } | |
1212 | ||
1213 | int | |
1214 | contentLen(const StoreEntry * e) | |
1215 | { | |
1216 | assert(e->mem_obj != NULL); | |
1217 | assert(e->mem_obj->reply != NULL); | |
1218 | return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz; | |
1219 | } | |
f3986a15 | 1220 | |
1221 | HttpReply * | |
1222 | storeEntryReply(StoreEntry * e) | |
1223 | { | |
1224 | if (NULL == e) | |
3d02186d | 1225 | return NULL; |
f3986a15 | 1226 | if (NULL == e->mem_obj) |
3d02186d | 1227 | return NULL; |
f3986a15 | 1228 | return e->mem_obj->reply; |
1229 | } | |
db1cd23c | 1230 | |
1231 | void | |
1232 | storeEntryReset(StoreEntry * e) | |
1233 | { | |
1234 | MemObject *mem = e->mem_obj; | |
1235 | debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e)); | |
1236 | assert(mem->swapout.fd == -1); | |
1237 | stmemFree(&mem->data_hdr); | |
1238 | mem->inmem_hi = mem->inmem_lo = 0; | |
1239 | httpReplyDestroy(mem->reply); | |
1240 | mem->reply = httpReplyCreate(); | |
1241 | } |