]>
Commit | Line | Data |
---|---|---|
164f7660 | 1 | |
30a4f2a8 | 2 | /* |
2b6662ba | 3 | * $Id: store.cc,v 1.536 2001/01/12 00:37:21 wessels Exp $ |
30a4f2a8 | 4 | * |
8638fc66 | 5 | * DEBUG: section 20 Storage Manager |
30a4f2a8 | 6 | * AUTHOR: Harvest Derived |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
e25c139f | 9 | * ---------------------------------------------------------- |
30a4f2a8 | 10 | * |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
30a4f2a8 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
2b6662ba | 24 | * |
30a4f2a8 | 25 | * This program is distributed in the hope that it will be useful, |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
2b6662ba | 29 | * |
30a4f2a8 | 30 | * You should have received a copy of the GNU General Public License |
31 | * along with this program; if not, write to the Free Software | |
cbdec147 | 32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. |
e25c139f | 33 | * |
c943f331 | 34 | */ |
090089c4 | 35 | |
f09f5b26 | 36 | #include "squid.h" |
090089c4 | 37 | |
38 | #define REBUILD_TIMESTAMP_DELTA_MAX 2 | |
227fbb74 | 39 | |
fcf5283d | 40 | #define STORE_IN_MEM_BUCKETS (229) |
090089c4 | 41 | |
0ee4272b | 42 | const char *memStatusStr[] = |
e62d2dea | 43 | { |
9dfb6c1c | 44 | "NOT_IN_MEMORY", |
9dfb6c1c | 45 | "IN_MEMORY" |
46 | }; | |
47 | ||
0ee4272b | 48 | const char *pingStatusStr[] = |
e62d2dea | 49 | { |
f17936ab | 50 | "PING_NONE", |
9dfb6c1c | 51 | "PING_WAITING", |
f17936ab | 52 | "PING_DONE" |
9dfb6c1c | 53 | }; |
54 | ||
0ee4272b | 55 | const char *storeStatusStr[] = |
e62d2dea | 56 | { |
9dfb6c1c | 57 | "STORE_OK", |
b7fe0ab0 | 58 | "STORE_PENDING" |
9dfb6c1c | 59 | }; |
60 | ||
0ee4272b | 61 | const char *swapStatusStr[] = |
e62d2dea | 62 | { |
8350fe9b | 63 | "SWAPOUT_NONE", |
8350fe9b | 64 | "SWAPOUT_WRITING", |
65 | "SWAPOUT_DONE" | |
9dfb6c1c | 66 | }; |
67 | ||
0a0bf5db | 68 | typedef struct lock_ctrl_t { |
582b6456 | 69 | SIH *callback; |
0a0bf5db | 70 | void *callback_data; |
71 | StoreEntry *e; | |
72 | } lock_ctrl_t; | |
73 | ||
65a53c8e | 74 | extern OBJH storeIOStats; |
75 | ||
e3ef2b09 | 76 | /* |
77 | * local function prototypes | |
78 | */ | |
f5b8bbc4 | 79 | static int storeEntryValidLength(const StoreEntry *); |
80 | static void storeGetMemSpace(int); | |
b93bcace | 81 | static void storeHashDelete(StoreEntry *); |
9fb13bb6 | 82 | static MemObject *new_MemObject(const char *, const char *); |
6cf028ab | 83 | static void destroy_MemObject(StoreEntry *); |
ec878047 | 84 | static FREE destroy_StoreEntry; |
f5b8bbc4 | 85 | static void storePurgeMem(StoreEntry *); |
6a566b9c | 86 | static void storeEntryReferenced(StoreEntry *); |
87 | static void storeEntryDereferenced(StoreEntry *); | |
007b8be4 | 88 | static int getKeyCounter(void); |
8350fe9b | 89 | static int storeKeepInMemory(const StoreEntry *); |
8423ff74 | 90 | static OBJH storeCheckCachableStats; |
e42d5181 | 91 | static EVH storeLateRelease; |
a21fbb54 | 92 | |
e3ef2b09 | 93 | /* |
94 | * local variables | |
95 | */ | |
e42d5181 | 96 | static Stack LateReleaseStack; |
66cedb85 | 97 | |
bc87dc25 | 98 | #if URL_CHECKSUM_DEBUG |
99 | unsigned int | |
100 | url_checksum(const char *url) | |
101 | { | |
102 | unsigned int ck; | |
103 | MD5_CTX M; | |
104 | static unsigned char digest[16]; | |
105 | MD5Init(&M); | |
106 | MD5Update(&M, (unsigned char *) url, strlen(url)); | |
107 | MD5Final(digest, &M); | |
108 | xmemcpy(&ck, digest, sizeof(ck)); | |
109 | return ck; | |
110 | } | |
111 | #endif | |
112 | ||
b8d8561b | 113 | static MemObject * |
9fb13bb6 | 114 | new_MemObject(const char *url, const char *log_url) |
227fbb74 | 115 | { |
7021844c | 116 | MemObject *mem = memAllocate(MEM_MEMOBJECT); |
cb69b4c7 | 117 | mem->reply = httpReplyCreate(); |
9fb13bb6 | 118 | mem->url = xstrdup(url); |
bc87dc25 | 119 | #if URL_CHECKSUM_DEBUG |
120 | mem->chksum = url_checksum(mem->url); | |
121 | #endif | |
88738790 | 122 | mem->log_url = xstrdup(log_url); |
07304bf9 | 123 | mem->object_sz = -1; |
ed03a839 | 124 | mem->fd = -1; |
59c4d35b | 125 | /* XXX account log_url */ |
a3d5953d | 126 | debug(20, 3) ("new_MemObject: returning %p\n", mem); |
30a4f2a8 | 127 | return mem; |
227fbb74 | 128 | } |
129 | ||
f09f5b26 | 130 | StoreEntry * |
9fb13bb6 | 131 | new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) |
090089c4 | 132 | { |
133 | StoreEntry *e = NULL; | |
7021844c | 134 | e = memAllocate(MEM_STOREENTRY); |
227fbb74 | 135 | if (mem_obj_flag) |
9fb13bb6 | 136 | e->mem_obj = new_MemObject(url, log_url); |
a3d5953d | 137 | debug(20, 3) ("new_StoreEntry: returning %p\n", e); |
e17dc75c | 138 | e->expires = e->lastmod = e->lastref = e->timestamp = -1; |
cd748f27 | 139 | e->swap_filen = -1; |
140 | e->swap_dirn = -1; | |
227fbb74 | 141 | return e; |
090089c4 | 142 | } |
143 | ||
b8d8561b | 144 | static void |
0e473d70 | 145 | destroy_MemObject(StoreEntry * e) |
090089c4 | 146 | { |
6cf028ab | 147 | MemObject *mem = e->mem_obj; |
123abbe1 | 148 | const Ctx ctx = ctx_enter(mem->url); |
a3d5953d | 149 | debug(20, 3) ("destroy_MemObject: destroying %p\n", mem); |
bc87dc25 | 150 | #if URL_CHECKSUM_DEBUG |
151 | assert(mem->chksum == url_checksum(mem->url)); | |
152 | #endif | |
41f7e38d | 153 | e->mem_obj = NULL; |
9e665466 | 154 | if (!shutting_down) |
2391a162 | 155 | assert(mem->swapout.sio == NULL); |
18fe65d0 | 156 | stmemFree(&mem->data_hdr); |
157 | mem->inmem_hi = 0; | |
6982a226 | 158 | /* |
159 | * There is no way to abort FD-less clients, so they might | |
160 | * still have mem->clients set if mem->fd == -1 | |
161 | */ | |
06d2839d | 162 | assert(mem->fd == -1 || mem->clients.head == NULL); |
cb69b4c7 | 163 | httpReplyDestroy(mem->reply); |
30a4f2a8 | 164 | requestUnlink(mem->request); |
165 | mem->request = NULL; | |
2ac76861 | 166 | ctx_exit(ctx); /* must exit before we free mem->url */ |
48959832 | 167 | safe_free(mem->url); |
0e4e0e7d | 168 | safe_free(mem->log_url); /* XXX account log_url */ |
db1cd23c | 169 | memFree(mem, MEM_MEMOBJECT); |
090089c4 | 170 | } |
171 | ||
b8d8561b | 172 | static void |
ec878047 | 173 | destroy_StoreEntry(void *data) |
090089c4 | 174 | { |
ec878047 | 175 | StoreEntry *e = data; |
a3d5953d | 176 | debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); |
9e975e4e | 177 | assert(e != NULL); |
227fbb74 | 178 | if (e->mem_obj) |
6cf028ab | 179 | destroy_MemObject(e); |
043b055b | 180 | storeHashDelete(e); |
186477c1 | 181 | assert(e->hash.key == NULL); |
db1cd23c | 182 | memFree(e, MEM_STOREENTRY); |
227fbb74 | 183 | } |
090089c4 | 184 | |
090089c4 | 185 | /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ |
186 | ||
f09f5b26 | 187 | void |
9fb13bb6 | 188 | storeHashInsert(StoreEntry * e, const cache_key * key) |
090089c4 | 189 | { |
a3d5953d | 190 | debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", |
9fb13bb6 | 191 | e, storeKeyText(key)); |
186477c1 | 192 | e->hash.key = storeKeyDup(key); |
193 | hash_join(store_table, &e->hash); | |
090089c4 | 194 | } |
195 | ||
b93bcace | 196 | static void |
b8d8561b | 197 | storeHashDelete(StoreEntry * e) |
090089c4 | 198 | { |
186477c1 | 199 | hash_remove_link(store_table, &e->hash); |
200 | storeKeyFree(e->hash.key); | |
201 | e->hash.key = NULL; | |
090089c4 | 202 | } |
203 | ||
090089c4 | 204 | /* -------------------------------------------------------------------------- */ |
205 | ||
090089c4 | 206 | |
207 | /* get rid of memory copy of the object */ | |
620da955 | 208 | /* Only call this if storeCheckPurgeMem(e) returns 1 */ |
24382924 | 209 | static void |
b8d8561b | 210 | storePurgeMem(StoreEntry * e) |
090089c4 | 211 | { |
2aca8433 | 212 | if (e->mem_obj == NULL) |
090089c4 | 213 | return; |
9fb13bb6 | 214 | debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", |
186477c1 | 215 | storeKeyText(e->hash.key)); |
090089c4 | 216 | storeSetMemStatus(e, NOT_IN_MEMORY); |
6cf028ab | 217 | destroy_MemObject(e); |
06e8899b | 218 | if (e->swap_status != SWAPOUT_DONE) |
8272aded | 219 | storeRelease(e); |
090089c4 | 220 | } |
221 | ||
6a566b9c | 222 | static void |
c1dd71ae | 223 | storeEntryReferenced(StoreEntry * e) |
090089c4 | 224 | { |
cd748f27 | 225 | SwapDir *SD; |
226 | ||
6a566b9c | 227 | /* Notify the fs that we're referencing this object again */ |
228 | if (e->swap_dirn > -1) { | |
a4b8110e | 229 | SD = INDEXSD(e->swap_dirn); |
6a566b9c | 230 | if (SD->refobj) |
231 | SD->refobj(SD, e); | |
232 | } | |
233 | /* Notify the memory cache that we're referencing this object again */ | |
c1dd71ae | 234 | if (e->mem_obj) { |
6a566b9c | 235 | if (mem_policy->Referenced) |
236 | mem_policy->Referenced(mem_policy, e, &e->mem_obj->repl); | |
237 | } | |
238 | } | |
cd748f27 | 239 | |
6a566b9c | 240 | static void |
c1dd71ae | 241 | storeEntryDereferenced(StoreEntry * e) |
6a566b9c | 242 | { |
243 | SwapDir *SD; | |
244 | ||
245 | /* Notify the fs that we're not referencing this object any more */ | |
246 | if (e->swap_filen > -1) { | |
c1dd71ae | 247 | SD = INDEXSD(e->swap_dirn); |
6a566b9c | 248 | if (SD->unrefobj != NULL) |
249 | SD->unrefobj(SD, e); | |
250 | } | |
251 | /* Notify the memory cache that we're not referencing this object any more */ | |
c1dd71ae | 252 | if (e->mem_obj) { |
6a566b9c | 253 | if (mem_policy->Dereferenced) |
254 | mem_policy->Dereferenced(mem_policy, e, &e->mem_obj->repl); | |
255 | } | |
256 | } | |
257 | ||
258 | void | |
259 | storeLockObject(StoreEntry * e) | |
260 | { | |
cd748f27 | 261 | e->lock_count++; |
a3d5953d | 262 | debug(20, 3) ("storeLockObject: key '%s' count=%d\n", |
186477c1 | 263 | storeKeyText(e->hash.key), (int) e->lock_count); |
b8de7ebe | 264 | e->lastref = squid_curtime; |
6a566b9c | 265 | storeEntryReferenced(e); |
090089c4 | 266 | } |
267 | ||
b8d8561b | 268 | void |
269 | storeReleaseRequest(StoreEntry * e) | |
2285407f | 270 | { |
d46a87a8 | 271 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
58bcd31b | 272 | return; |
186477c1 | 273 | debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->hash.key)); |
d46a87a8 | 274 | EBIT_SET(e->flags, RELEASE_REQUEST); |
f3e570e9 | 275 | /* |
276 | * Clear cachable flag here because we might get called before | |
277 | * anyone else even looks at the cachability flag. Also, this | |
278 | * prevents httpMakePublic from really setting a public key. | |
279 | */ | |
d46a87a8 | 280 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
fe54d06d | 281 | storeSetPrivateKey(e); |
2285407f | 282 | } |
283 | ||
090089c4 | 284 | /* unlock object, return -1 if object get released after unlock |
285 | * otherwise lock_count */ | |
b8d8561b | 286 | int |
287 | storeUnlockObject(StoreEntry * e) | |
090089c4 | 288 | { |
6c895381 | 289 | e->lock_count--; |
a3d5953d | 290 | debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", |
186477c1 | 291 | storeKeyText(e->hash.key), e->lock_count); |
30a4f2a8 | 292 | if (e->lock_count) |
a1e47288 | 293 | return (int) e->lock_count; |
b7fe0ab0 | 294 | if (e->store_status == STORE_PENDING) |
d46a87a8 | 295 | EBIT_SET(e->flags, RELEASE_REQUEST); |
9e975e4e | 296 | assert(storePendingNClients(e) == 0); |
d46a87a8 | 297 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
30a4f2a8 | 298 | storeRelease(e); |
8350fe9b | 299 | else if (storeKeepInMemory(e)) { |
6a566b9c | 300 | storeEntryDereferenced(e); |
8350fe9b | 301 | storeSetMemStatus(e, IN_MEMORY); |
302 | requestUnlink(e->mem_obj->request); | |
303 | e->mem_obj->request = NULL; | |
66f07209 | 304 | } else { |
30a4f2a8 | 305 | storePurgeMem(e); |
6a566b9c | 306 | storeEntryDereferenced(e); |
fc8b9fc0 | 307 | if (EBIT_TEST(e->flags, KEY_PRIVATE)) |
14b9e4c5 | 308 | debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); |
66f07209 | 309 | } |
6c895381 | 310 | return 0; |
090089c4 | 311 | } |
312 | ||
2b906e48 | 313 | /* Lookup an object in the cache. |
090089c4 | 314 | * return just a reference to object, don't start swapping in yet. */ |
b8d8561b | 315 | StoreEntry * |
9fb13bb6 | 316 | storeGet(const cache_key * key) |
090089c4 | 317 | { |
9fb13bb6 | 318 | debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); |
319 | return (StoreEntry *) hash_lookup(store_table, key); | |
090089c4 | 320 | } |
321 | ||
08e5d64f | 322 | StoreEntry * |
323 | storeGetPublic(const char *uri, const method_t method) | |
324 | { | |
6b8e7481 | 325 | return storeGet(storeKeyPublic(uri, method)); |
08e5d64f | 326 | } |
327 | ||
007b8be4 | 328 | static int |
329 | getKeyCounter(void) | |
04e8dbaa | 330 | { |
007b8be4 | 331 | static int key_counter = 0; |
332 | if (++key_counter < 0) | |
2cc3f720 | 333 | key_counter = 1; |
007b8be4 | 334 | return key_counter; |
04e8dbaa | 335 | } |
336 | ||
6c57e268 | 337 | void |
b8d8561b | 338 | storeSetPrivateKey(StoreEntry * e) |
227fbb74 | 339 | { |
9fb13bb6 | 340 | const cache_key *newkey; |
341 | MemObject *mem = e->mem_obj; | |
186477c1 | 342 | if (e->hash.key && EBIT_TEST(e->flags, KEY_PRIVATE)) |
6eb42cae | 343 | return; /* is already private */ |
186477c1 | 344 | if (e->hash.key) { |
cd748f27 | 345 | if (e->swap_filen > -1) |
b109de6b | 346 | storeDirSwapLog(e, SWAP_LOG_DEL); |
bc0bce21 | 347 | storeHashDelete(e); |
b109de6b | 348 | } |
9fb13bb6 | 349 | if (mem != NULL) { |
007b8be4 | 350 | mem->id = getKeyCounter(); |
351 | newkey = storeKeyPrivate(mem->url, mem->method, mem->id); | |
9fb13bb6 | 352 | } else { |
007b8be4 | 353 | newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); |
9fb13bb6 | 354 | } |
355 | assert(hash_lookup(store_table, newkey) == NULL); | |
d46a87a8 | 356 | EBIT_SET(e->flags, KEY_PRIVATE); |
8638fc66 | 357 | storeHashInsert(e, newkey); |
227fbb74 | 358 | } |
359 | ||
b8d8561b | 360 | void |
361 | storeSetPublicKey(StoreEntry * e) | |
227fbb74 | 362 | { |
6eb42cae | 363 | StoreEntry *e2 = NULL; |
9fb13bb6 | 364 | const cache_key *newkey; |
365 | MemObject *mem = e->mem_obj; | |
186477c1 | 366 | if (e->hash.key && !EBIT_TEST(e->flags, KEY_PRIVATE)) |
6eb42cae | 367 | return; /* is already public */ |
9fb13bb6 | 368 | assert(mem); |
f3e570e9 | 369 | /* |
370 | * We can't make RELEASE_REQUEST objects public. Depending on | |
371 | * when RELEASE_REQUEST gets set, we might not be swapping out | |
372 | * the object. If we're not swapping out, then subsequent | |
373 | * store clients won't be able to access object data which has | |
374 | * been freed from memory. | |
d87ebd78 | 375 | * |
376 | * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not | |
377 | * be set, and storeSetPublicKey() should not be called. | |
f3e570e9 | 378 | */ |
6a566b9c | 379 | #if MORE_DEBUG_OUTPUT |
2b906e48 | 380 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
1f38f50a | 381 | debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n", |
186477c1 | 382 | e->hash.key, mem->url); |
2b906e48 | 383 | #endif |
d46a87a8 | 384 | assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); |
2ac237e2 | 385 | newkey = storeKeyPublic(mem->url, mem->method); |
9fb13bb6 | 386 | if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { |
387 | debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); | |
07622ccd | 388 | storeSetPrivateKey(e2); |
30a4f2a8 | 389 | storeRelease(e2); |
2ac237e2 | 390 | newkey = storeKeyPublic(mem->url, mem->method); |
6eb42cae | 391 | } |
186477c1 | 392 | if (e->hash.key) |
bc0bce21 | 393 | storeHashDelete(e); |
d46a87a8 | 394 | EBIT_CLR(e->flags, KEY_PRIVATE); |
8638fc66 | 395 | storeHashInsert(e, newkey); |
cd748f27 | 396 | if (e->swap_filen > -1) |
b109de6b | 397 | storeDirSwapLog(e, SWAP_LOG_ADD); |
227fbb74 | 398 | } |
399 | ||
b8d8561b | 400 | StoreEntry * |
92695e5e | 401 | storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method) |
090089c4 | 402 | { |
090089c4 | 403 | StoreEntry *e = NULL; |
30a4f2a8 | 404 | MemObject *mem = NULL; |
92695e5e | 405 | debug(20, 3) ("storeCreateEntry: '%s'\n", url); |
090089c4 | 406 | |
f09f5b26 | 407 | e = new_StoreEntry(STORE_ENTRY_WITH_MEMOBJ, url, log_url); |
30a4f2a8 | 408 | e->lock_count = 1; /* Note lock here w/o calling storeLock() */ |
409 | mem = e->mem_obj; | |
2ac237e2 | 410 | mem->method = method; |
92695e5e | 411 | if (neighbors_do_private_keys || !flags.hierarchical) |
86101e40 | 412 | storeSetPrivateKey(e); |
413 | else | |
414 | storeSetPublicKey(e); | |
92695e5e | 415 | if (flags.cachable) { |
d46a87a8 | 416 | EBIT_SET(e->flags, ENTRY_CACHABLE); |
417 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
090089c4 | 418 | } else { |
d46a87a8 | 419 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
2daae136 | 420 | storeReleaseRequest(e); |
090089c4 | 421 | } |
234967c9 | 422 | e->store_status = STORE_PENDING; |
090089c4 | 423 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 424 | e->swap_status = SWAPOUT_NONE; |
cd748f27 | 425 | e->swap_filen = -1; |
426 | e->swap_dirn = -1; | |
090089c4 | 427 | e->refcount = 0; |
b8de7ebe | 428 | e->lastref = squid_curtime; |
d20b1cd0 | 429 | e->timestamp = -1; /* set in storeTimestampsSet() */ |
30a4f2a8 | 430 | e->ping_status = PING_NONE; |
d46a87a8 | 431 | EBIT_SET(e->flags, ENTRY_VALIDATED); |
090089c4 | 432 | return e; |
433 | } | |
434 | ||
6eb42cae | 435 | /* Mark object as expired */ |
b8d8561b | 436 | void |
437 | storeExpireNow(StoreEntry * e) | |
9174e204 | 438 | { |
186477c1 | 439 | debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->hash.key)); |
b8de7ebe | 440 | e->expires = squid_curtime; |
9174e204 | 441 | } |
442 | ||
090089c4 | 443 | /* Append incoming data from a primary server to an entry. */ |
b8d8561b | 444 | void |
b8014333 | 445 | storeAppend(StoreEntry * e, const char *buf, int len) |
090089c4 | 446 | { |
3a1c3e2f | 447 | MemObject *mem = e->mem_obj; |
448 | assert(mem != NULL); | |
449 | assert(len >= 0); | |
789f4e81 | 450 | assert(e->store_status == STORE_PENDING); |
090089c4 | 451 | if (len) { |
d0e2935f | 452 | debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", |
453 | len, | |
186477c1 | 454 | storeKeyText(e->hash.key)); |
38792624 | 455 | storeGetMemSpace(len); |
18fe65d0 | 456 | stmemAppend(&mem->data_hdr, buf, len); |
8350fe9b | 457 | mem->inmem_hi += len; |
090089c4 | 458 | } |
d46a87a8 | 459 | if (EBIT_TEST(e->flags, DELAY_SENDING)) |
d0e2935f | 460 | return; |
461 | InvokeHandlers(e); | |
2391a162 | 462 | storeSwapOut(e); |
090089c4 | 463 | } |
464 | ||
b8d8561b | 465 | void |
62d32805 | 466 | #if STDC_HEADERS |
fe4e214f | 467 | storeAppendPrintf(StoreEntry * e, const char *fmt,...) |
c30c5a73 | 468 | #else |
b8d8561b | 469 | storeAppendPrintf(va_alist) |
15c05bb0 | 470 | va_dcl |
62d32805 | 471 | #endif |
15c05bb0 | 472 | { |
62d32805 | 473 | #if STDC_HEADERS |
474 | va_list args; | |
475 | va_start(args, fmt); | |
476 | #else | |
15c05bb0 | 477 | va_list args; |
478 | StoreEntry *e = NULL; | |
0ee4272b | 479 | const char *fmt = NULL; |
15c05bb0 | 480 | va_start(args); |
481 | e = va_arg(args, StoreEntry *); | |
482 | fmt = va_arg(args, char *); | |
c30c5a73 | 483 | #endif |
cb69b4c7 | 484 | storeAppendVPrintf(e, fmt, args); |
485 | va_end(args); | |
486 | } | |
487 | ||
488 | /* used be storeAppendPrintf and Packer */ | |
489 | void | |
490 | storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs) | |
491 | { | |
492 | LOCAL_ARRAY(char, buf, 4096); | |
15c05bb0 | 493 | buf[0] = '\0'; |
cb69b4c7 | 494 | vsnprintf(buf, 4096, fmt, vargs); |
15c05bb0 | 495 | storeAppend(e, buf, strlen(buf)); |
c30c5a73 | 496 | } |
497 | ||
8423ff74 | 498 | struct _store_check_cachable_hist { |
499 | struct { | |
500 | int non_get; | |
501 | int not_entry_cachable; | |
502 | int release_request; | |
503 | int wrong_content_length; | |
504 | int negative_cached; | |
505 | int too_big; | |
d20b1cd0 | 506 | int too_small; |
8423ff74 | 507 | int private_key; |
c5f627c2 | 508 | int too_many_open_files; |
59ffcdf8 | 509 | int too_many_open_fds; |
8423ff74 | 510 | } no; |
511 | struct { | |
512 | int Default; | |
513 | } yes; | |
514 | } store_check_cachable_hist; | |
515 | ||
c47511fd | 516 | int |
517 | storeTooManyDiskFilesOpen(void) | |
518 | { | |
519 | if (Config.max_open_disk_fds == 0) | |
520 | return 0; | |
83a29c95 | 521 | if (store_open_disk_fd > Config.max_open_disk_fds) |
c47511fd | 522 | return 1; |
523 | return 0; | |
524 | } | |
525 | ||
d20b1cd0 | 526 | static int |
527 | storeCheckTooSmall(StoreEntry * e) | |
528 | { | |
529 | MemObject *mem = e->mem_obj; | |
42b51993 | 530 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) |
fbb4a198 | 531 | return 0; |
d20b1cd0 | 532 | if (STORE_OK == e->store_status) |
533 | if (mem->object_sz < Config.Store.minObjectSize) | |
534 | return 1; | |
535 | if (mem->reply->content_length > -1) | |
536 | if (mem->reply->content_length < (int) Config.Store.minObjectSize) | |
537 | return 1; | |
538 | return 0; | |
539 | } | |
540 | ||
f09f5b26 | 541 | int |
8350fe9b | 542 | storeCheckCachable(StoreEntry * e) |
6602e70e | 543 | { |
2ac237e2 | 544 | #if CACHE_ALL_METHODS |
545 | if (e->mem_obj->method != METHOD_GET) { | |
8350fe9b | 546 | debug(20, 2) ("storeCheckCachable: NO: non-GET method\n"); |
8423ff74 | 547 | store_check_cachable_hist.no.non_get++; |
2ac237e2 | 548 | } else |
549 | #endif | |
d46a87a8 | 550 | if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) { |
8350fe9b | 551 | debug(20, 2) ("storeCheckCachable: NO: not cachable\n"); |
8423ff74 | 552 | store_check_cachable_hist.no.not_entry_cachable++; |
d46a87a8 | 553 | } else if (EBIT_TEST(e->flags, RELEASE_REQUEST)) { |
8350fe9b | 554 | debug(20, 2) ("storeCheckCachable: NO: release requested\n"); |
8423ff74 | 555 | store_check_cachable_hist.no.release_request++; |
d46a87a8 | 556 | } else if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) { |
8350fe9b | 557 | debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n"); |
8423ff74 | 558 | store_check_cachable_hist.no.wrong_content_length++; |
d46a87a8 | 559 | } else if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) { |
c0dee43f | 560 | debug(20, 3) ("storeCheckCachable: NO: negative cached\n"); |
8423ff74 | 561 | store_check_cachable_hist.no.negative_cached++; |
3e98df20 | 562 | return 0; /* avoid release call below */ |
c1fc651e | 563 | } else if ((e->mem_obj->reply->content_length > 0 && |
564 | e->mem_obj->reply->content_length > Config.Store.maxObjectSize) || | |
a4b8110e | 565 | e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { |
8350fe9b | 566 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); |
8423ff74 | 567 | store_check_cachable_hist.no.too_big++; |
7e3ce7b9 | 568 | } else if (e->mem_obj->reply->content_length > (int) Config.Store.maxObjectSize) { |
569 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); | |
570 | store_check_cachable_hist.no.too_big++; | |
d20b1cd0 | 571 | } else if (storeCheckTooSmall(e)) { |
572 | debug(20, 2) ("storeCheckCachable: NO: too small\n"); | |
573 | store_check_cachable_hist.no.too_small++; | |
d46a87a8 | 574 | } else if (EBIT_TEST(e->flags, KEY_PRIVATE)) { |
8350fe9b | 575 | debug(20, 3) ("storeCheckCachable: NO: private key\n"); |
8423ff74 | 576 | store_check_cachable_hist.no.private_key++; |
83a29c95 | 577 | } else if (e->swap_status != SWAPOUT_NONE) { |
578 | /* | |
579 | * here we checked the swap_status because the remaining | |
580 | * cases are only relevant only if we haven't started swapping | |
581 | * out the object yet. | |
582 | */ | |
583 | return 1; | |
c47511fd | 584 | } else if (storeTooManyDiskFilesOpen()) { |
c5f627c2 | 585 | debug(20, 2) ("storeCheckCachable: NO: too many disk files open\n"); |
586 | store_check_cachable_hist.no.too_many_open_files++; | |
59ffcdf8 | 587 | } else if (fdNFree() < RESERVED_FD) { |
83a29c95 | 588 | debug(20, 2) ("storeCheckCachable: NO: too many FD's open\n"); |
59ffcdf8 | 589 | store_check_cachable_hist.no.too_many_open_fds++; |
d4432957 | 590 | } else { |
8423ff74 | 591 | store_check_cachable_hist.yes.Default++; |
6602e70e | 592 | return 1; |
d4432957 | 593 | } |
2daae136 | 594 | storeReleaseRequest(e); |
d46a87a8 | 595 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
6602e70e | 596 | return 0; |
597 | } | |
598 | ||
8423ff74 | 599 | static void |
600 | storeCheckCachableStats(StoreEntry * sentry) | |
601 | { | |
c40acff3 | 602 | storeAppendPrintf(sentry, "Category\t Count\n"); |
603 | ||
8423ff74 | 604 | storeAppendPrintf(sentry, "no.non_get\t%d\n", |
605 | store_check_cachable_hist.no.non_get); | |
606 | storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n", | |
607 | store_check_cachable_hist.no.not_entry_cachable); | |
608 | storeAppendPrintf(sentry, "no.release_request\t%d\n", | |
609 | store_check_cachable_hist.no.release_request); | |
610 | storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n", | |
611 | store_check_cachable_hist.no.wrong_content_length); | |
612 | storeAppendPrintf(sentry, "no.negative_cached\t%d\n", | |
613 | store_check_cachable_hist.no.negative_cached); | |
614 | storeAppendPrintf(sentry, "no.too_big\t%d\n", | |
615 | store_check_cachable_hist.no.too_big); | |
d20b1cd0 | 616 | storeAppendPrintf(sentry, "no.too_small\t%d\n", |
617 | store_check_cachable_hist.no.too_small); | |
8423ff74 | 618 | storeAppendPrintf(sentry, "no.private_key\t%d\n", |
619 | store_check_cachable_hist.no.private_key); | |
c5f627c2 | 620 | storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n", |
621 | store_check_cachable_hist.no.too_many_open_files); | |
59ffcdf8 | 622 | storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n", |
623 | store_check_cachable_hist.no.too_many_open_fds); | |
8423ff74 | 624 | storeAppendPrintf(sentry, "yes.default\t%d\n", |
625 | store_check_cachable_hist.yes.Default); | |
626 | } | |
627 | ||
090089c4 | 628 | /* Complete transfer into the local cache. */ |
b8d8561b | 629 | void |
630 | storeComplete(StoreEntry * e) | |
090089c4 | 631 | { |
186477c1 | 632 | debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->hash.key)); |
b6403fac | 633 | if (e->store_status != STORE_PENDING) { |
634 | /* | |
635 | * if we're not STORE_PENDING, then probably we got aborted | |
636 | * and there should be NO clients on this entry | |
637 | */ | |
7a1024ee | 638 | assert(EBIT_TEST(e->flags, ENTRY_ABORTED)); |
639 | assert(e->mem_obj->nclients == 0); | |
b6403fac | 640 | return; |
641 | } | |
07304bf9 | 642 | e->mem_obj->object_sz = e->mem_obj->inmem_hi; |
234967c9 | 643 | e->store_status = STORE_OK; |
8350fe9b | 644 | assert(e->mem_status == NOT_IN_MEMORY); |
41587298 | 645 | if (!storeEntryValidLength(e)) { |
d46a87a8 | 646 | EBIT_SET(e->flags, ENTRY_BAD_LENGTH); |
41587298 | 647 | storeReleaseRequest(e); |
648 | } | |
6cfa8966 | 649 | #if USE_CACHE_DIGESTS |
544b1fd4 | 650 | if (e->mem_obj->request) |
651 | e->mem_obj->request->hier.store_complete_stop = current_time; | |
39edba21 | 652 | #endif |
d20b1cd0 | 653 | /* |
654 | * We used to call InvokeHandlers, then storeSwapOut. However, | |
655 | * Madhukar Reddy <myreddy@persistence.com> reported that | |
656 | * responses without content length would sometimes get released | |
657 | * in client_side, thinking that the response is incomplete. | |
658 | */ | |
2391a162 | 659 | storeSwapOut(e); |
d20b1cd0 | 660 | InvokeHandlers(e); |
7e3e1d01 | 661 | } |
662 | ||
090089c4 | 663 | /* |
474cac1b | 664 | * Someone wants to abort this transfer. Set the reason in the |
665 | * request structure, call the server-side callback and mark the | |
2b906e48 | 666 | * entry for releasing |
090089c4 | 667 | */ |
b8d8561b | 668 | void |
7197b20d | 669 | storeAbort(StoreEntry * e) |
090089c4 | 670 | { |
3e98df20 | 671 | MemObject *mem = e->mem_obj; |
8f39b81d | 672 | assert(e->store_status == STORE_PENDING); |
673 | assert(mem != NULL); | |
186477c1 | 674 | debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->hash.key)); |
3d02186d | 675 | storeLockObject(e); /* lock while aborting */ |
79b5cc5f | 676 | storeNegativeCache(e); |
474cac1b | 677 | storeReleaseRequest(e); |
b7fe0ab0 | 678 | EBIT_SET(e->flags, ENTRY_ABORTED); |
8350fe9b | 679 | storeSetMemStatus(e, NOT_IN_MEMORY); |
789f4e81 | 680 | e->store_status = STORE_OK; |
496e1d76 | 681 | /* |
682 | * We assign an object length here. The only other place we assign | |
683 | * the object length is in storeComplete() | |
684 | */ | |
07304bf9 | 685 | mem->object_sz = mem->inmem_hi; |
474cac1b | 686 | /* Notify the server side */ |
7197b20d | 687 | if (mem->abort.callback) { |
e92e4e44 | 688 | eventAdd("mem->abort.callback", |
689 | mem->abort.callback, | |
690 | mem->abort.data, | |
691 | 0.0, | |
692 | 0); | |
bfcaf585 | 693 | mem->abort.callback = NULL; |
6801f8a8 | 694 | mem->abort.data = NULL; |
bfcaf585 | 695 | } |
474cac1b | 696 | /* Notify the client side */ |
090089c4 | 697 | InvokeHandlers(e); |
cd748f27 | 698 | /* Close any swapout file */ |
699 | storeSwapOutFileClose(e); | |
3d02186d | 700 | storeUnlockObject(e); /* unlock */ |
090089c4 | 701 | } |
702 | ||
090089c4 | 703 | /* Clear Memory storage to accommodate the given object len */ |
38792624 | 704 | static void |
d4432957 | 705 | storeGetMemSpace(int size) |
090089c4 | 706 | { |
b32508fb | 707 | StoreEntry *e = NULL; |
20cba4b4 | 708 | int released = 0; |
b32508fb | 709 | static time_t last_check = 0; |
e954773d | 710 | int pages_needed; |
6a566b9c | 711 | RemovalPurgeWalker *walker; |
b32508fb | 712 | if (squid_curtime == last_check) |
38792624 | 713 | return; |
b32508fb | 714 | last_check = squid_curtime; |
e954773d | 715 | pages_needed = (size / SM_PAGE_SIZE) + 1; |
43a70238 | 716 | if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max) |
38792624 | 717 | return; |
a3d5953d | 718 | debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed); |
6a566b9c | 719 | /* XXX what to set as max_scan here? */ |
720 | walker = mem_policy->PurgeInit(mem_policy, 100000); | |
c1dd71ae | 721 | while ((e = walker->Next(walker))) { |
2b906e48 | 722 | storePurgeMem(e); |
20cba4b4 | 723 | released++; |
43a70238 | 724 | if (memInUse(MEM_STMEM_BUF) + pages_needed < store_pages_max) |
8350fe9b | 725 | break; |
726 | } | |
6a566b9c | 727 | walker->Done(walker); |
5e3d4fef | 728 | debug(20, 3) ("storeGetMemSpace stats:\n"); |
59c4d35b | 729 | debug(20, 3) (" %6d HOT objects\n", hot_obj_count); |
20cba4b4 | 730 | debug(20, 3) (" %6d were released\n", released); |
090089c4 | 731 | } |
732 | ||
090089c4 | 733 | /* The maximum objects to scan for maintain storage space */ |
fcefe642 | 734 | #define MAINTAIN_MAX_SCAN 1024 |
735 | #define MAINTAIN_MAX_REMOVE 64 | |
090089c4 | 736 | |
2b906e48 | 737 | /* |
fcefe642 | 738 | * This routine is to be called by main loop in main.c. |
739 | * It removes expired objects on only one bucket for each time called. | |
740 | * returns the number of objects removed | |
741 | * | |
742 | * This should get called 1/s from main(). | |
743 | */ | |
679ac4f0 | 744 | void |
79d39a72 | 745 | storeMaintainSwapSpace(void *datanotused) |
090089c4 | 746 | { |
fc8b9fc0 | 747 | int i; |
cd748f27 | 748 | SwapDir *SD; |
6a566b9c | 749 | static time_t last_warn_time = 0; |
cd748f27 | 750 | |
751 | /* walk each fs */ | |
fc8b9fc0 | 752 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
cd748f27 | 753 | /* call the maintain function .. */ |
c1dd71ae | 754 | SD = INDEXSD(i); |
6a566b9c | 755 | /* XXX FixMe: This should be done "in parallell" on the different |
756 | * cache_dirs, not one at a time. | |
757 | */ | |
c1dd71ae | 758 | if (SD->maintainfs != NULL) |
cd748f27 | 759 | SD->maintainfs(SD); |
fc8b9fc0 | 760 | } |
6a566b9c | 761 | if (store_swap_size > Config.Swap.maxSize) { |
762 | if (squid_curtime - last_warn_time > 10) { | |
763 | debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", | |
764 | store_swap_size, Config.Swap.maxSize); | |
765 | last_warn_time = squid_curtime; | |
766 | } | |
767 | } | |
cd748f27 | 768 | /* Reregister a maintain event .. */ |
769 | eventAdd("MaintainSwapSpace", storeMaintainSwapSpace, NULL, 1.0, 1); | |
090089c4 | 770 | } |
771 | ||
772 | ||
773 | /* release an object from a cache */ | |
6c78a099 | 774 | void |
b8d8561b | 775 | storeRelease(StoreEntry * e) |
090089c4 | 776 | { |
186477c1 | 777 | debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->hash.key)); |
090089c4 | 778 | /* If, for any reason we can't discard this object because of an |
779 | * outstanding request, mark it for pending release */ | |
780 | if (storeEntryLocked(e)) { | |
9174e204 | 781 | storeExpireNow(e); |
a3d5953d | 782 | debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n"); |
2daae136 | 783 | storeReleaseRequest(e); |
6c78a099 | 784 | return; |
090089c4 | 785 | } |
cd748f27 | 786 | if (store_dirs_rebuilding && e->swap_filen > -1) { |
e42d5181 | 787 | storeSetPrivateKey(e); |
788 | if (e->mem_obj) { | |
789 | storeSetMemStatus(e, NOT_IN_MEMORY); | |
790 | destroy_MemObject(e); | |
791 | } | |
cd748f27 | 792 | if (e->swap_filen > -1) { |
f0b4acef | 793 | /* |
794 | * Fake a call to storeLockObject(). When rebuilding is done, | |
795 | * we'll just call storeUnlockObject() on these. | |
796 | */ | |
797 | e->lock_count++; | |
798 | EBIT_SET(e->flags, RELEASE_REQUEST); | |
799 | stackPush(&LateReleaseStack, e); | |
800 | return; | |
801 | } else { | |
802 | destroy_StoreEntry(e); | |
803 | } | |
43d9cf56 | 804 | } |
a65ff22e | 805 | storeLog(STORE_LOG_RELEASE, e); |
cd748f27 | 806 | if (e->swap_filen > -1) { |
807 | storeUnlink(e); | |
a65ff22e | 808 | if (e->swap_status == SWAPOUT_DONE) |
d46a87a8 | 809 | if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) |
cd748f27 | 810 | storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, -1); |
d46a87a8 | 811 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) |
b109de6b | 812 | storeDirSwapLog(e, SWAP_LOG_DEL); |
cd748f27 | 813 | #if 0 |
814 | /* From 2.4. I think we do this in storeUnlink? */ | |
fc8b9fc0 | 815 | storeSwapFileNumberSet(e, -1); |
cd748f27 | 816 | #endif |
090089c4 | 817 | } |
8350fe9b | 818 | storeSetMemStatus(e, NOT_IN_MEMORY); |
30a4f2a8 | 819 | destroy_StoreEntry(e); |
090089c4 | 820 | } |
821 | ||
e42d5181 | 822 | static void |
823 | storeLateRelease(void *unused) | |
824 | { | |
825 | StoreEntry *e; | |
826 | int i; | |
827 | static int n = 0; | |
b2c141d4 | 828 | if (store_dirs_rebuilding) { |
e42d5181 | 829 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); |
830 | return; | |
831 | } | |
832 | for (i = 0; i < 10; i++) { | |
833 | e = stackPop(&LateReleaseStack); | |
834 | if (e == NULL) { | |
835 | /* done! */ | |
1d8e515f | 836 | debug(20, 1) ("storeLateRelease: released %d objects\n", n); |
e42d5181 | 837 | return; |
838 | } | |
839 | storeUnlockObject(e); | |
840 | n++; | |
841 | } | |
842 | eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1); | |
843 | } | |
844 | ||
090089c4 | 845 | /* return 1 if a store entry is locked */ |
cd748f27 | 846 | int |
fe4e214f | 847 | storeEntryLocked(const StoreEntry * e) |
090089c4 | 848 | { |
30a4f2a8 | 849 | if (e->lock_count) |
850 | return 1; | |
8350fe9b | 851 | if (e->swap_status == SWAPOUT_WRITING) |
30a4f2a8 | 852 | return 1; |
a1e47288 | 853 | if (e->store_status == STORE_PENDING) |
854 | return 1; | |
b8890359 | 855 | /* |
856 | * SPECIAL, PUBLIC entries should be "locked" | |
857 | */ | |
d46a87a8 | 858 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) |
b8890359 | 859 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) |
860 | return 1; | |
30a4f2a8 | 861 | return 0; |
090089c4 | 862 | } |
863 | ||
24382924 | 864 | static int |
fe4e214f | 865 | storeEntryValidLength(const StoreEntry * e) |
6602e70e | 866 | { |
ffe4a367 | 867 | int diff; |
d8b249ef | 868 | const HttpReply *reply; |
8a5e92ce | 869 | assert(e->mem_obj != NULL); |
07304bf9 | 870 | reply = e->mem_obj->reply; |
186477c1 | 871 | debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->hash.key)); |
07304bf9 | 872 | debug(20, 5) ("storeEntryValidLength: object_len = %d\n", |
873 | objectLen(e)); | |
874 | debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", | |
875 | reply->hdr_sz); | |
876 | debug(20, 5) ("storeEntryValidLength: content_length = %d\n", | |
d8b249ef | 877 | reply->content_length); |
878 | if (reply->content_length < 0) { | |
1790d392 | 879 | debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", |
186477c1 | 880 | storeKeyText(e->hash.key)); |
ffe4a367 | 881 | return 1; |
882 | } | |
07304bf9 | 883 | if (reply->hdr_sz == 0) { |
1790d392 | 884 | debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", |
186477c1 | 885 | storeKeyText(e->hash.key)); |
ffe4a367 | 886 | return 1; |
887 | } | |
ebf4efff | 888 | if (e->mem_obj->method == METHOD_HEAD) { |
889 | debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n", | |
186477c1 | 890 | storeKeyText(e->hash.key)); |
ebf4efff | 891 | return 1; |
ffe4a367 | 892 | } |
cb69b4c7 | 893 | if (reply->sline.status == HTTP_NOT_MODIFIED) |
ebf4efff | 894 | return 1; |
cb69b4c7 | 895 | if (reply->sline.status == HTTP_NO_CONTENT) |
ebf4efff | 896 | return 1; |
d8b249ef | 897 | diff = reply->hdr_sz + reply->content_length - objectLen(e); |
ebf4efff | 898 | if (diff == 0) |
899 | return 1; | |
900 | debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", | |
901 | diff < 0 ? -diff : diff, | |
e69324cd | 902 | diff < 0 ? "big" : "small", |
186477c1 | 903 | storeKeyText(e->hash.key)); |
ebf4efff | 904 | return 0; |
ffe4a367 | 905 | } |
6602e70e | 906 | |
66cedb85 | 907 | static void |
908 | storeInitHashValues(void) | |
909 | { | |
910 | int i; | |
a7e59001 | 911 | /* Calculate size of hash table (maximum currently 64k buckets). */ |
38792624 | 912 | i = Config.Swap.maxSize / Config.Store.avgObjectSize; |
86101e40 | 913 | debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n", |
66cedb85 | 914 | Config.Swap.maxSize, i); |
38792624 | 915 | i /= Config.Store.objectsPerBucket; |
a3d5953d | 916 | debug(20, 1) ("Target number of buckets: %d\n", i); |
66cedb85 | 917 | /* ideally the full scan period should be configurable, for the |
918 | * moment it remains at approximately 24 hours. */ | |
9fb13bb6 | 919 | store_hash_buckets = storeKeyHashBuckets(i); |
c63ea1e2 | 920 | debug(20, 1) ("Using %d Store buckets\n", store_hash_buckets); |
43a70238 | 921 | debug(20, 1) ("Max Mem size: %d KB\n", Config.memMaxSize >> 10); |
a47b9029 | 922 | debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize); |
66cedb85 | 923 | } |
924 | ||
b8d8561b | 925 | void |
926 | storeInit(void) | |
c943f331 | 927 | { |
25535cbe | 928 | storeKeyInit(); |
66cedb85 | 929 | storeInitHashValues(); |
9fb13bb6 | 930 | store_table = hash_create(storeKeyHashCmp, |
931 | store_hash_buckets, storeKeyHashHash); | |
6a566b9c | 932 | mem_policy = createRemovalPolicy(Config.memPolicy); |
8638fc66 | 933 | storeDigestInit(); |
e3ef2b09 | 934 | storeLogOpen(); |
e42d5181 | 935 | stackInit(&LateReleaseStack); |
936 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); | |
9838d6c8 | 937 | storeDirInit(); |
b2c141d4 | 938 | storeRebuildStart(); |
4cb56589 | 939 | cachemgrRegister("storedir", |
22f3fd98 | 940 | "Store Directory Stats", |
1da3b90b | 941 | storeDirStats, 0, 1); |
8423ff74 | 942 | cachemgrRegister("store_check_cachable_stats", |
943 | "storeCheckCachable() Stats", | |
944 | storeCheckCachableStats, 0, 1); | |
65a53c8e | 945 | cachemgrRegister("store_io", |
946 | "Store IO Interface Stats", | |
947 | storeIOStats, 0, 1); | |
b1c0cc67 | 948 | } |
c943f331 | 949 | |
b8d8561b | 950 | void |
951 | storeConfigure(void) | |
b1c0cc67 | 952 | { |
b1c0cc67 | 953 | store_swap_high = (long) (((float) Config.Swap.maxSize * |
954 | (float) Config.Swap.highWaterMark) / (float) 100); | |
955 | store_swap_low = (long) (((float) Config.Swap.maxSize * | |
956 | (float) Config.Swap.lowWaterMark) / (float) 100); | |
43a70238 | 957 | store_pages_max = Config.memMaxSize / SM_PAGE_SIZE; |
090089c4 | 958 | } |
959 | ||
56f29785 | 960 | static int |
8350fe9b | 961 | storeKeepInMemory(const StoreEntry * e) |
56f29785 | 962 | { |
8350fe9b | 963 | MemObject *mem = e->mem_obj; |
964 | if (mem == NULL) | |
56f29785 | 965 | return 0; |
18fe65d0 | 966 | if (mem->data_hdr.head == NULL) |
8350fe9b | 967 | return 0; |
968 | return mem->inmem_lo == 0; | |
56f29785 | 969 | } |
970 | ||
b8d8561b | 971 | void |
972 | storeNegativeCache(StoreEntry * e) | |
79b5cc5f | 973 | { |
ff5731b1 | 974 | e->expires = squid_curtime + Config.negativeTtl; |
d46a87a8 | 975 | EBIT_SET(e->flags, ENTRY_NEGCACHED); |
79b5cc5f | 976 | } |
0a21bd84 | 977 | |
978 | void | |
979 | storeFreeMemory(void) | |
980 | { | |
ec878047 | 981 | hashFreeItems(store_table, destroy_StoreEntry); |
0a21bd84 | 982 | hashFreeMemory(store_table); |
afe95a7e | 983 | store_table = NULL; |
9bc73deb | 984 | #if USE_CACHE_DIGESTS |
8638fc66 | 985 | if (store_digest) |
986 | cacheDigestDestroy(store_digest); | |
c68e9c6b | 987 | #endif |
8638fc66 | 988 | store_digest = NULL; |
0a21bd84 | 989 | } |
a7e59001 | 990 | |
991 | int | |
992 | expiresMoreThan(time_t expires, time_t when) | |
993 | { | |
48f44632 | 994 | if (expires < 0) /* No Expires given */ |
995 | return 1; | |
996 | return (expires > (squid_curtime + when)); | |
a7e59001 | 997 | } |
fe54d06d | 998 | |
999 | int | |
1000 | storeEntryValidToSend(StoreEntry * e) | |
1001 | { | |
d46a87a8 | 1002 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
fe54d06d | 1003 | return 0; |
d46a87a8 | 1004 | if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) |
44f78c24 | 1005 | if (e->expires <= squid_curtime) |
fe54d06d | 1006 | return 0; |
b7fe0ab0 | 1007 | if (EBIT_TEST(e->flags, ENTRY_ABORTED)) |
fe54d06d | 1008 | return 0; |
1009 | return 1; | |
1010 | } | |
62663274 | 1011 | |
ca98227c | 1012 | void |
1013 | storeTimestampsSet(StoreEntry * entry) | |
1014 | { | |
2246b732 | 1015 | const HttpReply *reply = entry->mem_obj->reply; |
2f58241d | 1016 | time_t served_date = reply->date; |
efd900cb | 1017 | int age = httpHeaderGetInt(&reply->header, HDR_AGE); |
1018 | /* | |
1019 | * The timestamp calculations below tries to mimic the properties | |
1020 | * of the age calculation in RFC2616 section 13.2.3. The implementaion | |
1021 | * isn't complete, and the most notable exception from the RFC is that | |
1022 | * this does not account for response_delay, but it probably does | |
1023 | * not matter much as this is calculated immediately when the headers | |
1024 | * are received, not when the whole response has been received. | |
1025 | */ | |
2f58241d | 1026 | /* make sure that 0 <= served_date <= squid_curtime */ |
1027 | if (served_date < 0 || served_date > squid_curtime) | |
cb69b4c7 | 1028 | served_date = squid_curtime; |
efd900cb | 1029 | /* |
1030 | * Compensate with Age header if origin server clock is ahead of us | |
1031 | * and there is a cache in between us and the origin server | |
1032 | */ | |
1033 | if (age > squid_curtime - served_date) | |
1034 | served_date = squid_curtime - age; | |
d8b249ef | 1035 | entry->expires = reply->expires; |
1c3e77cd | 1036 | entry->lastmod = reply->last_modified; |
ca98227c | 1037 | entry->timestamp = served_date; |
1038 | } | |
429fdbec | 1039 | |
bfcaf585 | 1040 | void |
1041 | storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) | |
1042 | { | |
1043 | MemObject *mem = e->mem_obj; | |
1044 | assert(mem); | |
1045 | assert(mem->abort.callback == NULL); | |
1046 | mem->abort.callback = cb; | |
1047 | mem->abort.data = data; | |
1048 | } | |
1049 | ||
1050 | void | |
1051 | storeUnregisterAbort(StoreEntry * e) | |
1052 | { | |
1053 | MemObject *mem = e->mem_obj; | |
1054 | assert(mem); | |
1055 | mem->abort.callback = NULL; | |
1056 | } | |
88738790 | 1057 | |
1058 | void | |
1059 | storeMemObjectDump(MemObject * mem) | |
1060 | { | |
18fe65d0 | 1061 | debug(20, 1) ("MemObject->data.head: %p\n", |
1062 | mem->data_hdr.head); | |
1063 | debug(20, 1) ("MemObject->data.tail: %p\n", | |
1064 | mem->data_hdr.tail); | |
1065 | debug(20, 1) ("MemObject->data.origin_offset: %d\n", | |
1066 | mem->data_hdr.origin_offset); | |
88738790 | 1067 | debug(20, 1) ("MemObject->start_ping: %d.%06d\n", |
5f6ac48b | 1068 | (int) mem->start_ping.tv_sec, |
1069 | (int) mem->start_ping.tv_usec); | |
8350fe9b | 1070 | debug(20, 1) ("MemObject->inmem_hi: %d\n", |
5f6ac48b | 1071 | (int) mem->inmem_hi); |
8350fe9b | 1072 | debug(20, 1) ("MemObject->inmem_lo: %d\n", |
5f6ac48b | 1073 | (int) mem->inmem_lo); |
88738790 | 1074 | debug(20, 1) ("MemObject->clients: %p\n", |
1075 | mem->clients); | |
1076 | debug(20, 1) ("MemObject->nclients: %d\n", | |
1077 | mem->nclients); | |
88738790 | 1078 | debug(20, 1) ("MemObject->reply: %p\n", |
1079 | mem->reply); | |
1080 | debug(20, 1) ("MemObject->request: %p\n", | |
1081 | mem->request); | |
1082 | debug(20, 1) ("MemObject->log_url: %p %s\n", | |
1083 | mem->log_url, | |
1084 | checkNullString(mem->log_url)); | |
1085 | } | |
8350fe9b | 1086 | |
f09f5b26 | 1087 | void |
569c4638 | 1088 | storeEntryDump(const StoreEntry * e, int l) |
d377699f | 1089 | { |
186477c1 | 1090 | debug(20, l) ("StoreEntry->key: %s\n", storeKeyText(e->hash.key)); |
1091 | debug(20, l) ("StoreEntry->next: %p\n", e->hash.next); | |
e3ef2b09 | 1092 | debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj); |
1093 | debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp); | |
1094 | debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); | |
1095 | debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires); | |
1096 | debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod); | |
07304bf9 | 1097 | debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz); |
e3ef2b09 | 1098 | debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount); |
415e0dd2 | 1099 | debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e)); |
cd748f27 | 1100 | debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn); |
1101 | debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen); | |
e3ef2b09 | 1102 | debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count); |
1103 | debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status); | |
1104 | debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status); | |
1105 | debug(20, l) ("StoreEntry->store_status: %d\n", (int) e->store_status); | |
1106 | debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status); | |
d377699f | 1107 | } |
1108 | ||
1f38f50a | 1109 | /* |
1110 | * NOTE, this function assumes only two mem states | |
1111 | */ | |
f09f5b26 | 1112 | void |
8350fe9b | 1113 | storeSetMemStatus(StoreEntry * e, int new_status) |
1114 | { | |
b93bcace | 1115 | MemObject *mem = e->mem_obj; |
8350fe9b | 1116 | if (new_status == e->mem_status) |
1117 | return; | |
b93bcace | 1118 | assert(mem != NULL); |
1119 | if (new_status == IN_MEMORY) { | |
1120 | assert(mem->inmem_lo == 0); | |
6a566b9c | 1121 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { |
1122 | debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n", | |
1123 | mem->url); | |
1124 | } else { | |
1125 | mem_policy->Add(mem_policy, e, &mem->repl); | |
1126 | debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n", | |
1127 | mem->url); | |
2b906e48 | 1128 | } |
59c4d35b | 1129 | hot_obj_count++; |
b93bcace | 1130 | } else { |
6a566b9c | 1131 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { |
c1dd71ae | 1132 | debug(20, 4) ("storeSetMemStatus: special entry %s\n", |
1133 | mem->url); | |
6a566b9c | 1134 | } else { |
1135 | mem_policy->Remove(mem_policy, e, &mem->repl); | |
1136 | debug(20, 4) ("storeSetMemStatus: removed mem node %s\n", | |
1137 | mem->url); | |
2b906e48 | 1138 | } |
59c4d35b | 1139 | hot_obj_count--; |
b93bcace | 1140 | } |
8350fe9b | 1141 | e->mem_status = new_status; |
1142 | } | |
6e86c3e8 | 1143 | |
9fb13bb6 | 1144 | const char * |
1145 | storeUrl(const StoreEntry * e) | |
1146 | { | |
1147 | if (e == NULL) | |
24ffafb4 | 1148 | return "[null_entry]"; |
9fb13bb6 | 1149 | else if (e->mem_obj == NULL) |
24ffafb4 | 1150 | return "[null_mem_obj]"; |
9fb13bb6 | 1151 | else |
1152 | return e->mem_obj->url; | |
1153 | } | |
24ffafb4 | 1154 | |
1155 | void | |
1156 | storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url) | |
1157 | { | |
1158 | if (e->mem_obj) | |
1159 | return; | |
1160 | e->mem_obj = new_MemObject(url, log_url); | |
1161 | } | |
64763c37 | 1162 | |
438fc1e3 | 1163 | /* this just sets DELAY_SENDING */ |
1164 | void | |
8daca701 | 1165 | storeBuffer(StoreEntry * e) |
438fc1e3 | 1166 | { |
d46a87a8 | 1167 | EBIT_SET(e->flags, DELAY_SENDING); |
438fc1e3 | 1168 | } |
1169 | ||
1170 | /* this just clears DELAY_SENDING and Invokes the handlers */ | |
1171 | void | |
8daca701 | 1172 | storeBufferFlush(StoreEntry * e) |
438fc1e3 | 1173 | { |
d46a87a8 | 1174 | EBIT_CLR(e->flags, DELAY_SENDING); |
8daca701 | 1175 | InvokeHandlers(e); |
2391a162 | 1176 | storeSwapOut(e); |
25535cbe | 1177 | } |
07304bf9 | 1178 | |
1179 | int | |
1180 | objectLen(const StoreEntry * e) | |
1181 | { | |
1182 | assert(e->mem_obj != NULL); | |
1183 | return e->mem_obj->object_sz; | |
1184 | } | |
1185 | ||
1186 | int | |
1187 | contentLen(const StoreEntry * e) | |
1188 | { | |
1189 | assert(e->mem_obj != NULL); | |
1190 | assert(e->mem_obj->reply != NULL); | |
1191 | return e->mem_obj->object_sz - e->mem_obj->reply->hdr_sz; | |
1192 | } | |
f3986a15 | 1193 | |
1194 | HttpReply * | |
1195 | storeEntryReply(StoreEntry * e) | |
1196 | { | |
1197 | if (NULL == e) | |
3d02186d | 1198 | return NULL; |
f3986a15 | 1199 | if (NULL == e->mem_obj) |
3d02186d | 1200 | return NULL; |
f3986a15 | 1201 | return e->mem_obj->reply; |
1202 | } | |
db1cd23c | 1203 | |
1204 | void | |
1205 | storeEntryReset(StoreEntry * e) | |
1206 | { | |
1207 | MemObject *mem = e->mem_obj; | |
1208 | debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e)); | |
2391a162 | 1209 | assert(mem->swapout.sio == NULL); |
db1cd23c | 1210 | stmemFree(&mem->data_hdr); |
1211 | mem->inmem_hi = mem->inmem_lo = 0; | |
1212 | httpReplyDestroy(mem->reply); | |
1213 | mem->reply = httpReplyCreate(); | |
9bc73deb | 1214 | e->expires = e->lastmod = e->timestamp = -1; |
db1cd23c | 1215 | } |
2b906e48 | 1216 | |
cd748f27 | 1217 | /* |
1218 | * storeFsInit | |
1219 | * | |
1220 | * This routine calls the SETUP routine for each fs type. | |
1221 | * I don't know where the best place for this is, and I'm not going to shuffle | |
1222 | * around large chunks of code right now (that can be done once its working.) | |
1223 | */ | |
1224 | void | |
1225 | storeFsInit(void) | |
1226 | { | |
22d38e05 | 1227 | storeReplSetup(); |
cd748f27 | 1228 | storeFsSetup(); |
1229 | } | |
1230 | ||
1231 | ||
1232 | /* | |
1233 | * similar to above, but is called when a graceful shutdown is to occur | |
1234 | * of each fs module. | |
1235 | */ | |
1236 | void | |
1237 | storeFsDone(void) | |
1238 | { | |
1239 | int i = 0; | |
1240 | ||
1241 | while (storefs_list[i].typestr != NULL) { | |
a4b8110e | 1242 | storefs_list[i].donefunc(); |
cd748f27 | 1243 | i++; |
1244 | } | |
1245 | } | |
1246 | ||
1247 | /* | |
1248 | * called to add another store fs module | |
1249 | */ | |
a4b8110e | 1250 | void |
1251 | storeFsAdd(char *type, STSETUP * setup) | |
cd748f27 | 1252 | { |
1253 | int i; | |
1254 | /* find the number of currently known storefs types */ | |
1255 | for (i = 0; storefs_list && storefs_list[i].typestr; i++) { | |
a4b8110e | 1256 | assert(strcmp(storefs_list[i].typestr, type) != 0); |
cd748f27 | 1257 | } |
1258 | /* add the new type */ | |
1259 | storefs_list = xrealloc(storefs_list, (i + 2) * sizeof(storefs_entry_t)); | |
a4b8110e | 1260 | memset(&storefs_list[i + 1], 0, sizeof(storefs_entry_t)); |
cd748f27 | 1261 | storefs_list[i].typestr = type; |
1262 | /* Call the FS to set up capabilities and initialize the FS driver */ | |
1263 | setup(&storefs_list[i]); | |
1264 | } | |
1265 | ||
22d38e05 | 1266 | /* |
1267 | * called to add another store removal policy module | |
1268 | */ | |
1269 | void | |
1270 | storeReplAdd(char *type, REMOVALPOLICYCREATE * create) | |
1271 | { | |
1272 | int i; | |
1273 | /* find the number of currently known repl types */ | |
1274 | for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) { | |
1275 | assert(strcmp(storerepl_list[i].typestr, type) != 0); | |
1276 | } | |
1277 | /* add the new type */ | |
1278 | storerepl_list = xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t)); | |
1279 | memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t)); | |
1280 | storerepl_list[i].typestr = type; | |
1281 | storerepl_list[i].create = create; | |
1282 | } | |
1283 | ||
1284 | /* | |
1285 | * Create a removal policy instance | |
1286 | */ | |
1287 | RemovalPolicy * | |
1288 | createRemovalPolicy(RemovalPolicySettings * settings) | |
1289 | { | |
1290 | storerepl_entry_t *r; | |
1291 | for (r = storerepl_list; r && r->typestr; r++) { | |
1292 | if (strcmp(r->typestr, settings->type) == 0) | |
1293 | return r->create(settings->args); | |
1294 | } | |
0c5ccf11 | 1295 | debug(20, 1) ("ERROR: Unknown policy %s\n", settings->type); |
1296 | debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n"); | |
1297 | debug(20, 1) ("ERROR: and memory_replacement_policy in squid.conf!\n"); | |
1298 | fatalf("ERROR: Unknown policy %s\n", settings->type); | |
f0debecb | 1299 | return NULL; /* NOTREACHED */ |
22d38e05 | 1300 | } |
1301 | ||
cd748f27 | 1302 | #if 0 |
fc8b9fc0 | 1303 | void |
1304 | storeSwapFileNumberSet(StoreEntry * e, sfileno filn) | |
1305 | { | |
1306 | if (e->swap_file_number == filn) | |
1307 | return; | |
1308 | if (filn < 0) { | |
1309 | assert(-1 == filn); | |
1310 | storeDirMapBitReset(e->swap_file_number); | |
1311 | storeDirLRUDelete(e); | |
1312 | e->swap_file_number = -1; | |
1313 | } else { | |
1314 | assert(-1 == e->swap_file_number); | |
1315 | storeDirMapBitSet(e->swap_file_number = filn); | |
1316 | storeDirLRUAdd(e); | |
1317 | } | |
1318 | } | |
cd748f27 | 1319 | #endif |