]>
Commit | Line | Data |
---|---|---|
164f7660 | 1 | |
30a4f2a8 | 2 | /* |
25b6a907 | 3 | * $Id: store.cc,v 1.586 2006/04/22 05:29:20 robertc Exp $ |
30a4f2a8 | 4 | * |
8638fc66 | 5 | * DEBUG: section 20 Storage Manager |
30a4f2a8 | 6 | * AUTHOR: Harvest Derived |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
e25c139f | 9 | * ---------------------------------------------------------- |
30a4f2a8 | 10 | * |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
30a4f2a8 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
2b6662ba | 24 | * |
30a4f2a8 | 25 | * This program is distributed in the hope that it will be useful, |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
2b6662ba | 29 | * |
30a4f2a8 | 30 | * You should have received a copy of the GNU General Public License |
31 | * along with this program; if not, write to the Free Software | |
cbdec147 | 32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. |
e25c139f | 33 | * |
c943f331 | 34 | */ |
090089c4 | 35 | |
f09f5b26 | 36 | #include "squid.h" |
e6ccf245 | 37 | #include "Store.h" |
38 | #include "StoreClient.h" | |
528b2c61 | 39 | #include "stmem.h" |
40 | #include "HttpReply.h" | |
41 | #include "HttpRequest.h" | |
42 | #include "MemObject.h" | |
43 | #include "mem_node.h" | |
44 | #include "StoreMeta.h" | |
d3b3ab85 | 45 | #include "SwapDir.h" |
b67e2c8c | 46 | #if DELAY_POOLS |
47 | #include "DelayPools.h" | |
48 | #endif | |
91caca83 | 49 | #include "Stack.h" |
090089c4 | 50 | |
528b2c61 | 51 | static STMCB storeWriteComplete; |
52 | ||
090089c4 | 53 | #define REBUILD_TIMESTAMP_DELTA_MAX 2 |
227fbb74 | 54 | |
fcf5283d | 55 | #define STORE_IN_MEM_BUCKETS (229) |
090089c4 | 56 | |
0ee4272b | 57 | const char *memStatusStr[] = |
62e76326 | 58 | { |
59 | "NOT_IN_MEMORY", | |
60 | "IN_MEMORY" | |
61 | }; | |
9dfb6c1c | 62 | |
0ee4272b | 63 | const char *pingStatusStr[] = |
62e76326 | 64 | { |
65 | "PING_NONE", | |
66 | "PING_WAITING", | |
67 | "PING_DONE" | |
68 | }; | |
9dfb6c1c | 69 | |
0ee4272b | 70 | const char *storeStatusStr[] = |
62e76326 | 71 | { |
72 | "STORE_OK", | |
73 | "STORE_PENDING" | |
74 | }; | |
9dfb6c1c | 75 | |
0ee4272b | 76 | const char *swapStatusStr[] = |
62e76326 | 77 | { |
78 | "SWAPOUT_NONE", | |
79 | "SWAPOUT_WRITING", | |
80 | "SWAPOUT_DONE" | |
81 | }; | |
9dfb6c1c | 82 | |
62e76326 | 83 | typedef struct lock_ctrl_t |
84 | { | |
582b6456 | 85 | SIH *callback; |
0a0bf5db | 86 | void *callback_data; |
87 | StoreEntry *e; | |
62e76326 | 88 | } |
89 | ||
90 | lock_ctrl_t; | |
0a0bf5db | 91 | |
65a53c8e | 92 | extern OBJH storeIOStats; |
93 | ||
25b6a907 | 94 | |
95 | /* | |
96 | * This defines an repl type | |
97 | */ | |
98 | ||
99 | typedef struct _storerepl_entry storerepl_entry_t; | |
100 | ||
101 | struct _storerepl_entry | |
102 | { | |
103 | const char *typestr; | |
104 | REMOVALPOLICYCREATE *create; | |
105 | }; | |
106 | ||
107 | static storerepl_entry_t *storerepl_list = NULL; | |
108 | ||
109 | ||
e3ef2b09 | 110 | /* |
111 | * local function prototypes | |
112 | */ | |
f5b8bbc4 | 113 | static void storeGetMemSpace(int); |
b93bcace | 114 | static void storeHashDelete(StoreEntry *); |
6cf028ab | 115 | static void destroy_MemObject(StoreEntry *); |
f5b8bbc4 | 116 | static void storePurgeMem(StoreEntry *); |
007b8be4 | 117 | static int getKeyCounter(void); |
8350fe9b | 118 | static int storeKeepInMemory(const StoreEntry *); |
8423ff74 | 119 | static OBJH storeCheckCachableStats; |
e42d5181 | 120 | static EVH storeLateRelease; |
a21fbb54 | 121 | |
e3ef2b09 | 122 | /* |
123 | * local variables | |
124 | */ | |
91caca83 | 125 | static Stack<StoreEntry*> LateReleaseStack; |
b001e822 | 126 | MemImplementingAllocator *StoreEntry::pool = NULL; |
e6ccf245 | 127 | |
c8f4eac4 | 128 | StorePointer Store::CurrentRoot = NULL; |
129 | ||
130 | void | |
131 | Store::Root(Store * aRoot) | |
132 | { | |
133 | CurrentRoot = aRoot; | |
134 | } | |
135 | ||
136 | void | |
137 | Store::Root(StorePointer aRoot) | |
138 | { | |
139 | Root(aRoot.getRaw()); | |
140 | } | |
141 | ||
142 | void | |
143 | Store::Stats(StoreEntry * output) | |
144 | { | |
145 | assert (output); | |
146 | Root().stat(*output); | |
147 | } | |
148 | ||
149 | void | |
150 | Store::create() | |
151 | {} | |
152 | ||
153 | void | |
154 | Store::diskFull() | |
155 | {} | |
156 | ||
157 | void | |
158 | Store::sync() | |
159 | {} | |
160 | ||
161 | void | |
162 | Store::unlink (StoreEntry &anEntry) | |
163 | { | |
164 | fatal("Store::unlink on invalid Store\n"); | |
165 | } | |
166 | ||
e6ccf245 | 167 | void * |
3b13a8fd | 168 | StoreEntry::operator new (size_t bytecount) |
e6ccf245 | 169 | { |
3b13a8fd | 170 | assert (bytecount == sizeof (StoreEntry)); |
62e76326 | 171 | |
e6ccf245 | 172 | if (!pool) { |
b001e822 | 173 | pool = MemPools::GetInstance().create ("StoreEntry", bytecount); |
174 | pool->setChunkSize(2048 * 1024); | |
e6ccf245 | 175 | } |
62e76326 | 176 | |
b001e822 | 177 | return pool->alloc(); |
e6ccf245 | 178 | } |
179 | ||
180 | void | |
3b13a8fd | 181 | StoreEntry::operator delete (void *address) |
e6ccf245 | 182 | { |
b001e822 | 183 | pool->free(address); |
e6ccf245 | 184 | } |
185 | ||
5ed72359 | 186 | void |
187 | StoreEntry::makePublic() | |
188 | { | |
189 | /* This object can be cached for a long time */ | |
190 | ||
191 | if (EBIT_TEST(flags, ENTRY_CACHABLE)) | |
192 | storeSetPublicKey(this); | |
193 | } | |
194 | ||
195 | void | |
196 | StoreEntry::makePrivate() | |
197 | { | |
198 | /* This object should never be cached at all */ | |
199 | storeExpireNow(this); | |
200 | storeReleaseRequest(this); /* delete object when not used */ | |
201 | /* storeReleaseRequest clears ENTRY_CACHABLE flag */ | |
202 | } | |
203 | ||
204 | void | |
205 | StoreEntry::cacheNegatively() | |
206 | { | |
207 | /* This object may be negatively cached */ | |
208 | storeNegativeCache(this); | |
209 | ||
210 | if (EBIT_TEST(flags, ENTRY_CACHABLE)) | |
211 | storeSetPublicKey(this); | |
212 | } | |
213 | ||
e6ccf245 | 214 | size_t |
3b13a8fd | 215 | StoreEntry::inUseCount() |
e6ccf245 | 216 | { |
217 | if (!pool) | |
62e76326 | 218 | return 0; |
219 | ||
e6ccf245 | 220 | MemPoolStats stats; |
62e76326 | 221 | |
b001e822 | 222 | pool->getStats (&stats); |
62e76326 | 223 | |
e6ccf245 | 224 | return stats.items_inuse; |
225 | } | |
226 | ||
332dafa2 | 227 | const char * |
3b13a8fd | 228 | StoreEntry::getMD5Text() const |
332dafa2 | 229 | { |
230 | return storeKeyText((const cache_key *)key); | |
231 | } | |
232 | ||
a46d2c0e | 233 | #include "comm.h" |
234 | ||
235 | void | |
236 | StoreEntry::DeferReader(void *theContext, CommRead const &aRead) | |
e6ccf245 | 237 | { |
a46d2c0e | 238 | StoreEntry *anEntry = (StoreEntry *)theContext; |
239 | anEntry->delayAwareRead(aRead.fd, | |
240 | aRead.buf, | |
241 | aRead.len, | |
2d8c0b1a | 242 | aRead.callback.handler, |
243 | aRead.callback.data); | |
e6ccf245 | 244 | } |
66cedb85 | 245 | |
a46d2c0e | 246 | void |
247 | StoreEntry::delayAwareRead(int fd, char *buf, int len, IOCB *handler, void *data) | |
248 | { | |
249 | size_t amountToRead = bytesWanted(Range<size_t>(0, len)); | |
250 | /* sketch: readdeferer* = getdeferer. | |
251 | * ->deferRead (fd, buf, len, handler, data, DelayAwareRead, this) | |
252 | */ | |
253 | ||
254 | if (amountToRead == 0) { | |
255 | assert (mem_obj); | |
256 | /* read ahead limit */ | |
257 | /* Perhaps these two calls should both live in MemObject */ | |
a5f42284 | 258 | #if DELAY_POOLS |
a46d2c0e | 259 | |
260 | if (!mem_obj->readAheadPolicyCanRead()) { | |
a5f42284 | 261 | #endif |
a46d2c0e | 262 | mem_obj->delayRead(DeferredRead(DeferReader, this, CommRead(fd, buf, len, handler, data))); |
263 | return; | |
a5f42284 | 264 | #if DELAY_POOLS |
265 | ||
a46d2c0e | 266 | } |
267 | ||
268 | /* delay id limit */ | |
269 | mem_obj->mostBytesAllowed().delayRead(DeferredRead(DeferReader, this, CommRead(fd, buf, len, handler, data))); | |
270 | ||
271 | return; | |
a5f42284 | 272 | |
273 | #endif | |
274 | ||
a46d2c0e | 275 | } |
276 | ||
277 | comm_read(fd, buf, amountToRead, handler, data); | |
278 | } | |
279 | ||
280 | size_t | |
281 | StoreEntry::bytesWanted (Range<size_t> const aRange) const | |
528b2c61 | 282 | { |
a46d2c0e | 283 | assert (aRange.size()); |
62e76326 | 284 | |
528b2c61 | 285 | if (mem_obj == NULL) |
a46d2c0e | 286 | return aRange.end - 1; |
62e76326 | 287 | |
bc87dc25 | 288 | #if URL_CHECKSUM_DEBUG |
62e76326 | 289 | |
528b2c61 | 290 | mem_obj->checkUrlChecksum(); |
62e76326 | 291 | |
528b2c61 | 292 | #endif |
62e76326 | 293 | |
a46d2c0e | 294 | /* Always read *something* here - we haven't got the header yet */ |
295 | if (EBIT_TEST(flags, ENTRY_FWD_HDR_WAIT)) | |
296 | return aRange.end - 1; | |
62e76326 | 297 | |
a46d2c0e | 298 | if (!mem_obj->readAheadPolicyCanRead()) |
299 | return 0; | |
62e76326 | 300 | |
a46d2c0e | 301 | return mem_obj->mostBytesWanted(aRange.end - 1); |
302 | } | |
62e76326 | 303 | |
a46d2c0e | 304 | bool |
305 | StoreEntry::checkDeferRead(int fd) const | |
306 | { | |
307 | return (bytesWanted(Range<size_t>(0,INT_MAX)) == 0); | |
308 | } | |
62e76326 | 309 | |
a46d2c0e | 310 | void |
311 | StoreEntry::setNoDelay (bool const newValue) | |
312 | { | |
313 | if (mem_obj) | |
314 | mem_obj->setNoDelay(newValue); | |
528b2c61 | 315 | } |
bc87dc25 | 316 | |
528b2c61 | 317 | store_client_t |
318 | StoreEntry::storeClientType() const | |
227fbb74 | 319 | { |
7d31d5fa | 320 | /* The needed offset isn't in memory |
321 | * XXX TODO: this is wrong for range requests | |
322 | * as the needed offset may *not* be 0, AND | |
323 | * offset 0 in the memory object is the HTTP headers. | |
324 | */ | |
325 | ||
528b2c61 | 326 | if (mem_obj->inmem_lo) |
62e76326 | 327 | return STORE_DISK_CLIENT; |
328 | ||
528b2c61 | 329 | if (EBIT_TEST(flags, ENTRY_ABORTED)) { |
62e76326 | 330 | /* I don't think we should be adding clients to aborted entries */ |
331 | debug(20, 1) ("storeClientType: adding to ENTRY_ABORTED entry\n"); | |
332 | return STORE_MEM_CLIENT; | |
528b2c61 | 333 | } |
62e76326 | 334 | |
528b2c61 | 335 | if (store_status == STORE_OK) { |
7d31d5fa | 336 | /* the object has completed. */ |
337 | ||
62e76326 | 338 | if (mem_obj->inmem_lo == 0 && !isEmpty()) |
7d31d5fa | 339 | /* hot object */ |
62e76326 | 340 | return STORE_MEM_CLIENT; |
341 | else | |
342 | return STORE_DISK_CLIENT; | |
528b2c61 | 343 | } |
62e76326 | 344 | |
528b2c61 | 345 | /* here and past, entry is STORE_PENDING */ |
346 | /* | |
347 | * If this is the first client, let it be the mem client | |
348 | */ | |
349 | if (mem_obj->nclients == 1) | |
62e76326 | 350 | return STORE_MEM_CLIENT; |
351 | ||
528b2c61 | 352 | /* |
353 | * If there is no disk file to open yet, we must make this a | |
354 | * mem client. If we can't open the swapin file before writing | |
355 | * to the client, there is no guarantee that we will be able | |
356 | * to open it later when we really need it. | |
357 | */ | |
358 | if (swap_status == SWAPOUT_NONE) | |
62e76326 | 359 | return STORE_MEM_CLIENT; |
360 | ||
528b2c61 | 361 | /* |
362 | * otherwise, make subsequent clients read from disk so they | |
363 | * can not delay the first, and vice-versa. | |
364 | */ | |
365 | return STORE_DISK_CLIENT; | |
227fbb74 | 366 | } |
367 | ||
c8f4eac4 | 368 | StoreEntry::StoreEntry() |
090089c4 | 369 | { |
59dbbc5c | 370 | debugs(20, 3, HERE << "new StoreEntry " << this); |
c8f4eac4 | 371 | mem_obj = NULL; |
62e76326 | 372 | |
c8f4eac4 | 373 | expires = lastmod = lastref = timestamp = -1; |
62e76326 | 374 | |
c8f4eac4 | 375 | swap_filen = -1; |
376 | swap_dirn = -1; | |
377 | } | |
62e76326 | 378 | |
c8f4eac4 | 379 | StoreEntry::StoreEntry(const char *url, const char *log_url) |
380 | { | |
59dbbc5c | 381 | debugs(20, 3, HERE << "new StoreEntry " << this); |
c8f4eac4 | 382 | mem_obj = new MemObject(url, log_url); |
62e76326 | 383 | |
c8f4eac4 | 384 | expires = lastmod = lastref = timestamp = -1; |
385 | ||
386 | swap_filen = -1; | |
387 | swap_dirn = -1; | |
090089c4 | 388 | } |
389 | ||
b8d8561b | 390 | static void |
0e473d70 | 391 | destroy_MemObject(StoreEntry * e) |
090089c4 | 392 | { |
59dbbc5c | 393 | debugs(20, 3, HERE << "destroy mem_obj" << e->mem_obj); |
528b2c61 | 394 | storeSetMemStatus(e, NOT_IN_MEMORY); |
6cf028ab | 395 | MemObject *mem = e->mem_obj; |
41f7e38d | 396 | e->mem_obj = NULL; |
528b2c61 | 397 | delete mem; |
090089c4 | 398 | } |
399 | ||
c8f4eac4 | 400 | void |
528b2c61 | 401 | destroyStoreEntry(void *data) |
090089c4 | 402 | { |
59dbbc5c | 403 | debugs(20, 3, HERE << "destroyStoreEntry: destroying " << data); |
c8f4eac4 | 404 | StoreEntry *e = static_cast<StoreEntry *>(static_cast<hash_link *>(data)); |
9e975e4e | 405 | assert(e != NULL); |
62e76326 | 406 | |
e6ccf245 | 407 | if (e == NullStoreEntry::getInstance()) |
62e76326 | 408 | return; |
409 | ||
528b2c61 | 410 | destroy_MemObject(e); |
62e76326 | 411 | |
043b055b | 412 | storeHashDelete(e); |
62e76326 | 413 | |
332dafa2 | 414 | assert(e->key == NULL); |
62e76326 | 415 | |
e6ccf245 | 416 | delete e; |
227fbb74 | 417 | } |
090089c4 | 418 | |
090089c4 | 419 | /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ |
420 | ||
f09f5b26 | 421 | void |
9fb13bb6 | 422 | storeHashInsert(StoreEntry * e, const cache_key * key) |
090089c4 | 423 | { |
a3d5953d | 424 | debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", |
62e76326 | 425 | e, storeKeyText(key)); |
332dafa2 | 426 | e->key = storeKeyDup(key); |
427 | hash_join(store_table, e); | |
090089c4 | 428 | } |
429 | ||
b93bcace | 430 | static void |
b8d8561b | 431 | storeHashDelete(StoreEntry * e) |
090089c4 | 432 | { |
332dafa2 | 433 | hash_remove_link(store_table, e); |
434 | storeKeyFree((const cache_key *)e->key); | |
435 | e->key = NULL; | |
090089c4 | 436 | } |
437 | ||
090089c4 | 438 | /* -------------------------------------------------------------------------- */ |
439 | ||
090089c4 | 440 | |
441 | /* get rid of memory copy of the object */ | |
24382924 | 442 | static void |
b8d8561b | 443 | storePurgeMem(StoreEntry * e) |
090089c4 | 444 | { |
2aca8433 | 445 | if (e->mem_obj == NULL) |
62e76326 | 446 | return; |
447 | ||
9fb13bb6 | 448 | debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", |
62e76326 | 449 | e->getMD5Text()); |
450 | ||
6cf028ab | 451 | destroy_MemObject(e); |
62e76326 | 452 | |
06e8899b | 453 | if (e->swap_status != SWAPOUT_DONE) |
62e76326 | 454 | storeRelease(e); |
090089c4 | 455 | } |
456 | ||
c8f4eac4 | 457 | /* RBC 20050104 this is wrong- memory ref counting |
458 | * is not at all equivalent to the store 'usage' concept | |
459 | * which the replacement policies should be acting upon. | |
460 | * specifically, object iteration within stores needs | |
461 | * memory ref counting to prevent race conditions, | |
462 | * but this should not influence store replacement. | |
463 | */ | |
6a566b9c | 464 | void |
465 | storeLockObject(StoreEntry * e) | |
466 | { | |
cd748f27 | 467 | e->lock_count++; |
a3d5953d | 468 | debug(20, 3) ("storeLockObject: key '%s' count=%d\n", |
62e76326 | 469 | e->getMD5Text(), (int) e->lock_count); |
b8de7ebe | 470 | e->lastref = squid_curtime; |
c8f4eac4 | 471 | Store::Root().reference(*e); |
090089c4 | 472 | } |
473 | ||
43ae1d95 | 474 | void |
475 | StoreEntry::setReleaseFlag() | |
476 | { | |
477 | if (EBIT_TEST(flags, RELEASE_REQUEST)) | |
478 | return; | |
479 | ||
480 | debug(20, 3) ("StoreEntry::setReleaseFlag: '%s'\n", getMD5Text()); | |
481 | ||
482 | EBIT_SET(flags, RELEASE_REQUEST); | |
483 | } | |
484 | ||
b8d8561b | 485 | void |
486 | storeReleaseRequest(StoreEntry * e) | |
2285407f | 487 | { |
d46a87a8 | 488 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
62e76326 | 489 | return; |
490 | ||
43ae1d95 | 491 | e->setReleaseFlag(); |
62e76326 | 492 | |
f3e570e9 | 493 | /* |
494 | * Clear cachable flag here because we might get called before | |
495 | * anyone else even looks at the cachability flag. Also, this | |
496 | * prevents httpMakePublic from really setting a public key. | |
497 | */ | |
d46a87a8 | 498 | EBIT_CLR(e->flags, ENTRY_CACHABLE); |
62e76326 | 499 | |
fe54d06d | 500 | storeSetPrivateKey(e); |
2285407f | 501 | } |
502 | ||
090089c4 | 503 | /* unlock object, return -1 if object get released after unlock |
504 | * otherwise lock_count */ | |
b8d8561b | 505 | int |
506 | storeUnlockObject(StoreEntry * e) | |
090089c4 | 507 | { |
6c895381 | 508 | e->lock_count--; |
a3d5953d | 509 | debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", |
62e76326 | 510 | e->getMD5Text(), e->lock_count); |
511 | ||
30a4f2a8 | 512 | if (e->lock_count) |
62e76326 | 513 | return (int) e->lock_count; |
514 | ||
b7fe0ab0 | 515 | if (e->store_status == STORE_PENDING) |
43ae1d95 | 516 | e->setReleaseFlag(); |
62e76326 | 517 | |
9e975e4e | 518 | assert(storePendingNClients(e) == 0); |
62e76326 | 519 | |
d46a87a8 | 520 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
62e76326 | 521 | storeRelease(e); |
8350fe9b | 522 | else if (storeKeepInMemory(e)) { |
c8f4eac4 | 523 | Store::Root().dereference(*e); |
62e76326 | 524 | storeSetMemStatus(e, IN_MEMORY); |
525 | e->mem_obj->unlinkRequest(); | |
66f07209 | 526 | } else { |
c8f4eac4 | 527 | Store::Root().dereference(*e); |
62e76326 | 528 | |
529 | if (EBIT_TEST(e->flags, KEY_PRIVATE)) | |
530 | debug(20, 1) ("WARNING: %s:%d: found KEY_PRIVATE\n", __FILE__, __LINE__); | |
42fc2858 | 531 | |
532 | /* storePurgeMem may free e */ | |
533 | storePurgeMem(e); | |
66f07209 | 534 | } |
62e76326 | 535 | |
6c895381 | 536 | return 0; |
090089c4 | 537 | } |
538 | ||
e6ccf245 | 539 | void |
190154cf | 540 | StoreEntry::getPublicByRequestMethod (StoreClient *aClient, HttpRequest * request, const method_t method) |
e6ccf245 | 541 | { |
542 | assert (aClient); | |
3b13a8fd | 543 | StoreEntry *result = storeGetPublicByRequestMethod( request, method); |
62e76326 | 544 | |
e6ccf245 | 545 | if (!result) |
62e76326 | 546 | aClient->created (NullStoreEntry::getInstance()); |
e6ccf245 | 547 | else |
62e76326 | 548 | aClient->created (result); |
e6ccf245 | 549 | } |
550 | ||
551 | void | |
190154cf | 552 | StoreEntry::getPublicByRequest (StoreClient *aClient, HttpRequest * request) |
e6ccf245 | 553 | { |
554 | assert (aClient); | |
3b13a8fd | 555 | StoreEntry *result = storeGetPublicByRequest (request); |
62e76326 | 556 | |
e6ccf245 | 557 | if (!result) |
62e76326 | 558 | result = NullStoreEntry::getInstance(); |
559 | ||
e6ccf245 | 560 | aClient->created (result); |
561 | } | |
562 | ||
563 | void | |
3b13a8fd | 564 | StoreEntry::getPublic (StoreClient *aClient, const char *uri, const method_t method) |
e6ccf245 | 565 | { |
566 | assert (aClient); | |
3b13a8fd | 567 | StoreEntry *result = storeGetPublic (uri, method); |
62e76326 | 568 | |
e6ccf245 | 569 | if (!result) |
62e76326 | 570 | result = NullStoreEntry::getInstance(); |
571 | ||
e6ccf245 | 572 | aClient->created (result); |
573 | } | |
574 | ||
08e5d64f | 575 | StoreEntry * |
576 | storeGetPublic(const char *uri, const method_t method) | |
577 | { | |
c8f4eac4 | 578 | return Store::Root().get(storeKeyPublic(uri, method)); |
08e5d64f | 579 | } |
580 | ||
f66a9ef4 | 581 | StoreEntry * |
190154cf | 582 | storeGetPublicByRequestMethod(HttpRequest * req, const method_t method) |
f66a9ef4 | 583 | { |
c8f4eac4 | 584 | return Store::Root().get(storeKeyPublicByRequestMethod(req, method)); |
f66a9ef4 | 585 | } |
586 | ||
587 | StoreEntry * | |
190154cf | 588 | storeGetPublicByRequest(HttpRequest * req) |
f66a9ef4 | 589 | { |
590 | StoreEntry *e = storeGetPublicByRequestMethod(req, req->method); | |
62e76326 | 591 | |
f66a9ef4 | 592 | if (e == NULL && req->method == METHOD_HEAD) |
62e76326 | 593 | /* We can generate a HEAD reply from a cached GET object */ |
594 | e = storeGetPublicByRequestMethod(req, METHOD_GET); | |
595 | ||
f66a9ef4 | 596 | return e; |
597 | } | |
598 | ||
007b8be4 | 599 | static int |
600 | getKeyCounter(void) | |
04e8dbaa | 601 | { |
007b8be4 | 602 | static int key_counter = 0; |
62e76326 | 603 | |
007b8be4 | 604 | if (++key_counter < 0) |
62e76326 | 605 | key_counter = 1; |
606 | ||
007b8be4 | 607 | return key_counter; |
04e8dbaa | 608 | } |
609 | ||
c8f4eac4 | 610 | /* RBC 20050104 AFAICT this should become simpler: |
611 | * rather than reinserting with a special key it should be marked | |
612 | * as 'released' and then cleaned up when refcounting indicates. | |
613 | * the StoreHashIndex could well implement its 'released' in the | |
614 | * current manner. | |
615 | * Also, clean log writing should skip over ia,t | |
616 | * Otherwise, we need a 'remove from the index but not the store | |
617 | * concept'. | |
618 | */ | |
6c57e268 | 619 | void |
b8d8561b | 620 | storeSetPrivateKey(StoreEntry * e) |
227fbb74 | 621 | { |
9fb13bb6 | 622 | const cache_key *newkey; |
623 | MemObject *mem = e->mem_obj; | |
62e76326 | 624 | |
332dafa2 | 625 | if (e->key && EBIT_TEST(e->flags, KEY_PRIVATE)) |
62e76326 | 626 | return; /* is already private */ |
627 | ||
332dafa2 | 628 | if (e->key) { |
62e76326 | 629 | if (e->swap_filen > -1) |
630 | storeDirSwapLog(e, SWAP_LOG_DEL); | |
631 | ||
632 | storeHashDelete(e); | |
b109de6b | 633 | } |
62e76326 | 634 | |
9fb13bb6 | 635 | if (mem != NULL) { |
62e76326 | 636 | mem->id = getKeyCounter(); |
637 | newkey = storeKeyPrivate(mem->url, mem->method, mem->id); | |
9fb13bb6 | 638 | } else { |
62e76326 | 639 | newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); |
9fb13bb6 | 640 | } |
62e76326 | 641 | |
9fb13bb6 | 642 | assert(hash_lookup(store_table, newkey) == NULL); |
d46a87a8 | 643 | EBIT_SET(e->flags, KEY_PRIVATE); |
8638fc66 | 644 | storeHashInsert(e, newkey); |
227fbb74 | 645 | } |
646 | ||
b8d8561b | 647 | void |
648 | storeSetPublicKey(StoreEntry * e) | |
227fbb74 | 649 | { |
6eb42cae | 650 | StoreEntry *e2 = NULL; |
9fb13bb6 | 651 | const cache_key *newkey; |
652 | MemObject *mem = e->mem_obj; | |
62e76326 | 653 | |
332dafa2 | 654 | if (e->key && !EBIT_TEST(e->flags, KEY_PRIVATE)) |
62e76326 | 655 | return; /* is already public */ |
656 | ||
9fb13bb6 | 657 | assert(mem); |
62e76326 | 658 | |
f3e570e9 | 659 | /* |
660 | * We can't make RELEASE_REQUEST objects public. Depending on | |
661 | * when RELEASE_REQUEST gets set, we might not be swapping out | |
662 | * the object. If we're not swapping out, then subsequent | |
663 | * store clients won't be able to access object data which has | |
664 | * been freed from memory. | |
d87ebd78 | 665 | * |
666 | * If RELEASE_REQUEST is set, then ENTRY_CACHABLE should not | |
667 | * be set, and storeSetPublicKey() should not be called. | |
f3e570e9 | 668 | */ |
6a566b9c | 669 | #if MORE_DEBUG_OUTPUT |
62e76326 | 670 | |
2b906e48 | 671 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
62e76326 | 672 | debug(20, 1) ("assertion failed: RELEASE key %s, url %s\n", |
673 | e->key, mem->url); | |
674 | ||
2b906e48 | 675 | #endif |
62e76326 | 676 | |
d46a87a8 | 677 | assert(!EBIT_TEST(e->flags, RELEASE_REQUEST)); |
62e76326 | 678 | |
f66a9ef4 | 679 | if (mem->request) { |
62e76326 | 680 | StoreEntry *pe; |
190154cf | 681 | HttpRequest *request = mem->request; |
62e76326 | 682 | |
683 | if (!mem->vary_headers) { | |
684 | /* First handle the case where the object no longer varies */ | |
685 | safe_free(request->vary_headers); | |
686 | } else { | |
687 | if (request->vary_headers && strcmp(request->vary_headers, mem->vary_headers) != 0) { | |
688 | /* Oops.. the variance has changed. Kill the base object | |
689 | * to record the new variance key | |
690 | */ | |
691 | safe_free(request->vary_headers); /* free old "bad" variance key */ | |
692 | pe = storeGetPublic(mem->url, mem->method); | |
693 | ||
694 | if (pe) | |
695 | storeRelease(pe); | |
696 | } | |
697 | ||
698 | /* Make sure the request knows the variance status */ | |
699 | if (!request->vary_headers) { | |
700 | const char *vary = httpMakeVaryMark(request, mem->getReply()); | |
701 | ||
702 | if (vary) | |
703 | request->vary_headers = xstrdup(vary); | |
704 | } | |
705 | } | |
706 | ||
707 | if (mem->vary_headers && !storeGetPublic(mem->url, mem->method)) { | |
708 | /* Create "vary" base object */ | |
62e76326 | 709 | String vary; |
710 | pe = storeCreateEntry(mem->url, mem->log_url, request->flags, request->method); | |
450e0c10 | 711 | HttpVersion version(1, 0); |
62e76326 | 712 | /* We are allowed to do this typecast */ |
06a5ae20 | 713 | HttpReply *rep = (HttpReply *) pe->getReply(); // bypass const |
714 | rep->setHeaders(version, HTTP_OK, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); | |
62e76326 | 715 | vary = httpHeaderGetList(&mem->getReply()->header, HDR_VARY); |
716 | ||
717 | if (vary.size()) { | |
718 | /* Again, we own this structure layout */ | |
719 | httpHeaderPutStr((HttpHeader *)&pe->getReply()->header, HDR_VARY, vary.buf()); | |
720 | vary.clean(); | |
721 | } | |
722 | ||
f66a9ef4 | 723 | #if X_ACCELERATOR_VARY |
62e76326 | 724 | vary = httpHeaderGetList(&mem->getReply()->header, HDR_X_ACCELERATOR_VARY); |
725 | ||
726 | if (vary.buf()) { | |
727 | /* Again, we own this structure layout */ | |
728 | httpHeaderPutStr((HttpHeader *)&pe->getReply()->header, HDR_X_ACCELERATOR_VARY, vary.buf()); | |
729 | vary.clean(); | |
730 | } | |
731 | ||
f66a9ef4 | 732 | #endif |
62e76326 | 733 | storeSetPublicKey(pe); |
734 | ||
ba27c9a0 | 735 | storeBuffer(pe); |
736 | ||
62e76326 | 737 | /* TODO: remove this when the metadata is separated */ |
738 | { | |
739 | Packer p; | |
740 | packerToStoreInit(&p, pe); | |
06a5ae20 | 741 | pe->getReply()->packHeadersInto(&p); |
62e76326 | 742 | packerClean(&p); |
743 | } | |
744 | ||
745 | storeBufferFlush(pe); | |
746 | storeTimestampsSet(pe); | |
747 | pe->complete(); | |
748 | storeUnlockObject(pe); | |
749 | } | |
750 | ||
751 | newkey = storeKeyPublicByRequest(mem->request); | |
f66a9ef4 | 752 | } else |
62e76326 | 753 | newkey = storeKeyPublic(mem->url, mem->method); |
754 | ||
9fb13bb6 | 755 | if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { |
62e76326 | 756 | debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); |
757 | storeSetPrivateKey(e2); | |
758 | storeRelease(e2); | |
759 | ||
760 | if (mem->request) | |
761 | newkey = storeKeyPublicByRequest(mem->request); | |
762 | else | |
763 | newkey = storeKeyPublic(mem->url, mem->method); | |
6eb42cae | 764 | } |
62e76326 | 765 | |
332dafa2 | 766 | if (e->key) |
62e76326 | 767 | storeHashDelete(e); |
768 | ||
d46a87a8 | 769 | EBIT_CLR(e->flags, KEY_PRIVATE); |
62e76326 | 770 | |
8638fc66 | 771 | storeHashInsert(e, newkey); |
62e76326 | 772 | |
cd748f27 | 773 | if (e->swap_filen > -1) |
62e76326 | 774 | storeDirSwapLog(e, SWAP_LOG_ADD); |
227fbb74 | 775 | } |
776 | ||
b8d8561b | 777 | StoreEntry * |
92695e5e | 778 | storeCreateEntry(const char *url, const char *log_url, request_flags flags, method_t method) |
090089c4 | 779 | { |
090089c4 | 780 | StoreEntry *e = NULL; |
30a4f2a8 | 781 | MemObject *mem = NULL; |
92695e5e | 782 | debug(20, 3) ("storeCreateEntry: '%s'\n", url); |
090089c4 | 783 | |
c8f4eac4 | 784 | e = new StoreEntry(url, log_url); |
30a4f2a8 | 785 | e->lock_count = 1; /* Note lock here w/o calling storeLock() */ |
786 | mem = e->mem_obj; | |
2ac237e2 | 787 | mem->method = method; |
62e76326 | 788 | |
92695e5e | 789 | if (neighbors_do_private_keys || !flags.hierarchical) |
62e76326 | 790 | storeSetPrivateKey(e); |
86101e40 | 791 | else |
62e76326 | 792 | storeSetPublicKey(e); |
793 | ||
92695e5e | 794 | if (flags.cachable) { |
62e76326 | 795 | EBIT_SET(e->flags, ENTRY_CACHABLE); |
796 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
090089c4 | 797 | } else { |
62e76326 | 798 | /* storeReleaseRequest() clears ENTRY_CACHABLE */ |
799 | storeReleaseRequest(e); | |
090089c4 | 800 | } |
62e76326 | 801 | |
234967c9 | 802 | e->store_status = STORE_PENDING; |
090089c4 | 803 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 804 | e->swap_status = SWAPOUT_NONE; |
cd748f27 | 805 | e->swap_filen = -1; |
806 | e->swap_dirn = -1; | |
090089c4 | 807 | e->refcount = 0; |
b8de7ebe | 808 | e->lastref = squid_curtime; |
d20b1cd0 | 809 | e->timestamp = -1; /* set in storeTimestampsSet() */ |
30a4f2a8 | 810 | e->ping_status = PING_NONE; |
d46a87a8 | 811 | EBIT_SET(e->flags, ENTRY_VALIDATED); |
090089c4 | 812 | return e; |
813 | } | |
814 | ||
6eb42cae | 815 | /* Mark object as expired */ |
b8d8561b | 816 | void |
817 | storeExpireNow(StoreEntry * e) | |
9174e204 | 818 | { |
332dafa2 | 819 | debug(20, 3) ("storeExpireNow: '%s'\n", e->getMD5Text()); |
b8de7ebe | 820 | e->expires = squid_curtime; |
9174e204 | 821 | } |
822 | ||
528b2c61 | 823 | void |
824 | storeWriteComplete (void *data, StoreIOBuffer wroteBuffer) | |
825 | { | |
1d5161bd | 826 | PROF_start(storeWriteComplete); |
528b2c61 | 827 | StoreEntry *e = (StoreEntry *)data; |
62e76326 | 828 | |
1d5161bd | 829 | if (EBIT_TEST(e->flags, DELAY_SENDING)) { |
830 | PROF_stop(storeWriteComplete); | |
62e76326 | 831 | return; |
1d5161bd | 832 | } |
62e76326 | 833 | |
528b2c61 | 834 | InvokeHandlers(e); |
1d5161bd | 835 | PROF_stop(storeWriteComplete); |
528b2c61 | 836 | } |
837 | ||
838 | void | |
839 | StoreEntry::write (StoreIOBuffer writeBuffer) | |
840 | { | |
841 | assert(mem_obj != NULL); | |
842 | assert(writeBuffer.length >= 0); | |
843 | /* This assert will change when we teach the store to update */ | |
844 | assert(store_status == STORE_PENDING); | |
62e76326 | 845 | |
528b2c61 | 846 | if (!writeBuffer.length) |
62e76326 | 847 | return; |
848 | ||
1d5161bd | 849 | PROF_start(StoreEntry_write); |
850 | ||
e4049756 | 851 | debugs(20, 5, "storeWrite: writing " << writeBuffer.length << |
852 | " bytes for '" << getMD5Text() << "'"); | |
62e76326 | 853 | |
528b2c61 | 854 | storeGetMemSpace(writeBuffer.length); |
62e76326 | 855 | |
528b2c61 | 856 | mem_obj->write (writeBuffer, storeWriteComplete, this); |
1d5161bd | 857 | |
858 | PROF_stop(StoreEntry_write); | |
528b2c61 | 859 | } |
860 | ||
090089c4 | 861 | /* Append incoming data from a primary server to an entry. */ |
b8d8561b | 862 | void |
b8014333 | 863 | storeAppend(StoreEntry * e, const char *buf, int len) |
090089c4 | 864 | { |
3a1c3e2f | 865 | MemObject *mem = e->mem_obj; |
866 | assert(mem != NULL); | |
867 | assert(len >= 0); | |
789f4e81 | 868 | assert(e->store_status == STORE_PENDING); |
528b2c61 | 869 | |
870 | StoreIOBuffer tempBuffer; | |
871 | tempBuffer.data = (char *)buf; | |
872 | tempBuffer.length = len; | |
aa18a4ca | 873 | /* |
874 | * XXX sigh, offset might be < 0 here, but it gets "corrected" | |
875 | * later. This offset crap is such a mess. | |
876 | */ | |
528b2c61 | 877 | tempBuffer.offset = mem->endOffset() - (e->getReply() ? e->getReply()->hdr_sz : 0); |
878 | e->write(tempBuffer); | |
090089c4 | 879 | } |
880 | ||
b8d8561b | 881 | void |
62d32805 | 882 | #if STDC_HEADERS |
fe4e214f | 883 | storeAppendPrintf(StoreEntry * e, const char *fmt,...) |
c30c5a73 | 884 | #else |
b8d8561b | 885 | storeAppendPrintf(va_alist) |
62e76326 | 886 | va_dcl |
62d32805 | 887 | #endif |
15c05bb0 | 888 | { |
62d32805 | 889 | #if STDC_HEADERS |
890 | va_list args; | |
891 | va_start(args, fmt); | |
892 | #else | |
62e76326 | 893 | |
15c05bb0 | 894 | va_list args; |
895 | StoreEntry *e = NULL; | |
0ee4272b | 896 | const char *fmt = NULL; |
15c05bb0 | 897 | va_start(args); |
898 | e = va_arg(args, StoreEntry *); | |
899 | fmt = va_arg(args, char *); | |
c30c5a73 | 900 | #endif |
62e76326 | 901 | |
cb69b4c7 | 902 | storeAppendVPrintf(e, fmt, args); |
903 | va_end(args); | |
904 | } | |
905 | ||
906 | /* used be storeAppendPrintf and Packer */ | |
907 | void | |
908 | storeAppendVPrintf(StoreEntry * e, const char *fmt, va_list vargs) | |
909 | { | |
910 | LOCAL_ARRAY(char, buf, 4096); | |
15c05bb0 | 911 | buf[0] = '\0'; |
cb69b4c7 | 912 | vsnprintf(buf, 4096, fmt, vargs); |
15c05bb0 | 913 | storeAppend(e, buf, strlen(buf)); |
c30c5a73 | 914 | } |
915 | ||
62e76326 | 916 | struct _store_check_cachable_hist |
917 | { | |
918 | ||
919 | struct | |
920 | { | |
921 | int non_get; | |
922 | int not_entry_cachable; | |
923 | int wrong_content_length; | |
924 | int negative_cached; | |
925 | int too_big; | |
926 | int too_small; | |
927 | int private_key; | |
928 | int too_many_open_files; | |
929 | int too_many_open_fds; | |
930 | } | |
931 | ||
932 | no; | |
933 | ||
934 | struct | |
935 | { | |
936 | int Default; | |
937 | } | |
938 | ||
939 | yes; | |
940 | } | |
941 | ||
942 | store_check_cachable_hist; | |
8423ff74 | 943 | |
c47511fd | 944 | int |
945 | storeTooManyDiskFilesOpen(void) | |
946 | { | |
947 | if (Config.max_open_disk_fds == 0) | |
62e76326 | 948 | return 0; |
949 | ||
83a29c95 | 950 | if (store_open_disk_fd > Config.max_open_disk_fds) |
62e76326 | 951 | return 1; |
952 | ||
c47511fd | 953 | return 0; |
954 | } | |
955 | ||
d20b1cd0 | 956 | static int |
957 | storeCheckTooSmall(StoreEntry * e) | |
958 | { | |
528b2c61 | 959 | MemObject * const mem = e->mem_obj; |
62e76326 | 960 | |
42b51993 | 961 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) |
62e76326 | 962 | return 0; |
963 | ||
d20b1cd0 | 964 | if (STORE_OK == e->store_status) |
62e76326 | 965 | if (mem->object_sz < 0 || |
966 | static_cast<size_t>(mem->object_sz) | |
967 | < Config.Store.minObjectSize) | |
968 | return 1; | |
969 | if (e->getReply() | |
970 | ->content_length > -1) | |
971 | if (e->getReply() | |
972 | ->content_length < (int) Config.Store.minObjectSize) | |
973 | return 1; | |
d20b1cd0 | 974 | return 0; |
975 | } | |
976 | ||
f09f5b26 | 977 | int |
8350fe9b | 978 | storeCheckCachable(StoreEntry * e) |
6602e70e | 979 | { |
2ac237e2 | 980 | #if CACHE_ALL_METHODS |
62e76326 | 981 | |
2ac237e2 | 982 | if (e->mem_obj->method != METHOD_GET) { |
62e76326 | 983 | debug(20, 2) ("storeCheckCachable: NO: non-GET method\n"); |
984 | store_check_cachable_hist.no.non_get++; | |
2ac237e2 | 985 | } else |
986 | #endif | |
be3b12f0 | 987 | if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) { |
62e76326 | 988 | debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n"); |
989 | store_check_cachable_hist.no.wrong_content_length++; | |
be3b12f0 | 990 | } else if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) { |
991 | debug(20, 2) ("storeCheckCachable: NO: not cachable\n"); | |
992 | store_check_cachable_hist.no.not_entry_cachable++; | |
62e76326 | 993 | } else if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) { |
994 | debug(20, 3) ("storeCheckCachable: NO: negative cached\n"); | |
995 | store_check_cachable_hist.no.negative_cached++; | |
996 | return 0; /* avoid release call below */ | |
997 | } else if ((e->getReply()->content_length > 0 && | |
998 | static_cast<size_t>(e->getReply()->content_length) | |
999 | > Config.Store.maxObjectSize) || | |
1000 | static_cast<size_t>(e->mem_obj->endOffset()) > Config.Store.maxObjectSize) { | |
1001 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); | |
1002 | store_check_cachable_hist.no.too_big++; | |
1003 | } else if (e->getReply()->content_length > (int) Config.Store.maxObjectSize) { | |
1004 | debug(20, 2) | |
1005 | ("storeCheckCachable: NO: too big\n"); | |
1006 | store_check_cachable_hist.no.too_big++; | |
1007 | } else if (storeCheckTooSmall(e)) { | |
1008 | debug(20, 2) | |
1009 | ("storeCheckCachable: NO: too small\n"); | |
1010 | store_check_cachable_hist.no.too_small++; | |
1011 | } else if (EBIT_TEST(e->flags, KEY_PRIVATE)) { | |
1012 | debug(20, 3) | |
1013 | ("storeCheckCachable: NO: private key\n"); | |
1014 | store_check_cachable_hist.no.private_key++; | |
1015 | } else if (e->swap_status != SWAPOUT_NONE) { | |
1016 | /* | |
1017 | * here we checked the swap_status because the remaining | |
1018 | * cases are only relevant only if we haven't started swapping | |
1019 | * out the object yet. | |
1020 | */ | |
1021 | return 1; | |
1022 | } else if (storeTooManyDiskFilesOpen()) { | |
1023 | debug(20, 2) | |
1024 | ("storeCheckCachable: NO: too many disk files open\n"); | |
1025 | store_check_cachable_hist.no.too_many_open_files++; | |
1026 | } else if (fdNFree() < RESERVED_FD) { | |
1027 | debug(20, 2) | |
1028 | ("storeCheckCachable: NO: too many FD's open\n"); | |
1029 | store_check_cachable_hist.no.too_many_open_fds++; | |
1030 | } else { | |
1031 | store_check_cachable_hist.yes.Default++; | |
1032 | return 1; | |
1033 | } | |
1034 | ||
2daae136 | 1035 | storeReleaseRequest(e); |
2d5ab5c5 | 1036 | /* storeReleaseRequest() cleared ENTRY_CACHABLE */ |
6602e70e | 1037 | return 0; |
1038 | } | |
1039 | ||
8423ff74 | 1040 | static void |
1041 | storeCheckCachableStats(StoreEntry * sentry) | |
1042 | { | |
c40acff3 | 1043 | storeAppendPrintf(sentry, "Category\t Count\n"); |
1044 | ||
a0cd8f99 | 1045 | #if CACHE_ALL_METHODS |
62e76326 | 1046 | |
8423ff74 | 1047 | storeAppendPrintf(sentry, "no.non_get\t%d\n", |
62e76326 | 1048 | store_check_cachable_hist.no.non_get); |
a0cd8f99 | 1049 | #endif |
62e76326 | 1050 | |
8423ff74 | 1051 | storeAppendPrintf(sentry, "no.not_entry_cachable\t%d\n", |
62e76326 | 1052 | store_check_cachable_hist.no.not_entry_cachable); |
8423ff74 | 1053 | storeAppendPrintf(sentry, "no.wrong_content_length\t%d\n", |
62e76326 | 1054 | store_check_cachable_hist.no.wrong_content_length); |
8423ff74 | 1055 | storeAppendPrintf(sentry, "no.negative_cached\t%d\n", |
62e76326 | 1056 | store_check_cachable_hist.no.negative_cached); |
8423ff74 | 1057 | storeAppendPrintf(sentry, "no.too_big\t%d\n", |
62e76326 | 1058 | store_check_cachable_hist.no.too_big); |
d20b1cd0 | 1059 | storeAppendPrintf(sentry, "no.too_small\t%d\n", |
62e76326 | 1060 | store_check_cachable_hist.no.too_small); |
8423ff74 | 1061 | storeAppendPrintf(sentry, "no.private_key\t%d\n", |
62e76326 | 1062 | store_check_cachable_hist.no.private_key); |
c5f627c2 | 1063 | storeAppendPrintf(sentry, "no.too_many_open_files\t%d\n", |
62e76326 | 1064 | store_check_cachable_hist.no.too_many_open_files); |
59ffcdf8 | 1065 | storeAppendPrintf(sentry, "no.too_many_open_fds\t%d\n", |
62e76326 | 1066 | store_check_cachable_hist.no.too_many_open_fds); |
8423ff74 | 1067 | storeAppendPrintf(sentry, "yes.default\t%d\n", |
62e76326 | 1068 | store_check_cachable_hist.yes.Default); |
8423ff74 | 1069 | } |
1070 | ||
b8d8561b | 1071 | void |
528b2c61 | 1072 | StoreEntry::complete() |
090089c4 | 1073 | { |
528b2c61 | 1074 | debug(20, 3) ("storeComplete: '%s'\n", getMD5Text()); |
62e76326 | 1075 | |
528b2c61 | 1076 | if (store_status != STORE_PENDING) { |
62e76326 | 1077 | /* |
1078 | * if we're not STORE_PENDING, then probably we got aborted | |
1079 | * and there should be NO clients on this entry | |
1080 | */ | |
1081 | assert(EBIT_TEST(flags, ENTRY_ABORTED)); | |
1082 | assert(mem_obj->nclients == 0); | |
1083 | return; | |
b6403fac | 1084 | } |
62e76326 | 1085 | |
7d31d5fa | 1086 | /* This is suspect: mem obj offsets include the headers. do we adjust for that |
1087 | * in use of object_sz? | |
1088 | */ | |
528b2c61 | 1089 | mem_obj->object_sz = mem_obj->endOffset(); |
7d31d5fa | 1090 | |
528b2c61 | 1091 | store_status = STORE_OK; |
7d31d5fa | 1092 | |
528b2c61 | 1093 | assert(mem_status == NOT_IN_MEMORY); |
62e76326 | 1094 | |
528b2c61 | 1095 | if (!validLength()) { |
62e76326 | 1096 | EBIT_SET(flags, ENTRY_BAD_LENGTH); |
1097 | storeReleaseRequest(this); | |
41587298 | 1098 | } |
62e76326 | 1099 | |
6cfa8966 | 1100 | #if USE_CACHE_DIGESTS |
528b2c61 | 1101 | if (mem_obj->request) |
62e76326 | 1102 | mem_obj->request->hier.store_complete_stop = current_time; |
1103 | ||
39edba21 | 1104 | #endif |
d20b1cd0 | 1105 | /* |
1106 | * We used to call InvokeHandlers, then storeSwapOut. However, | |
1107 | * Madhukar Reddy <myreddy@persistence.com> reported that | |
1108 | * responses without content length would sometimes get released | |
1109 | * in client_side, thinking that the response is incomplete. | |
1110 | */ | |
528b2c61 | 1111 | InvokeHandlers(this); |
7e3e1d01 | 1112 | } |
1113 | ||
090089c4 | 1114 | /* |
474cac1b | 1115 | * Someone wants to abort this transfer. Set the reason in the |
1116 | * request structure, call the server-side callback and mark the | |
2b906e48 | 1117 | * entry for releasing |
090089c4 | 1118 | */ |
b8d8561b | 1119 | void |
7197b20d | 1120 | storeAbort(StoreEntry * e) |
090089c4 | 1121 | { |
528b2c61 | 1122 | statCounter.aborted_requests++; |
3e98df20 | 1123 | MemObject *mem = e->mem_obj; |
8f39b81d | 1124 | assert(e->store_status == STORE_PENDING); |
1125 | assert(mem != NULL); | |
332dafa2 | 1126 | debug(20, 6) ("storeAbort: %s\n", e->getMD5Text()); |
3d02186d | 1127 | storeLockObject(e); /* lock while aborting */ |
79b5cc5f | 1128 | storeNegativeCache(e); |
474cac1b | 1129 | storeReleaseRequest(e); |
b7fe0ab0 | 1130 | EBIT_SET(e->flags, ENTRY_ABORTED); |
8350fe9b | 1131 | storeSetMemStatus(e, NOT_IN_MEMORY); |
789f4e81 | 1132 | e->store_status = STORE_OK; |
496e1d76 | 1133 | /* |
1134 | * We assign an object length here. The only other place we assign | |
1135 | * the object length is in storeComplete() | |
1136 | */ | |
528b2c61 | 1137 | /* RBC: What do we need an object length for? we've just aborted the |
1138 | * request, the request is private and negatively cached. Surely | |
1139 | * the object length is inappropriate to set. | |
1140 | */ | |
1141 | mem->object_sz = mem->endOffset(); | |
474cac1b | 1142 | /* Notify the server side */ |
62e76326 | 1143 | |
7197b20d | 1144 | if (mem->abort.callback) { |
62e76326 | 1145 | eventAdd("mem->abort.callback", |
1146 | mem->abort.callback, | |
1147 | mem->abort.data, | |
1148 | 0.0, | |
1149 | 0); | |
1150 | mem->abort.callback = NULL; | |
1151 | mem->abort.data = NULL; | |
bfcaf585 | 1152 | } |
62e76326 | 1153 | |
1154 | /* XXX Should we reverse these two, so that there is no | |
528b2c61 | 1155 | * unneeded disk swapping triggered? |
1156 | */ | |
474cac1b | 1157 | /* Notify the client side */ |
090089c4 | 1158 | InvokeHandlers(e); |
62e76326 | 1159 | |
cd748f27 | 1160 | /* Close any swapout file */ |
1161 | storeSwapOutFileClose(e); | |
62e76326 | 1162 | |
3d02186d | 1163 | storeUnlockObject(e); /* unlock */ |
090089c4 | 1164 | } |
1165 | ||
090089c4 | 1166 | /* Clear Memory storage to accommodate the given object len */ |
38792624 | 1167 | static void |
d4432957 | 1168 | storeGetMemSpace(int size) |
090089c4 | 1169 | { |
1d5161bd | 1170 | PROF_start(storeGetMemSpace); |
b32508fb | 1171 | StoreEntry *e = NULL; |
20cba4b4 | 1172 | int released = 0; |
b32508fb | 1173 | static time_t last_check = 0; |
528b2c61 | 1174 | size_t pages_needed; |
6a566b9c | 1175 | RemovalPurgeWalker *walker; |
62e76326 | 1176 | |
1d5161bd | 1177 | if (squid_curtime == last_check) { |
1178 | PROF_stop(storeGetMemSpace); | |
62e76326 | 1179 | return; |
1d5161bd | 1180 | } |
62e76326 | 1181 | |
b32508fb | 1182 | last_check = squid_curtime; |
62e76326 | 1183 | |
e954773d | 1184 | pages_needed = (size / SM_PAGE_SIZE) + 1; |
62e76326 | 1185 | |
1d5161bd | 1186 | if (mem_node::InUseCount() + pages_needed < store_pages_max) { |
1187 | PROF_stop(storeGetMemSpace); | |
62e76326 | 1188 | return; |
1d5161bd | 1189 | } |
62e76326 | 1190 | |
e4049756 | 1191 | debugs(20, 2, "storeGetMemSpace: Starting, need " << pages_needed << |
1192 | " pages"); | |
62e76326 | 1193 | |
6a566b9c | 1194 | /* XXX what to set as max_scan here? */ |
1195 | walker = mem_policy->PurgeInit(mem_policy, 100000); | |
62e76326 | 1196 | |
c1dd71ae | 1197 | while ((e = walker->Next(walker))) { |
62e76326 | 1198 | storePurgeMem(e); |
1199 | released++; | |
1200 | ||
1201 | if (mem_node::InUseCount() + pages_needed < store_pages_max) | |
1202 | break; | |
8350fe9b | 1203 | } |
62e76326 | 1204 | |
6a566b9c | 1205 | walker->Done(walker); |
5e3d4fef | 1206 | debug(20, 3) ("storeGetMemSpace stats:\n"); |
59c4d35b | 1207 | debug(20, 3) (" %6d HOT objects\n", hot_obj_count); |
20cba4b4 | 1208 | debug(20, 3) (" %6d were released\n", released); |
1d5161bd | 1209 | PROF_stop(storeGetMemSpace); |
090089c4 | 1210 | } |
1211 | ||
c8f4eac4 | 1212 | |
1213 | /* thunk through to Store::Root().maintain(). Note that this would be better still | |
1214 | * if registered against the root store itself, but that requires more complex | |
1215 | * update logic - bigger fish to fry first. Long term each store when | |
1216 | * it becomes active will self register | |
1217 | */ | |
1218 | void | |
1219 | Store::Maintain(void *notused) | |
1220 | { | |
1221 | Store::Root().maintain(); | |
1222 | ||
1223 | /* Reregister a maintain event .. */ | |
1224 | eventAdd("MaintainSwapSpace", Maintain, NULL, 1.0, 1); | |
1225 | ||
1226 | } | |
1227 | ||
090089c4 | 1228 | /* The maximum objects to scan for maintain storage space */ |
fcefe642 | 1229 | #define MAINTAIN_MAX_SCAN 1024 |
1230 | #define MAINTAIN_MAX_REMOVE 64 | |
090089c4 | 1231 | |
2b906e48 | 1232 | /* |
fcefe642 | 1233 | * This routine is to be called by main loop in main.c. |
1234 | * It removes expired objects on only one bucket for each time called. | |
fcefe642 | 1235 | * |
1236 | * This should get called 1/s from main(). | |
1237 | */ | |
679ac4f0 | 1238 | void |
c8f4eac4 | 1239 | StoreController::maintain() |
090089c4 | 1240 | { |
6a566b9c | 1241 | static time_t last_warn_time = 0; |
cd748f27 | 1242 | |
88bfe092 | 1243 | PROF_start(storeMaintainSwapSpace); |
c8f4eac4 | 1244 | swapDir->maintain(); |
62e76326 | 1245 | |
c8f4eac4 | 1246 | /* this should be emitted by the oversize dir, not globally */ |
62e76326 | 1247 | |
c8f4eac4 | 1248 | if (store_swap_size > Store::Root().maxSize()) { |
62e76326 | 1249 | if (squid_curtime - last_warn_time > 10) { |
c8f4eac4 | 1250 | debugs(20, 0, "WARNING: Disk space over limit: " << store_swap_size << " KB > " |
1251 | << Store::Root().maxSize() << " KB"); | |
62e76326 | 1252 | last_warn_time = squid_curtime; |
1253 | } | |
6a566b9c | 1254 | } |
62e76326 | 1255 | |
88bfe092 | 1256 | PROF_stop(storeMaintainSwapSpace); |
090089c4 | 1257 | } |
1258 | ||
1259 | ||
1260 | /* release an object from a cache */ | |
6c78a099 | 1261 | void |
b8d8561b | 1262 | storeRelease(StoreEntry * e) |
090089c4 | 1263 | { |
88bfe092 | 1264 | PROF_start(storeRelease); |
332dafa2 | 1265 | debug(20, 3) ("storeRelease: Releasing: '%s'\n", e->getMD5Text()); |
090089c4 | 1266 | /* If, for any reason we can't discard this object because of an |
1267 | * outstanding request, mark it for pending release */ | |
62e76326 | 1268 | |
090089c4 | 1269 | if (storeEntryLocked(e)) { |
62e76326 | 1270 | storeExpireNow(e); |
1271 | debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n"); | |
1272 | storeReleaseRequest(e); | |
1273 | PROF_stop(storeRelease); | |
1274 | return; | |
090089c4 | 1275 | } |
62e76326 | 1276 | |
cd748f27 | 1277 | if (store_dirs_rebuilding && e->swap_filen > -1) { |
62e76326 | 1278 | storeSetPrivateKey(e); |
1279 | ||
1280 | if (e->mem_obj) | |
1281 | destroy_MemObject(e); | |
1282 | ||
1283 | if (e->swap_filen > -1) { | |
1284 | /* | |
1285 | * Fake a call to storeLockObject(). When rebuilding is done, | |
1286 | * we'll just call storeUnlockObject() on these. | |
1287 | */ | |
1288 | e->lock_count++; | |
43ae1d95 | 1289 | e->setReleaseFlag(); |
91caca83 | 1290 | LateReleaseStack.push_back(e); |
62e76326 | 1291 | PROF_stop(storeRelease); |
1292 | return; | |
1293 | } else { | |
c8f4eac4 | 1294 | destroyStoreEntry(static_cast<hash_link *>(e)); |
62e76326 | 1295 | } |
43d9cf56 | 1296 | } |
62e76326 | 1297 | |
a65ff22e | 1298 | storeLog(STORE_LOG_RELEASE, e); |
62e76326 | 1299 | |
cd748f27 | 1300 | if (e->swap_filen > -1) { |
c8f4eac4 | 1301 | e->unlink(); |
62e76326 | 1302 | |
1303 | if (e->swap_status == SWAPOUT_DONE) | |
1304 | if (EBIT_TEST(e->flags, ENTRY_VALIDATED)) | |
c8f4eac4 | 1305 | e->store()->updateSize(e->swap_file_sz, -1); |
62e76326 | 1306 | |
1307 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) | |
1308 | storeDirSwapLog(e, SWAP_LOG_DEL); | |
1309 | ||
cd748f27 | 1310 | #if 0 |
62e76326 | 1311 | /* From 2.4. I think we do this in storeUnlink? */ |
1312 | storeSwapFileNumberSet(e, -1); | |
1313 | ||
cd748f27 | 1314 | #endif |
62e76326 | 1315 | |
090089c4 | 1316 | } |
62e76326 | 1317 | |
8350fe9b | 1318 | storeSetMemStatus(e, NOT_IN_MEMORY); |
c8f4eac4 | 1319 | destroyStoreEntry(static_cast<hash_link *>(e)); |
88bfe092 | 1320 | PROF_stop(storeRelease); |
090089c4 | 1321 | } |
1322 | ||
e42d5181 | 1323 | static void |
1324 | storeLateRelease(void *unused) | |
1325 | { | |
1326 | StoreEntry *e; | |
1327 | int i; | |
1328 | static int n = 0; | |
62e76326 | 1329 | |
b2c141d4 | 1330 | if (store_dirs_rebuilding) { |
62e76326 | 1331 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); |
1332 | return; | |
e42d5181 | 1333 | } |
62e76326 | 1334 | |
e42d5181 | 1335 | for (i = 0; i < 10; i++) { |
91caca83 | 1336 | e = LateReleaseStack.pop(); |
62e76326 | 1337 | |
1338 | if (e == NULL) { | |
1339 | /* done! */ | |
1340 | debug(20, 1) ("storeLateRelease: released %d objects\n", n); | |
1341 | return; | |
1342 | } | |
1343 | ||
1344 | storeUnlockObject(e); | |
1345 | n++; | |
e42d5181 | 1346 | } |
62e76326 | 1347 | |
e42d5181 | 1348 | eventAdd("storeLateRelease", storeLateRelease, NULL, 0.0, 1); |
1349 | } | |
1350 | ||
090089c4 | 1351 | /* return 1 if a store entry is locked */ |
cd748f27 | 1352 | int |
fe4e214f | 1353 | storeEntryLocked(const StoreEntry * e) |
090089c4 | 1354 | { |
30a4f2a8 | 1355 | if (e->lock_count) |
62e76326 | 1356 | return 1; |
1357 | ||
8350fe9b | 1358 | if (e->swap_status == SWAPOUT_WRITING) |
62e76326 | 1359 | return 1; |
1360 | ||
a1e47288 | 1361 | if (e->store_status == STORE_PENDING) |
62e76326 | 1362 | return 1; |
1363 | ||
b8890359 | 1364 | /* |
1365 | * SPECIAL, PUBLIC entries should be "locked" | |
1366 | */ | |
d46a87a8 | 1367 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) |
62e76326 | 1368 | if (!EBIT_TEST(e->flags, KEY_PRIVATE)) |
1369 | return 1; | |
1370 | ||
30a4f2a8 | 1371 | return 0; |
090089c4 | 1372 | } |
1373 | ||
528b2c61 | 1374 | bool |
1375 | StoreEntry::validLength() const | |
6602e70e | 1376 | { |
ffe4a367 | 1377 | int diff; |
d8b249ef | 1378 | const HttpReply *reply; |
528b2c61 | 1379 | assert(mem_obj != NULL); |
1380 | reply = getReply(); | |
1381 | debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", getMD5Text()); | |
e4049756 | 1382 | debugs(20, 5, "storeEntryValidLength: object_len = " << |
1383 | objectLen(this)); | |
07304bf9 | 1384 | debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", |
62e76326 | 1385 | reply->hdr_sz); |
07304bf9 | 1386 | debug(20, 5) ("storeEntryValidLength: content_length = %d\n", |
62e76326 | 1387 | reply->content_length); |
1388 | ||
d8b249ef | 1389 | if (reply->content_length < 0) { |
62e76326 | 1390 | debug(20, 5) ("storeEntryValidLength: Unspecified content length: %s\n", |
1391 | getMD5Text()); | |
1392 | return 1; | |
ffe4a367 | 1393 | } |
62e76326 | 1394 | |
07304bf9 | 1395 | if (reply->hdr_sz == 0) { |
62e76326 | 1396 | debug(20, 5) ("storeEntryValidLength: Zero header size: %s\n", |
1397 | getMD5Text()); | |
1398 | return 1; | |
ffe4a367 | 1399 | } |
62e76326 | 1400 | |
528b2c61 | 1401 | if (mem_obj->method == METHOD_HEAD) { |
62e76326 | 1402 | debug(20, 5) ("storeEntryValidLength: HEAD request: %s\n", |
1403 | getMD5Text()); | |
1404 | return 1; | |
ffe4a367 | 1405 | } |
62e76326 | 1406 | |
cb69b4c7 | 1407 | if (reply->sline.status == HTTP_NOT_MODIFIED) |
62e76326 | 1408 | return 1; |
1409 | ||
cb69b4c7 | 1410 | if (reply->sline.status == HTTP_NO_CONTENT) |
62e76326 | 1411 | return 1; |
1412 | ||
528b2c61 | 1413 | diff = reply->hdr_sz + reply->content_length - objectLen(this); |
62e76326 | 1414 | |
ebf4efff | 1415 | if (diff == 0) |
62e76326 | 1416 | return 1; |
1417 | ||
ebf4efff | 1418 | debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", |
62e76326 | 1419 | diff < 0 ? -diff : diff, |
1420 | diff < 0 ? "big" : "small", | |
1421 | getMD5Text()); | |
1422 | ||
ebf4efff | 1423 | return 0; |
ffe4a367 | 1424 | } |
6602e70e | 1425 | |
b8d8561b | 1426 | void |
1427 | storeInit(void) | |
c943f331 | 1428 | { |
25535cbe | 1429 | storeKeyInit(); |
6a566b9c | 1430 | mem_policy = createRemovalPolicy(Config.memPolicy); |
8638fc66 | 1431 | storeDigestInit(); |
e3ef2b09 | 1432 | storeLogOpen(); |
e42d5181 | 1433 | eventAdd("storeLateRelease", storeLateRelease, NULL, 1.0, 1); |
c8f4eac4 | 1434 | Store::Root().init(); |
b2c141d4 | 1435 | storeRebuildStart(); |
4cb56589 | 1436 | cachemgrRegister("storedir", |
62e76326 | 1437 | "Store Directory Stats", |
c8f4eac4 | 1438 | Store::Stats, 0, 1); |
8423ff74 | 1439 | cachemgrRegister("store_check_cachable_stats", |
62e76326 | 1440 | "storeCheckCachable() Stats", |
1441 | storeCheckCachableStats, 0, 1); | |
65a53c8e | 1442 | cachemgrRegister("store_io", |
62e76326 | 1443 | "Store IO Interface Stats", |
1444 | storeIOStats, 0, 1); | |
b1c0cc67 | 1445 | } |
c943f331 | 1446 | |
b8d8561b | 1447 | void |
1448 | storeConfigure(void) | |
b1c0cc67 | 1449 | { |
c8f4eac4 | 1450 | store_swap_high = (long) (((float) Store::Root().maxSize() * |
62e76326 | 1451 | (float) Config.Swap.highWaterMark) / (float) 100); |
c8f4eac4 | 1452 | store_swap_low = (long) (((float) Store::Root().maxSize() * |
62e76326 | 1453 | (float) Config.Swap.lowWaterMark) / (float) 100); |
43a70238 | 1454 | store_pages_max = Config.memMaxSize / SM_PAGE_SIZE; |
090089c4 | 1455 | } |
1456 | ||
56f29785 | 1457 | static int |
8350fe9b | 1458 | storeKeepInMemory(const StoreEntry * e) |
56f29785 | 1459 | { |
8350fe9b | 1460 | MemObject *mem = e->mem_obj; |
62e76326 | 1461 | |
8350fe9b | 1462 | if (mem == NULL) |
62e76326 | 1463 | return 0; |
1464 | ||
42a503bd | 1465 | if (mem->data_hdr.size() == 0) |
62e76326 | 1466 | return 0; |
1467 | ||
8350fe9b | 1468 | return mem->inmem_lo == 0; |
56f29785 | 1469 | } |
1470 | ||
edce4d98 | 1471 | int |
1472 | storeCheckNegativeHit(StoreEntry * e) | |
1473 | { | |
1474 | if (!EBIT_TEST(e->flags, ENTRY_NEGCACHED)) | |
62e76326 | 1475 | return 0; |
1476 | ||
edce4d98 | 1477 | if (e->expires <= squid_curtime) |
62e76326 | 1478 | return 0; |
1479 | ||
edce4d98 | 1480 | if (e->store_status != STORE_OK) |
62e76326 | 1481 | return 0; |
1482 | ||
edce4d98 | 1483 | return 1; |
1484 | } | |
1485 | ||
b8d8561b | 1486 | void |
1487 | storeNegativeCache(StoreEntry * e) | |
79b5cc5f | 1488 | { |
ff5731b1 | 1489 | e->expires = squid_curtime + Config.negativeTtl; |
d46a87a8 | 1490 | EBIT_SET(e->flags, ENTRY_NEGCACHED); |
79b5cc5f | 1491 | } |
0a21bd84 | 1492 | |
1493 | void | |
1494 | storeFreeMemory(void) | |
1495 | { | |
c8f4eac4 | 1496 | Store::Root(NULL); |
9bc73deb | 1497 | #if USE_CACHE_DIGESTS |
62e76326 | 1498 | |
8638fc66 | 1499 | if (store_digest) |
62e76326 | 1500 | cacheDigestDestroy(store_digest); |
1501 | ||
c68e9c6b | 1502 | #endif |
62e76326 | 1503 | |
8638fc66 | 1504 | store_digest = NULL; |
0a21bd84 | 1505 | } |
a7e59001 | 1506 | |
1507 | int | |
1508 | expiresMoreThan(time_t expires, time_t when) | |
1509 | { | |
48f44632 | 1510 | if (expires < 0) /* No Expires given */ |
62e76326 | 1511 | return 1; |
1512 | ||
48f44632 | 1513 | return (expires > (squid_curtime + when)); |
a7e59001 | 1514 | } |
fe54d06d | 1515 | |
1516 | int | |
1517 | storeEntryValidToSend(StoreEntry * e) | |
1518 | { | |
d46a87a8 | 1519 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) |
62e76326 | 1520 | return 0; |
1521 | ||
d46a87a8 | 1522 | if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) |
62e76326 | 1523 | if (e->expires <= squid_curtime) |
1524 | return 0; | |
1525 | ||
b7fe0ab0 | 1526 | if (EBIT_TEST(e->flags, ENTRY_ABORTED)) |
62e76326 | 1527 | return 0; |
1528 | ||
fe54d06d | 1529 | return 1; |
1530 | } | |
62663274 | 1531 | |
ca98227c | 1532 | void |
1533 | storeTimestampsSet(StoreEntry * entry) | |
1534 | { | |
528b2c61 | 1535 | const HttpReply *reply = entry->getReply(); |
2f58241d | 1536 | time_t served_date = reply->date; |
efd900cb | 1537 | int age = httpHeaderGetInt(&reply->header, HDR_AGE); |
1538 | /* | |
1539 | * The timestamp calculations below tries to mimic the properties | |
1540 | * of the age calculation in RFC2616 section 13.2.3. The implementaion | |
1541 | * isn't complete, and the most notable exception from the RFC is that | |
1542 | * this does not account for response_delay, but it probably does | |
1543 | * not matter much as this is calculated immediately when the headers | |
1544 | * are received, not when the whole response has been received. | |
1545 | */ | |
2f58241d | 1546 | /* make sure that 0 <= served_date <= squid_curtime */ |
62e76326 | 1547 | |
2f58241d | 1548 | if (served_date < 0 || served_date > squid_curtime) |
62e76326 | 1549 | served_date = squid_curtime; |
1550 | ||
efd900cb | 1551 | /* |
212cbb48 | 1552 | * Compensate with Age header if origin server clock is ahead |
1553 | * of us and there is a cache in between us and the origin | |
1554 | * server. But DONT compensate if the age value is larger than | |
1555 | * squid_curtime because it results in a negative served_date. | |
efd900cb | 1556 | */ |
1557 | if (age > squid_curtime - served_date) | |
62e76326 | 1558 | if (squid_curtime > age) |
1559 | served_date = squid_curtime - age; | |
1560 | ||
d8b249ef | 1561 | entry->expires = reply->expires; |
62e76326 | 1562 | |
1c3e77cd | 1563 | entry->lastmod = reply->last_modified; |
62e76326 | 1564 | |
ca98227c | 1565 | entry->timestamp = served_date; |
1566 | } | |
429fdbec | 1567 | |
bfcaf585 | 1568 | void |
1569 | storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) | |
1570 | { | |
1571 | MemObject *mem = e->mem_obj; | |
1572 | assert(mem); | |
1573 | assert(mem->abort.callback == NULL); | |
1574 | mem->abort.callback = cb; | |
1575 | mem->abort.data = data; | |
1576 | } | |
1577 | ||
1578 | void | |
1579 | storeUnregisterAbort(StoreEntry * e) | |
1580 | { | |
1581 | MemObject *mem = e->mem_obj; | |
1582 | assert(mem); | |
1583 | mem->abort.callback = NULL; | |
1584 | } | |
88738790 | 1585 | |
f09f5b26 | 1586 | void |
569c4638 | 1587 | storeEntryDump(const StoreEntry * e, int l) |
d377699f | 1588 | { |
332dafa2 | 1589 | debug(20, l) ("StoreEntry->key: %s\n", e->getMD5Text()); |
1590 | debug(20, l) ("StoreEntry->next: %p\n", e->next); | |
e3ef2b09 | 1591 | debug(20, l) ("StoreEntry->mem_obj: %p\n", e->mem_obj); |
1592 | debug(20, l) ("StoreEntry->timestamp: %d\n", (int) e->timestamp); | |
1593 | debug(20, l) ("StoreEntry->lastref: %d\n", (int) e->lastref); | |
1594 | debug(20, l) ("StoreEntry->expires: %d\n", (int) e->expires); | |
1595 | debug(20, l) ("StoreEntry->lastmod: %d\n", (int) e->lastmod); | |
07304bf9 | 1596 | debug(20, l) ("StoreEntry->swap_file_sz: %d\n", (int) e->swap_file_sz); |
e3ef2b09 | 1597 | debug(20, l) ("StoreEntry->refcount: %d\n", e->refcount); |
415e0dd2 | 1598 | debug(20, l) ("StoreEntry->flags: %s\n", storeEntryFlags(e)); |
cd748f27 | 1599 | debug(20, l) ("StoreEntry->swap_dirn: %d\n", (int) e->swap_dirn); |
1600 | debug(20, l) ("StoreEntry->swap_filen: %d\n", (int) e->swap_filen); | |
e3ef2b09 | 1601 | debug(20, l) ("StoreEntry->lock_count: %d\n", (int) e->lock_count); |
1602 | debug(20, l) ("StoreEntry->mem_status: %d\n", (int) e->mem_status); | |
1603 | debug(20, l) ("StoreEntry->ping_status: %d\n", (int) e->ping_status); | |
1604 | debug(20, l) ("StoreEntry->store_status: %d\n", (int) e->store_status); | |
1605 | debug(20, l) ("StoreEntry->swap_status: %d\n", (int) e->swap_status); | |
d377699f | 1606 | } |
1607 | ||
1f38f50a | 1608 | /* |
1609 | * NOTE, this function assumes only two mem states | |
1610 | */ | |
f09f5b26 | 1611 | void |
e6ccf245 | 1612 | storeSetMemStatus(StoreEntry * e, mem_status_t new_status) |
8350fe9b | 1613 | { |
b93bcace | 1614 | MemObject *mem = e->mem_obj; |
62e76326 | 1615 | |
8350fe9b | 1616 | if (new_status == e->mem_status) |
62e76326 | 1617 | return; |
1618 | ||
b93bcace | 1619 | assert(mem != NULL); |
62e76326 | 1620 | |
b93bcace | 1621 | if (new_status == IN_MEMORY) { |
62e76326 | 1622 | assert(mem->inmem_lo == 0); |
1623 | ||
1624 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { | |
1625 | debug(20, 4) ("storeSetMemStatus: not inserting special %s into policy\n", | |
1626 | mem->url); | |
1627 | } else { | |
1628 | mem_policy->Add(mem_policy, e, &mem->repl); | |
1629 | debug(20, 4) ("storeSetMemStatus: inserted mem node %s\n", | |
1630 | mem->url); | |
1631 | } | |
1632 | ||
1633 | hot_obj_count++; | |
b93bcace | 1634 | } else { |
62e76326 | 1635 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { |
1636 | debug(20, 4) ("storeSetMemStatus: special entry %s\n", | |
1637 | mem->url); | |
1638 | } else { | |
1639 | mem_policy->Remove(mem_policy, e, &mem->repl); | |
1640 | debug(20, 4) ("storeSetMemStatus: removed mem node %s\n", | |
1641 | mem->url); | |
1642 | } | |
1643 | ||
1644 | hot_obj_count--; | |
b93bcace | 1645 | } |
62e76326 | 1646 | |
8350fe9b | 1647 | e->mem_status = new_status; |
1648 | } | |
6e86c3e8 | 1649 | |
9fb13bb6 | 1650 | const char * |
1651 | storeUrl(const StoreEntry * e) | |
1652 | { | |
1653 | if (e == NULL) | |
62e76326 | 1654 | return "[null_entry]"; |
9fb13bb6 | 1655 | else if (e->mem_obj == NULL) |
62e76326 | 1656 | return "[null_mem_obj]"; |
9fb13bb6 | 1657 | else |
62e76326 | 1658 | return e->mem_obj->url; |
9fb13bb6 | 1659 | } |
24ffafb4 | 1660 | |
1661 | void | |
1662 | storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url) | |
1663 | { | |
1664 | if (e->mem_obj) | |
62e76326 | 1665 | return; |
1666 | ||
528b2c61 | 1667 | e->mem_obj = new MemObject(url, log_url); |
24ffafb4 | 1668 | } |
64763c37 | 1669 | |
438fc1e3 | 1670 | /* this just sets DELAY_SENDING */ |
1671 | void | |
8daca701 | 1672 | storeBuffer(StoreEntry * e) |
438fc1e3 | 1673 | { |
d46a87a8 | 1674 | EBIT_SET(e->flags, DELAY_SENDING); |
438fc1e3 | 1675 | } |
1676 | ||
1677 | /* this just clears DELAY_SENDING and Invokes the handlers */ | |
1678 | void | |
8daca701 | 1679 | storeBufferFlush(StoreEntry * e) |
438fc1e3 | 1680 | { |
b66315e4 | 1681 | if (EBIT_TEST(e->flags, DELAY_SENDING)) { |
1682 | EBIT_CLR(e->flags, DELAY_SENDING); | |
1683 | InvokeHandlers(e); | |
1684 | } | |
25535cbe | 1685 | } |
07304bf9 | 1686 | |
e6ccf245 | 1687 | ssize_t |
07304bf9 | 1688 | objectLen(const StoreEntry * e) |
1689 | { | |
1690 | assert(e->mem_obj != NULL); | |
1691 | return e->mem_obj->object_sz; | |
1692 | } | |
1693 | ||
1694 | int | |
1695 | contentLen(const StoreEntry * e) | |
1696 | { | |
1697 | assert(e->mem_obj != NULL); | |
528b2c61 | 1698 | assert(e->getReply() != NULL); |
1699 | return objectLen(e) - e->getReply()->hdr_sz; | |
1700 | ||
07304bf9 | 1701 | } |
f3986a15 | 1702 | |
528b2c61 | 1703 | HttpReply const * |
1704 | StoreEntry::getReply () const | |
f3986a15 | 1705 | { |
528b2c61 | 1706 | if (NULL == mem_obj) |
62e76326 | 1707 | return NULL; |
1708 | ||
528b2c61 | 1709 | return mem_obj->getReply(); |
f3986a15 | 1710 | } |
db1cd23c | 1711 | |
1712 | void | |
1713 | storeEntryReset(StoreEntry * e) | |
1714 | { | |
1715 | MemObject *mem = e->mem_obj; | |
528b2c61 | 1716 | assert (mem); |
db1cd23c | 1717 | debug(20, 3) ("storeEntryReset: %s\n", storeUrl(e)); |
528b2c61 | 1718 | mem->reset(); |
06a5ae20 | 1719 | HttpReply *rep = (HttpReply *) e->getReply(); // bypass const |
1720 | rep->reset(); | |
9bc73deb | 1721 | e->expires = e->lastmod = e->timestamp = -1; |
db1cd23c | 1722 | } |
2b906e48 | 1723 | |
cd748f27 | 1724 | /* |
1725 | * storeFsInit | |
1726 | * | |
1727 | * This routine calls the SETUP routine for each fs type. | |
1728 | * I don't know where the best place for this is, and I'm not going to shuffle | |
1729 | * around large chunks of code right now (that can be done once its working.) | |
1730 | */ | |
1731 | void | |
1732 | storeFsInit(void) | |
1733 | { | |
22d38e05 | 1734 | storeReplSetup(); |
cd748f27 | 1735 | } |
1736 | ||
22d38e05 | 1737 | /* |
1738 | * called to add another store removal policy module | |
1739 | */ | |
1740 | void | |
a2c963ae | 1741 | storeReplAdd(const char *type, REMOVALPOLICYCREATE * create) |
22d38e05 | 1742 | { |
1743 | int i; | |
1744 | /* find the number of currently known repl types */ | |
62e76326 | 1745 | |
22d38e05 | 1746 | for (i = 0; storerepl_list && storerepl_list[i].typestr; i++) { |
62e76326 | 1747 | assert(strcmp(storerepl_list[i].typestr, type) != 0); |
22d38e05 | 1748 | } |
62e76326 | 1749 | |
22d38e05 | 1750 | /* add the new type */ |
e6ccf245 | 1751 | storerepl_list = static_cast<storerepl_entry_t *>(xrealloc(storerepl_list, (i + 2) * sizeof(storerepl_entry_t))); |
62e76326 | 1752 | |
22d38e05 | 1753 | memset(&storerepl_list[i + 1], 0, sizeof(storerepl_entry_t)); |
62e76326 | 1754 | |
22d38e05 | 1755 | storerepl_list[i].typestr = type; |
62e76326 | 1756 | |
22d38e05 | 1757 | storerepl_list[i].create = create; |
1758 | } | |
1759 | ||
1760 | /* | |
1761 | * Create a removal policy instance | |
1762 | */ | |
1763 | RemovalPolicy * | |
1764 | createRemovalPolicy(RemovalPolicySettings * settings) | |
1765 | { | |
1766 | storerepl_entry_t *r; | |
62e76326 | 1767 | |
22d38e05 | 1768 | for (r = storerepl_list; r && r->typestr; r++) { |
62e76326 | 1769 | if (strcmp(r->typestr, settings->type) == 0) |
1770 | return r->create(settings->args); | |
22d38e05 | 1771 | } |
62e76326 | 1772 | |
0c5ccf11 | 1773 | debug(20, 1) ("ERROR: Unknown policy %s\n", settings->type); |
1774 | debug(20, 1) ("ERROR: Be sure to have set cache_replacement_policy\n"); | |
1775 | debug(20, 1) ("ERROR: and memory_replacement_policy in squid.conf!\n"); | |
1776 | fatalf("ERROR: Unknown policy %s\n", settings->type); | |
f0debecb | 1777 | return NULL; /* NOTREACHED */ |
22d38e05 | 1778 | } |
1779 | ||
cd748f27 | 1780 | #if 0 |
fc8b9fc0 | 1781 | void |
1782 | storeSwapFileNumberSet(StoreEntry * e, sfileno filn) | |
1783 | { | |
1784 | if (e->swap_file_number == filn) | |
62e76326 | 1785 | return; |
1786 | ||
fc8b9fc0 | 1787 | if (filn < 0) { |
62e76326 | 1788 | assert(-1 == filn); |
1789 | storeDirMapBitReset(e->swap_file_number); | |
1790 | storeDirLRUDelete(e); | |
1791 | e->swap_file_number = -1; | |
fc8b9fc0 | 1792 | } else { |
62e76326 | 1793 | assert(-1 == e->swap_file_number); |
1794 | storeDirMapBitSet(e->swap_file_number = filn); | |
1795 | storeDirLRUAdd(e); | |
fc8b9fc0 | 1796 | } |
1797 | } | |
62e76326 | 1798 | |
cd748f27 | 1799 | #endif |
e6ccf245 | 1800 | |
4a56ee8d | 1801 | |
db237875 | 1802 | /* |
1803 | * Replace a store entry with | |
528b2c61 | 1804 | * a new reply. This eats the reply. |
1805 | */ | |
4a56ee8d | 1806 | void |
1807 | StoreEntry::replaceHttpReply(HttpReply *rep) | |
1808 | { | |
db237875 | 1809 | debug(20, 3) ("StoreEntry::replaceHttpReply: %s\n", storeUrl(this)); |
528b2c61 | 1810 | Packer p; |
62e76326 | 1811 | |
4a56ee8d | 1812 | if (!mem_obj) { |
62e76326 | 1813 | debug (20,0)("Attempt to replace object with no in-memory representation\n"); |
1814 | return; | |
528b2c61 | 1815 | } |
62e76326 | 1816 | |
4a56ee8d | 1817 | mem_obj->replaceHttpReply(rep); |
1818 | ||
528b2c61 | 1819 | /* TODO: when we store headers serparately remove the header portion */ |
1820 | /* TODO: mark the length of the headers ? */ | |
1821 | /* We ONLY want the headers */ | |
4a56ee8d | 1822 | packerToStoreInit(&p, this); |
62e76326 | 1823 | |
4a56ee8d | 1824 | assert (isEmpty()); |
62e76326 | 1825 | |
4a56ee8d | 1826 | getReply()->packHeadersInto(&p); |
62e76326 | 1827 | |
4a56ee8d | 1828 | rep->hdr_sz = mem_obj->endOffset(); |
62e76326 | 1829 | |
4a56ee8d | 1830 | httpBodyPackInto(&getReply()->body, &p); |
62e76326 | 1831 | |
528b2c61 | 1832 | packerClean(&p); |
1833 | } | |
62e76326 | 1834 | |
e6ccf245 | 1835 | |
528b2c61 | 1836 | char const * |
1837 | StoreEntry::getSerialisedMetaData() | |
1838 | { | |
1839 | StoreMeta *tlv_list = storeSwapMetaBuild(this); | |
1840 | int swap_hdr_sz; | |
1841 | char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz); | |
1842 | storeSwapTLVFree(tlv_list); | |
1843 | assert (swap_hdr_sz >= 0); | |
1844 | mem_obj->swap_hdr_sz = (size_t) swap_hdr_sz; | |
1845 | return result; | |
1846 | } | |
1847 | ||
1848 | bool | |
1849 | StoreEntry::swapoutPossible() | |
1850 | { | |
1851 | /* should we swap something out to disk? */ | |
1852 | debug(20, 7) ("storeSwapOut: %s\n", storeUrl(this)); | |
1853 | debug(20, 7) ("storeSwapOut: store_status = %s\n", | |
62e76326 | 1854 | storeStatusStr[store_status]); |
1855 | ||
528b2c61 | 1856 | if (EBIT_TEST(flags, ENTRY_ABORTED)) { |
62e76326 | 1857 | assert(EBIT_TEST(flags, RELEASE_REQUEST)); |
1858 | storeSwapOutFileClose(this); | |
1859 | return false; | |
528b2c61 | 1860 | } |
62e76326 | 1861 | |
528b2c61 | 1862 | if (EBIT_TEST(flags, ENTRY_SPECIAL)) { |
62e76326 | 1863 | debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(this)); |
1864 | return false; | |
528b2c61 | 1865 | } |
62e76326 | 1866 | |
528b2c61 | 1867 | return true; |
1868 | } | |
1869 | ||
1870 | void | |
1871 | StoreEntry::trimMemory() | |
1872 | { | |
1873 | if (mem_obj->policyLowestOffsetToKeep() == 0) | |
62e76326 | 1874 | /* Nothing to do */ |
1875 | return; | |
1876 | ||
528b2c61 | 1877 | assert (mem_obj->policyLowestOffsetToKeep() > 0); |
62e76326 | 1878 | |
528b2c61 | 1879 | if (!storeSwapOutAble(this)) { |
62e76326 | 1880 | /* |
1881 | * Its not swap-able, and we're about to delete a chunk, | |
1882 | * so we must make it PRIVATE. This is tricky/ugly because | |
1883 | * for the most part, we treat swapable == cachable here. | |
1884 | */ | |
1885 | storeReleaseRequest(this); | |
1886 | mem_obj->trimUnSwappable (); | |
528b2c61 | 1887 | } else { |
62e76326 | 1888 | mem_obj->trimSwappable (); |
528b2c61 | 1889 | } |
1890 | } | |
62e76326 | 1891 | |
0655fa4d | 1892 | bool |
190154cf | 1893 | StoreEntry::modifiedSince(HttpRequest * request) const |
0655fa4d | 1894 | { |
1895 | int object_length; | |
1896 | time_t mod_time = lastmod; | |
1897 | ||
1898 | if (mod_time < 0) | |
1899 | mod_time = timestamp; | |
1900 | ||
1901 | debug(88, 3) ("modifiedSince: '%s'\n", storeUrl(this)); | |
1902 | ||
1903 | debug(88, 3) ("modifiedSince: mod_time = %ld\n", (long int) mod_time); | |
1904 | ||
1905 | if (mod_time < 0) | |
1906 | return true; | |
1907 | ||
1908 | /* Find size of the object */ | |
1909 | object_length = getReply()->content_length; | |
1910 | ||
1911 | if (object_length < 0) | |
1912 | object_length = contentLen(this); | |
1913 | ||
1914 | if (mod_time > request->ims) { | |
1915 | debug(88, 3) ("--> YES: entry newer than client\n"); | |
1916 | return true; | |
1917 | } else if (mod_time < request->ims) { | |
1918 | debug(88, 3) ("--> NO: entry older than client\n"); | |
1919 | return false; | |
1920 | } else if (request->imslen < 0) { | |
1921 | debug(88, 3) ("--> NO: same LMT, no client length\n"); | |
1922 | return false; | |
1923 | } else if (request->imslen == object_length) { | |
1924 | debug(88, 3) ("--> NO: same LMT, same length\n"); | |
1925 | return false; | |
1926 | } else { | |
1927 | debug(88, 3) ("--> YES: same LMT, different length\n"); | |
1928 | return true; | |
1929 | } | |
1930 | } | |
1931 | ||
c8f4eac4 | 1932 | StorePointer |
1933 | StoreEntry::store() const | |
1934 | { | |
1935 | assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured); | |
1936 | return INDEXSD(swap_dirn); | |
1937 | } | |
1938 | ||
1939 | void | |
1940 | StoreEntry::unlink() | |
1941 | { | |
1942 | store()->unlink(*this); | |
1943 | } | |
0655fa4d | 1944 | |
aa18a4ca | 1945 | /* |
1946 | * return true if the entry is in a state where | |
1947 | * it can accept more data (ie with write() method) | |
1948 | */ | |
1949 | bool | |
1950 | StoreEntry::isAccepting() const | |
1951 | { | |
1952 | if (STORE_PENDING != store_status) | |
1953 | return false; | |
1954 | ||
1955 | if (EBIT_TEST(flags, ENTRY_ABORTED)) | |
1956 | return false; | |
1957 | ||
1958 | return true; | |
1959 | } | |
1960 | ||
e6ccf245 | 1961 | /* NullStoreEntry */ |
1962 | ||
1963 | NullStoreEntry NullStoreEntry::_instance; | |
1964 | ||
1965 | NullStoreEntry * | |
1966 | NullStoreEntry::getInstance() | |
1967 | { | |
1968 | return &_instance; | |
1969 | } | |
332dafa2 | 1970 | |
1971 | char const * | |
1972 | NullStoreEntry::getMD5Text() const | |
1973 | { | |
1974 | return "N/A"; | |
1975 | } | |
528b2c61 | 1976 | |
43ae1d95 | 1977 | void |
1978 | NullStoreEntry::operator delete(void*) | |
1979 | { | |
1980 | fatal ("Attempt to delete NullStoreEntry\n"); | |
1981 | } | |
1982 | ||
528b2c61 | 1983 | char const * |
1984 | NullStoreEntry::getSerialisedMetaData() | |
1985 | { | |
1986 | return NULL; | |
1987 | } | |
1988 | ||
1989 | #ifndef _USE_INLINE_ | |
1990 | #include "Store.cci" | |
1991 | #endif |