]>
Commit | Line | Data |
---|---|---|
ff5731b1 | 1 | |
30a4f2a8 | 2 | /* |
79d39a72 | 3 | * $Id: store.cc,v 1.335 1997/11/05 05:29:38 wessels Exp $ |
30a4f2a8 | 4 | * |
5 | * DEBUG: section 20 Storeage Manager | |
6 | * AUTHOR: Harvest Derived | |
7 | * | |
42c04c16 | 8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
30a4f2a8 | 9 | * -------------------------------------------------------- |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
13 | * National Laboratory for Applied Network Research and funded by | |
14 | * the National Science Foundation. | |
15 | * | |
16 | * This program is free software; you can redistribute it and/or modify | |
17 | * it under the terms of the GNU General Public License as published by | |
18 | * the Free Software Foundation; either version 2 of the License, or | |
19 | * (at your option) any later version. | |
20 | * | |
21 | * This program is distributed in the hope that it will be useful, | |
22 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
24 | * GNU General Public License for more details. | |
25 | * | |
26 | * You should have received a copy of the GNU General Public License | |
27 | * along with this program; if not, write to the Free Software | |
28 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | * | |
30 | */ | |
c943f331 | 31 | |
32 | /* | |
30a4f2a8 | 33 | * Copyright (c) 1994, 1995. All rights reserved. |
34 | * | |
35 | * The Harvest software was developed by the Internet Research Task | |
36 | * Force Research Group on Resource Discovery (IRTF-RD): | |
37 | * | |
38 | * Mic Bowman of Transarc Corporation. | |
39 | * Peter Danzig of the University of Southern California. | |
40 | * Darren R. Hardy of the University of Colorado at Boulder. | |
41 | * Udi Manber of the University of Arizona. | |
42 | * Michael F. Schwartz of the University of Colorado at Boulder. | |
43 | * Duane Wessels of the University of Colorado at Boulder. | |
44 | * | |
45 | * This copyright notice applies to software in the Harvest | |
46 | * ``src/'' directory only. Users should consult the individual | |
47 | * copyright notices in the ``components/'' subdirectories for | |
48 | * copyright information about other software bundled with the | |
49 | * Harvest source code distribution. | |
50 | * | |
51 | * TERMS OF USE | |
52 | * | |
53 | * The Harvest software may be used and re-distributed without | |
54 | * charge, provided that the software origin and research team are | |
55 | * cited in any use of the system. Most commonly this is | |
56 | * accomplished by including a link to the Harvest Home Page | |
57 | * (http://harvest.cs.colorado.edu/) from the query page of any | |
58 | * Broker you deploy, as well as in the query result pages. These | |
59 | * links are generated automatically by the standard Broker | |
60 | * software distribution. | |
61 | * | |
62 | * The Harvest software is provided ``as is'', without express or | |
63 | * implied warranty, and with no support nor obligation to assist | |
64 | * in its use, correction, modification or enhancement. We assume | |
65 | * no liability with respect to the infringement of copyrights, | |
66 | * trade secrets, or any patents, and are not responsible for | |
67 | * consequential damages. Proper use of the Harvest software is | |
68 | * entirely the responsibility of the user. | |
69 | * | |
70 | * DERIVATIVE WORKS | |
71 | * | |
72 | * Users may make derivative works from the Harvest software, subject | |
73 | * to the following constraints: | |
74 | * | |
75 | * - You must include the above copyright notice and these | |
76 | * accompanying paragraphs in all forms of derivative works, | |
77 | * and any documentation and other materials related to such | |
78 | * distribution and use acknowledge that the software was | |
79 | * developed at the above institutions. | |
80 | * | |
81 | * - You must notify IRTF-RD regarding your distribution of | |
82 | * the derivative work. | |
83 | * | |
84 | * - You must clearly notify users that your are distributing | |
85 | * a modified version and not the original Harvest software. | |
86 | * | |
87 | * - Any derivative product is also subject to these copyright | |
88 | * and use restrictions. | |
89 | * | |
90 | * Note that the Harvest software is NOT in the public domain. We | |
91 | * retain copyright, as specified above. | |
92 | * | |
93 | * HISTORY OF FREE SOFTWARE STATUS | |
94 | * | |
95 | * Originally we required sites to license the software in cases | |
96 | * where they were going to build commercial products/services | |
97 | * around Harvest. In June 1995 we changed this policy. We now | |
98 | * allow people to use the core Harvest software (the code found in | |
99 | * the Harvest ``src/'' directory) for free. We made this change | |
100 | * in the interest of encouraging the widest possible deployment of | |
101 | * the technology. The Harvest software is really a reference | |
102 | * implementation of a set of protocols and formats, some of which | |
103 | * we intend to standardize. We encourage commercial | |
104 | * re-implementations of code complying to this set of standards. | |
c943f331 | 105 | */ |
090089c4 | 106 | |
44a47c6e | 107 | #include "squid.h" /* goes first */ |
090089c4 | 108 | |
109 | #define REBUILD_TIMESTAMP_DELTA_MAX 2 | |
090089c4 | 110 | #define SWAP_BUF DISK_PAGE_SIZE |
8350fe9b | 111 | #define VM_WINDOW_SZ DISK_PAGE_SIZE |
090089c4 | 112 | |
227fbb74 | 113 | #define WITH_MEMOBJ 1 |
114 | #define WITHOUT_MEMOBJ 0 | |
115 | ||
fcf5283d | 116 | #define STORE_IN_MEM_BUCKETS (229) |
090089c4 | 117 | |
2285407f | 118 | #define STORE_LOG_CREATE 0 |
119 | #define STORE_LOG_SWAPIN 1 | |
120 | #define STORE_LOG_SWAPOUT 2 | |
121 | #define STORE_LOG_RELEASE 3 | |
122 | ||
147d3115 | 123 | static char *storeLogTags[] = |
124 | { | |
125 | "CREATE", | |
126 | "SWAPIN", | |
127 | "SWAPOUT", | |
128 | "RELEASE" | |
2285407f | 129 | }; |
130 | ||
0ee4272b | 131 | const char *memStatusStr[] = |
e62d2dea | 132 | { |
9dfb6c1c | 133 | "NOT_IN_MEMORY", |
9dfb6c1c | 134 | "IN_MEMORY" |
135 | }; | |
136 | ||
0ee4272b | 137 | const char *pingStatusStr[] = |
e62d2dea | 138 | { |
f17936ab | 139 | "PING_NONE", |
9dfb6c1c | 140 | "PING_WAITING", |
141 | "PING_TIMEOUT", | |
f17936ab | 142 | "PING_DONE" |
9dfb6c1c | 143 | }; |
144 | ||
0ee4272b | 145 | const char *storeStatusStr[] = |
e62d2dea | 146 | { |
9dfb6c1c | 147 | "STORE_OK", |
148 | "STORE_PENDING", | |
149 | "STORE_ABORTED" | |
150 | }; | |
151 | ||
0ee4272b | 152 | const char *swapStatusStr[] = |
e62d2dea | 153 | { |
8350fe9b | 154 | "SWAPOUT_NONE", |
155 | "SWAPOUT_OPENING", | |
156 | "SWAPOUT_WRITING", | |
157 | "SWAPOUT_DONE" | |
9dfb6c1c | 158 | }; |
159 | ||
5608850b | 160 | struct storeRebuildState { |
161 | struct _rebuild_dir { | |
162 | int dirn; | |
3e347513 | 163 | FILE *log; |
5608850b | 164 | int speed; |
165 | int clean; | |
166 | struct _rebuild_dir *next; | |
3e347513 | 167 | } *rebuild_dir; |
3facc31e | 168 | int objcount; /* # objects successfully reloaded */ |
169 | int expcount; /* # objects expired */ | |
170 | int linecount; /* # lines parsed from cache logfile */ | |
171 | int clashcount; /* # swapfile clashes avoided */ | |
172 | int dupcount; /* # duplicates purged */ | |
994bbf93 | 173 | int invalid; /* # bad lines */ |
a47b9029 | 174 | int badflags; /* # bad e->flags */ |
f182d1c5 | 175 | int need_to_validate; |
5608850b | 176 | time_t start; |
177 | time_t stop; | |
178 | char *line_in; | |
179 | size_t line_in_sz; | |
3facc31e | 180 | }; |
181 | ||
0a0bf5db | 182 | typedef struct storeCleanList { |
9fb13bb6 | 183 | const cache_key *key; |
0a0bf5db | 184 | struct storeCleanList *next; |
185 | } storeCleanList; | |
186 | ||
ea3a2a69 | 187 | typedef void (VCB) (void *); |
0a0bf5db | 188 | |
189 | typedef struct valid_ctrl_t { | |
190 | struct stat *sb; | |
191 | StoreEntry *e; | |
d7b78fbb | 192 | VCB *callback; |
0a0bf5db | 193 | void *callback_data; |
194 | } valid_ctrl_t; | |
195 | ||
196 | typedef struct swapin_ctrl_t { | |
197 | StoreEntry *e; | |
198 | char *path; | |
582b6456 | 199 | SIH *callback; |
0a0bf5db | 200 | void *callback_data; |
43329771 | 201 | store_client *sc; |
0a0bf5db | 202 | } swapin_ctrl_t; |
203 | ||
204 | typedef struct lock_ctrl_t { | |
582b6456 | 205 | SIH *callback; |
0a0bf5db | 206 | void *callback_data; |
207 | StoreEntry *e; | |
208 | } lock_ctrl_t; | |
209 | ||
210 | typedef struct swapout_ctrl_t { | |
211 | char *swapfilename; | |
212 | int oldswapstatus; | |
213 | StoreEntry *e; | |
0a0bf5db | 214 | } swapout_ctrl_t; |
215 | ||
30a4f2a8 | 216 | /* Static Functions */ |
f5b8bbc4 | 217 | static int storeCheckExpired(const StoreEntry *, int flag); |
218 | static store_client *storeClientListSearch(const MemObject *, void *); | |
f5b8bbc4 | 219 | static int storeEntryLocked(const StoreEntry *); |
220 | static int storeEntryValidLength(const StoreEntry *); | |
221 | static void storeGetMemSpace(int); | |
b93bcace | 222 | static void storeHashDelete(StoreEntry *); |
d7b78fbb | 223 | static VCB storeSwapInValidateComplete; |
f5b8bbc4 | 224 | static mem_hdr *new_MemObjectData(void); |
9fb13bb6 | 225 | static MemObject *new_MemObject(const char *, const char *); |
226 | static StoreEntry *new_StoreEntry(int, const char *, const char *); | |
227 | static StoreEntry *storeAddDiskRestore(const cache_key *, | |
ea3a2a69 | 228 | int, |
229 | int, | |
230 | time_t, | |
231 | time_t, | |
232 | time_t, | |
233 | time_t, | |
234 | u_num32, | |
235 | u_num32, | |
236 | int); | |
f5b8bbc4 | 237 | static void destroy_MemObject(MemObject *); |
238 | static void destroy_MemObjectData(MemObject *); | |
239 | static void destroy_StoreEntry(StoreEntry *); | |
240 | static void storePurgeMem(StoreEntry *); | |
241 | static void storeStartRebuildFromDisk(void); | |
242 | static void storeSwapOutStart(StoreEntry * e); | |
d89d1fb6 | 243 | static DWCB storeSwapOutHandle; |
f5b8bbc4 | 244 | static void storeSetPrivateKey(StoreEntry *); |
d7b78fbb | 245 | static EVH storeDoRebuildFromDisk; |
246 | static EVH storeCleanup; | |
247 | static VCB storeCleanupComplete; | |
f5b8bbc4 | 248 | static void storeValidate(StoreEntry *, VCB *, void *); |
d7b78fbb | 249 | static AIOCB storeValidateComplete; |
ea3a2a69 | 250 | static void storeRebuiltFromDisk(struct storeRebuildState *data); |
f5b8bbc4 | 251 | static unsigned int getKeyCounter(void); |
252 | static void storePutUnusedFileno(int fileno); | |
253 | static int storeGetUnusedFileno(void); | |
8350fe9b | 254 | static void storeCheckSwapOut(StoreEntry * e); |
255 | static void storeSwapoutFileOpened(void *data, int fd); | |
256 | static int storeCheckCachable(StoreEntry * e); | |
257 | static int storeKeepInMemory(const StoreEntry *); | |
258 | static SIH storeClientCopyFileOpened; | |
259 | static DRCB storeClientCopyHandleRead; | |
260 | static FOCB storeSwapInFileOpened; | |
5d86029a | 261 | static void storeClientCopyFileRead(store_client * sc); |
b93bcace | 262 | static void storeEntryListAdd(StoreEntry * e, dlink_node *, dlink_list *); |
263 | static void storeEntryListDelete(dlink_node *, dlink_list *); | |
8350fe9b | 264 | static void storeSetMemStatus(StoreEntry * e, int); |
dbfda5cb | 265 | static void storeClientCopy2(StoreEntry *, store_client *); |
9fb13bb6 | 266 | static void storeHashInsert(StoreEntry * e, const cache_key *); |
6e86c3e8 | 267 | static void storeSwapOutFileClose(StoreEntry * e); |
8350fe9b | 268 | |
090089c4 | 269 | /* Now, this table is inaccessible to outsider. They have to use a method |
270 | * to access a value in internal storage data structure. */ | |
35ac11a3 | 271 | static hash_table *store_table = NULL; |
b93bcace | 272 | static dlink_list inmem_list; |
273 | static dlink_list all_list; | |
090089c4 | 274 | |
e954773d | 275 | static int store_pages_high = 0; |
276 | static int store_pages_low = 0; | |
090089c4 | 277 | |
090089c4 | 278 | /* current file name, swap file, use number as a filename */ |
58104eab | 279 | static int store_swap_high = 0; |
280 | static int store_swap_low = 0; | |
2285407f | 281 | static int storelog_fd = -1; |
090089c4 | 282 | |
66cedb85 | 283 | /* expiration parameters and stats */ |
9fb13bb6 | 284 | static int store_hash_buckets; |
e924600d | 285 | static int store_maintain_rate; |
66cedb85 | 286 | static int store_maintain_buckets; |
66cedb85 | 287 | |
b8d8561b | 288 | static MemObject * |
9fb13bb6 | 289 | new_MemObject(const char *url, const char *log_url) |
227fbb74 | 290 | { |
30a4f2a8 | 291 | MemObject *mem = get_free_mem_obj(); |
292 | mem->reply = xcalloc(1, sizeof(struct _http_reply)); | |
33b589ff | 293 | mem->reply->date = -2; |
294 | mem->reply->expires = -2; | |
295 | mem->reply->last_modified = -2; | |
9fb13bb6 | 296 | mem->url = xstrdup(url); |
88738790 | 297 | mem->log_url = xstrdup(log_url); |
8350fe9b | 298 | mem->swapout.fd = -1; |
579a45cd | 299 | meta_data.mem_obj_count++; |
30a4f2a8 | 300 | meta_data.misc += sizeof(struct _http_reply); |
9fb13bb6 | 301 | meta_data.misc += strlen(log_url); |
a3d5953d | 302 | debug(20, 3) ("new_MemObject: returning %p\n", mem); |
30a4f2a8 | 303 | return mem; |
227fbb74 | 304 | } |
305 | ||
b8d8561b | 306 | static StoreEntry * |
9fb13bb6 | 307 | new_StoreEntry(int mem_obj_flag, const char *url, const char *log_url) |
090089c4 | 308 | { |
309 | StoreEntry *e = NULL; | |
30a4f2a8 | 310 | e = xcalloc(1, sizeof(StoreEntry)); |
227fbb74 | 311 | meta_data.store_entries++; |
312 | if (mem_obj_flag) | |
9fb13bb6 | 313 | e->mem_obj = new_MemObject(url, log_url); |
a3d5953d | 314 | debug(20, 3) ("new_StoreEntry: returning %p\n", e); |
227fbb74 | 315 | return e; |
090089c4 | 316 | } |
317 | ||
b8d8561b | 318 | static void |
319 | destroy_MemObject(MemObject * mem) | |
090089c4 | 320 | { |
a3d5953d | 321 | debug(20, 3) ("destroy_MemObject: destroying %p\n", mem); |
645afd3f | 322 | assert(mem->swapout.fd == -1); |
30a4f2a8 | 323 | destroy_MemObjectData(mem); |
9fb13bb6 | 324 | meta_data.misc -= strlen(mem->log_url); |
52d4522b | 325 | safe_free(mem->clients); |
30a4f2a8 | 326 | safe_free(mem->reply); |
9fb13bb6 | 327 | safe_free(mem->url); |
88738790 | 328 | safe_free(mem->log_url); |
30a4f2a8 | 329 | requestUnlink(mem->request); |
330 | mem->request = NULL; | |
331 | put_free_mem_obj(mem); | |
579a45cd | 332 | meta_data.mem_obj_count--; |
30a4f2a8 | 333 | meta_data.misc -= sizeof(struct _http_reply); |
090089c4 | 334 | } |
335 | ||
b8d8561b | 336 | static void |
337 | destroy_StoreEntry(StoreEntry * e) | |
090089c4 | 338 | { |
a3d5953d | 339 | debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); |
9e975e4e | 340 | assert(e != NULL); |
227fbb74 | 341 | if (e->mem_obj) |
342 | destroy_MemObject(e->mem_obj); | |
9fb13bb6 | 343 | assert(e->key == NULL); |
227fbb74 | 344 | xfree(e); |
345 | meta_data.store_entries--; | |
090089c4 | 346 | } |
347 | ||
f1dc9b30 | 348 | static mem_hdr * |
b8d8561b | 349 | new_MemObjectData(void) |
227fbb74 | 350 | { |
a3d5953d | 351 | debug(20, 3) ("new_MemObjectData: calling memInit()\n"); |
579a45cd | 352 | meta_data.mem_data_count++; |
227fbb74 | 353 | return memInit(); |
354 | } | |
090089c4 | 355 | |
b8d8561b | 356 | static void |
357 | destroy_MemObjectData(MemObject * mem) | |
090089c4 | 358 | { |
a3d5953d | 359 | debug(20, 3) ("destroy_MemObjectData: destroying %p, %d bytes\n", |
8350fe9b | 360 | mem->data, mem->inmem_hi); |
30a4f2a8 | 361 | if (mem->data) { |
d7b78fbb | 362 | memFree(mem->data); |
30a4f2a8 | 363 | mem->data = NULL; |
579a45cd | 364 | meta_data.mem_data_count--; |
090089c4 | 365 | } |
8350fe9b | 366 | mem->inmem_hi = 0; |
090089c4 | 367 | } |
368 | ||
090089c4 | 369 | /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ |
370 | ||
76dd1215 | 371 | static void |
9fb13bb6 | 372 | storeHashInsert(StoreEntry * e, const cache_key * key) |
090089c4 | 373 | { |
a3d5953d | 374 | debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", |
9fb13bb6 | 375 | e, storeKeyText(key)); |
376 | e->key = storeKeyDup(key); | |
b93bcace | 377 | hash_join(store_table, (hash_link *) e); |
378 | storeEntryListAdd(e, &e->lru, &all_list); | |
090089c4 | 379 | } |
380 | ||
b93bcace | 381 | static void |
b8d8561b | 382 | storeHashDelete(StoreEntry * e) |
090089c4 | 383 | { |
b93bcace | 384 | hash_remove_link(store_table, (hash_link *) e); |
385 | storeEntryListDelete(&e->lru, &all_list); | |
9fb13bb6 | 386 | storeKeyFree(e->key); |
387 | e->key = NULL; | |
090089c4 | 388 | } |
389 | ||
090089c4 | 390 | /* -------------------------------------------------------------------------- */ |
391 | ||
b8d8561b | 392 | static void |
fe4e214f | 393 | storeLog(int tag, const StoreEntry * e) |
2285407f | 394 | { |
0be3ec60 | 395 | LOCAL_ARRAY(char, logmsg, MAX_URL << 1); |
ca98227c | 396 | MemObject *mem = e->mem_obj; |
397 | struct _http_reply *reply; | |
2285407f | 398 | if (storelog_fd < 0) |
399 | return; | |
ca98227c | 400 | if (mem == NULL) |
401 | return; | |
88738790 | 402 | if (mem->log_url == NULL) { |
9fb13bb6 | 403 | debug(20, 1) ("storeLog: NULL log_url for %s\n", mem->url); |
88738790 | 404 | storeMemObjectDump(mem); |
9fb13bb6 | 405 | mem->log_url = xstrdup(mem->url); |
88738790 | 406 | } |
ca98227c | 407 | reply = mem->reply; |
8350fe9b | 408 | snprintf(logmsg, MAX_URL << 1, "%9d.%03d %-7s %08X %4d %9d %9d %9d %s %d/%d %s %s\n", |
2285407f | 409 | (int) current_time.tv_sec, |
410 | (int) current_time.tv_usec / 1000, | |
411 | storeLogTags[tag], | |
2abe4822 | 412 | e->swap_file_number, |
ca98227c | 413 | reply->code, |
414 | (int) reply->date, | |
415 | (int) reply->last_modified, | |
416 | (int) reply->expires, | |
417 | reply->content_type[0] ? reply->content_type : "unknown", | |
418 | reply->content_length, | |
711982d8 | 419 | (int) (mem->inmem_hi - mem->reply->hdr_sz), |
ca98227c | 420 | RequestMethodStr[e->method], |
88738790 | 421 | mem->log_url); |
2285407f | 422 | file_write(storelog_fd, |
423 | xstrdup(logmsg), | |
424 | strlen(logmsg), | |
2285407f | 425 | NULL, |
e47afcdb | 426 | NULL, |
427 | xfree); | |
2285407f | 428 | } |
429 | ||
090089c4 | 430 | |
431 | /* get rid of memory copy of the object */ | |
620da955 | 432 | /* Only call this if storeCheckPurgeMem(e) returns 1 */ |
24382924 | 433 | static void |
b8d8561b | 434 | storePurgeMem(StoreEntry * e) |
090089c4 | 435 | { |
2aca8433 | 436 | if (e->mem_obj == NULL) |
090089c4 | 437 | return; |
9fb13bb6 | 438 | debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", |
439 | storeKeyText(e->key)); | |
090089c4 | 440 | storeSetMemStatus(e, NOT_IN_MEMORY); |
227fbb74 | 441 | destroy_MemObject(e->mem_obj); |
442 | e->mem_obj = NULL; | |
06e8899b | 443 | if (e->swap_status != SWAPOUT_DONE) |
8272aded | 444 | storeRelease(e); |
090089c4 | 445 | } |
446 | ||
0a0bf5db | 447 | void |
95c4b18f | 448 | storeLockObject(StoreEntry * e) |
090089c4 | 449 | { |
b93bcace | 450 | if (e->lock_count++ == 0) { |
451 | storeEntryListDelete(&e->lru, &all_list); | |
452 | storeEntryListAdd(e, &e->lru, &all_list); | |
453 | } | |
a3d5953d | 454 | debug(20, 3) ("storeLockObject: key '%s' count=%d\n", |
9fb13bb6 | 455 | storeKeyText(e->key), (int) e->lock_count); |
b8de7ebe | 456 | e->lastref = squid_curtime; |
090089c4 | 457 | } |
458 | ||
b8d8561b | 459 | void |
460 | storeReleaseRequest(StoreEntry * e) | |
2285407f | 461 | { |
aadcfbad | 462 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
58bcd31b | 463 | return; |
9e4ad609 | 464 | if (!storeEntryLocked(e)) |
465 | fatal_dump("storeReleaseRequest: unlocked entry"); | |
9fb13bb6 | 466 | debug(20, 3) ("storeReleaseRequest: '%s'\n", storeKeyText(e->key)); |
9e4ad609 | 467 | BIT_SET(e->flag, RELEASE_REQUEST); |
fe54d06d | 468 | storeSetPrivateKey(e); |
2285407f | 469 | } |
470 | ||
090089c4 | 471 | /* unlock object, return -1 if object get released after unlock |
472 | * otherwise lock_count */ | |
b8d8561b | 473 | int |
474 | storeUnlockObject(StoreEntry * e) | |
090089c4 | 475 | { |
6c895381 | 476 | e->lock_count--; |
a3d5953d | 477 | debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", |
9fb13bb6 | 478 | storeKeyText(e->key), e->lock_count); |
30a4f2a8 | 479 | if (e->lock_count) |
a1e47288 | 480 | return (int) e->lock_count; |
5b00be7a | 481 | if (e->store_status == STORE_PENDING) { |
f182d1c5 | 482 | assert(!BIT_TEST(e->flag, ENTRY_DISPATCHED)); |
483 | BIT_SET(e->flag, RELEASE_REQUEST); | |
5b00be7a | 484 | } |
9e975e4e | 485 | assert(storePendingNClients(e) == 0); |
8350fe9b | 486 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
30a4f2a8 | 487 | storeRelease(e); |
8350fe9b | 488 | else if (storeKeepInMemory(e)) { |
489 | storeSetMemStatus(e, IN_MEMORY); | |
490 | requestUnlink(e->mem_obj->request); | |
491 | e->mem_obj->request = NULL; | |
492 | } else | |
30a4f2a8 | 493 | storePurgeMem(e); |
6c895381 | 494 | return 0; |
090089c4 | 495 | } |
496 | ||
497 | /* Lookup an object in the cache. | |
498 | * return just a reference to object, don't start swapping in yet. */ | |
b8d8561b | 499 | StoreEntry * |
9fb13bb6 | 500 | storeGet(const cache_key * key) |
090089c4 | 501 | { |
9fb13bb6 | 502 | debug(20, 3) ("storeGet: looking up %s\n", storeKeyText(key)); |
503 | return (StoreEntry *) hash_lookup(store_table, key); | |
090089c4 | 504 | } |
505 | ||
88738790 | 506 | static unsigned int |
b8d8561b | 507 | getKeyCounter(void) |
04e8dbaa | 508 | { |
2d25dc1a | 509 | static unsigned int key_counter = 0; |
7cb386d0 | 510 | if (++key_counter == (1 << 24)) |
2cc3f720 | 511 | key_counter = 1; |
2d25dc1a | 512 | return key_counter; |
04e8dbaa | 513 | } |
514 | ||
fe54d06d | 515 | static void |
b8d8561b | 516 | storeSetPrivateKey(StoreEntry * e) |
227fbb74 | 517 | { |
9fb13bb6 | 518 | const cache_key *newkey; |
519 | MemObject *mem = e->mem_obj; | |
6eb42cae | 520 | if (e->key && BIT_TEST(e->flag, KEY_PRIVATE)) |
521 | return; /* is already private */ | |
07622ccd | 522 | if (e->key) |
edae7bc0 | 523 | storeHashDelete(e); |
9fb13bb6 | 524 | if (mem != NULL) { |
525 | mem->reqnum = getKeyCounter(); | |
526 | newkey = storeKeyPrivate(mem->url, e->method, mem->reqnum); | |
527 | } else { | |
528 | newkey = storeKeyPrivate("JUNK", METHOD_NONE, getKeyCounter()); | |
529 | } | |
530 | assert(hash_lookup(store_table, newkey) == NULL); | |
531 | storeHashInsert(e, newkey); | |
6eb42cae | 532 | BIT_SET(e->flag, KEY_PRIVATE); |
227fbb74 | 533 | } |
534 | ||
b8d8561b | 535 | void |
536 | storeSetPublicKey(StoreEntry * e) | |
227fbb74 | 537 | { |
6eb42cae | 538 | StoreEntry *e2 = NULL; |
9fb13bb6 | 539 | const cache_key *newkey; |
540 | MemObject *mem = e->mem_obj; | |
6eb42cae | 541 | if (e->key && !BIT_TEST(e->flag, KEY_PRIVATE)) |
542 | return; /* is already public */ | |
9fb13bb6 | 543 | assert(mem); |
544 | newkey = storeKeyPublic(mem->url, e->method); | |
545 | if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { | |
546 | debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", mem->url); | |
07622ccd | 547 | storeSetPrivateKey(e2); |
30a4f2a8 | 548 | storeRelease(e2); |
9fb13bb6 | 549 | newkey = storeKeyPublic(mem->url, e->method); |
6eb42cae | 550 | } |
07622ccd | 551 | if (e->key) |
edae7bc0 | 552 | storeHashDelete(e); |
9fb13bb6 | 553 | storeHashInsert(e, newkey); |
d3f89c29 | 554 | BIT_CLR(e->flag, KEY_PRIVATE); |
227fbb74 | 555 | } |
556 | ||
b8d8561b | 557 | StoreEntry * |
88738790 | 558 | storeCreateEntry(const char *url, const char *log_url, int flags, method_t method) |
090089c4 | 559 | { |
090089c4 | 560 | StoreEntry *e = NULL; |
30a4f2a8 | 561 | MemObject *mem = NULL; |
a3d5953d | 562 | debug(20, 3) ("storeCreateEntry: '%s' icp flags=%x\n", url, flags); |
090089c4 | 563 | |
9fb13bb6 | 564 | e = new_StoreEntry(WITH_MEMOBJ, url, log_url); |
30a4f2a8 | 565 | e->lock_count = 1; /* Note lock here w/o calling storeLock() */ |
566 | mem = e->mem_obj; | |
73ddce89 | 567 | e->method = method; |
86101e40 | 568 | if (neighbors_do_private_keys || !BIT_TEST(flags, REQ_HIERARCHICAL)) |
569 | storeSetPrivateKey(e); | |
570 | else | |
571 | storeSetPublicKey(e); | |
234967c9 | 572 | if (BIT_TEST(flags, REQ_CACHABLE)) { |
1c481e00 | 573 | BIT_SET(e->flag, ENTRY_CACHABLE); |
d3f89c29 | 574 | BIT_CLR(e->flag, RELEASE_REQUEST); |
090089c4 | 575 | } else { |
d3f89c29 | 576 | BIT_CLR(e->flag, ENTRY_CACHABLE); |
2daae136 | 577 | storeReleaseRequest(e); |
090089c4 | 578 | } |
234967c9 | 579 | if (BIT_TEST(flags, REQ_HIERARCHICAL)) |
580 | BIT_SET(e->flag, HIERARCHICAL); | |
581 | else | |
d3f89c29 | 582 | BIT_CLR(e->flag, HIERARCHICAL); |
234967c9 | 583 | e->store_status = STORE_PENDING; |
090089c4 | 584 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 585 | e->swap_status = SWAPOUT_NONE; |
090089c4 | 586 | e->swap_file_number = -1; |
30a4f2a8 | 587 | mem->data = new_MemObjectData(); |
090089c4 | 588 | e->refcount = 0; |
b8de7ebe | 589 | e->lastref = squid_curtime; |
ca98227c | 590 | e->timestamp = 0; /* set in storeTimestampsSet() */ |
30a4f2a8 | 591 | e->ping_status = PING_NONE; |
0a0bf5db | 592 | BIT_SET(e->flag, ENTRY_VALIDATED); |
090089c4 | 593 | return e; |
594 | } | |
595 | ||
596 | /* Add a new object to the cache with empty memory copy and pointer to disk | |
597 | * use to rebuild store from disk. */ | |
24382924 | 598 | static StoreEntry * |
9fb13bb6 | 599 | storeAddDiskRestore(const cache_key * key, |
23ff6968 | 600 | int file_number, |
601 | int size, | |
602 | time_t expires, | |
603 | time_t timestamp, | |
604 | time_t lastref, | |
605 | time_t lastmod, | |
606 | u_num32 refcount, | |
607 | u_num32 flags, | |
608 | int clean) | |
090089c4 | 609 | { |
610 | StoreEntry *e = NULL; | |
9fb13bb6 | 611 | debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); |
3facc31e | 612 | /* if you call this you'd better be sure file_number is not |
613 | * already in use! */ | |
9fb13bb6 | 614 | e = new_StoreEntry(WITHOUT_MEMOBJ, NULL, NULL); |
73ddce89 | 615 | e->method = METHOD_GET; |
9fb13bb6 | 616 | storeHashInsert(e, key); |
234967c9 | 617 | e->store_status = STORE_OK; |
090089c4 | 618 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 619 | e->swap_status = SWAPOUT_DONE; |
090089c4 | 620 | e->swap_file_number = file_number; |
090089c4 | 621 | e->object_len = size; |
622 | e->lock_count = 0; | |
090089c4 | 623 | e->refcount = 0; |
919877b6 | 624 | e->lastref = lastref; |
f5469bce | 625 | e->timestamp = timestamp; |
626 | e->expires = expires; | |
627 | e->lastmod = lastmod; | |
c54e9052 | 628 | e->refcount = refcount; |
629 | e->flag = flags; | |
89e079ed | 630 | BIT_SET(e->flag, ENTRY_CACHABLE); |
d3f89c29 | 631 | BIT_CLR(e->flag, RELEASE_REQUEST); |
632 | BIT_CLR(e->flag, KEY_PRIVATE); | |
30a4f2a8 | 633 | e->ping_status = PING_NONE; |
311ea387 | 634 | if (clean) { |
3e347513 | 635 | BIT_SET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 636 | /* Only set the file bit if we know its a valid entry */ |
637 | /* otherwise, set it in the validation procedure */ | |
638 | storeDirMapBitSet(file_number); | |
116ca1e0 | 639 | storeDirUpdateSwapSize(e->swap_file_number, size, 1); |
9e4ad609 | 640 | } else { |
d3f89c29 | 641 | BIT_CLR(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 642 | } |
090089c4 | 643 | return e; |
644 | } | |
645 | ||
b8d8561b | 646 | int |
382d851a | 647 | storeUnregister(StoreEntry * e, void *data) |
090089c4 | 648 | { |
6e40f263 | 649 | MemObject *mem = e->mem_obj; |
43329771 | 650 | store_client *sc; |
651 | store_client **S; | |
148fbac1 | 652 | STCB *callback; |
38792624 | 653 | if (mem == NULL) |
654 | return 0; | |
9fb13bb6 | 655 | debug(20, 3) ("storeUnregister: called for '%s'\n", storeKeyText(e->key)); |
79d39a72 | 656 | for (S = &mem->clients; (sc = *S) != NULL; S = &(*S)->next) { |
43329771 | 657 | if (sc->callback_data == data) |
658 | break; | |
659 | } | |
8350fe9b | 660 | if (sc == NULL) |
52d4522b | 661 | return 0; |
43329771 | 662 | *S = sc->next; |
663 | mem->nclients--; | |
645afd3f | 664 | if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) |
6a54c60e | 665 | storeCheckSwapOut(e); |
43329771 | 666 | if (sc->swapin_fd > -1) { |
667 | commSetSelect(sc->swapin_fd, COMM_SELECT_READ, NULL, NULL, 0); | |
8350fe9b | 668 | file_close(sc->swapin_fd); |
43329771 | 669 | } |
79d39a72 | 670 | if ((callback = sc->callback) != NULL) { |
7ffb49ff | 671 | /* callback with ssize = -1 to indicate unexpected termination */ |
9fb13bb6 | 672 | debug(20, 3) ("storeUnregister: store_client for %s has a callback\n", |
673 | mem->url); | |
148fbac1 | 674 | sc->callback = NULL; |
7ffb49ff | 675 | callback(sc->callback_data, sc->copy_buf, -1); |
148fbac1 | 676 | } |
43329771 | 677 | cbdataFree(sc); |
52d4522b | 678 | return 1; |
090089c4 | 679 | } |
680 | ||
8350fe9b | 681 | off_t |
682 | storeLowestMemReaderOffset(const StoreEntry * entry) | |
234967c9 | 683 | { |
0ee4272b | 684 | const MemObject *mem = entry->mem_obj; |
8350fe9b | 685 | off_t lowest = mem->inmem_hi; |
43329771 | 686 | store_client *sc; |
687 | store_client *nx = NULL; | |
5d86029a | 688 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 689 | nx = sc->next; |
8350fe9b | 690 | if (sc->callback_data == NULL) /* open slot */ |
234967c9 | 691 | continue; |
43329771 | 692 | if (sc->type != STORE_MEM_CLIENT) |
8350fe9b | 693 | continue; |
694 | if (sc->copy_offset < lowest) | |
695 | lowest = sc->copy_offset; | |
234967c9 | 696 | } |
697 | return lowest; | |
698 | } | |
699 | ||
090089c4 | 700 | /* Call handlers waiting for data to be appended to E. */ |
498d87ae | 701 | void |
dbfda5cb | 702 | InvokeHandlers(StoreEntry * e) |
090089c4 | 703 | { |
43329771 | 704 | int i = 0; |
9dfb6c1c | 705 | MemObject *mem = e->mem_obj; |
d89d1fb6 | 706 | STCB *callback = NULL; |
43329771 | 707 | store_client *sc; |
708 | store_client *nx = NULL; | |
9e975e4e | 709 | assert(mem->clients != NULL || mem->nclients == 0); |
9fb13bb6 | 710 | debug(20, 3) ("InvokeHandlers: %s\n", storeKeyText(e->key)); |
d89d1fb6 | 711 | /* walk the entire list looking for valid callbacks */ |
5d86029a | 712 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 713 | nx = sc->next; |
714 | debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); | |
382d851a | 715 | if (sc->callback_data == NULL) |
63cc9284 | 716 | continue; |
d89d1fb6 | 717 | if ((callback = sc->callback) == NULL) |
63cc9284 | 718 | continue; |
dbfda5cb | 719 | storeClientCopy2(e, sc); |
090089c4 | 720 | } |
090089c4 | 721 | } |
722 | ||
6eb42cae | 723 | /* Mark object as expired */ |
b8d8561b | 724 | void |
725 | storeExpireNow(StoreEntry * e) | |
9174e204 | 726 | { |
9fb13bb6 | 727 | debug(20, 3) ("storeExpireNow: '%s'\n", storeKeyText(e->key)); |
b8de7ebe | 728 | e->expires = squid_curtime; |
9174e204 | 729 | } |
730 | ||
8350fe9b | 731 | static void |
732 | storeSwapoutFileOpened(void *data, int fd) | |
090089c4 | 733 | { |
8350fe9b | 734 | swapout_ctrl_t *ctrlp = data; |
735 | int oldswapstatus = ctrlp->oldswapstatus; | |
736 | char *swapfilename = ctrlp->swapfilename; | |
737 | StoreEntry *e = ctrlp->e; | |
738 | MemObject *mem; | |
739 | xfree(ctrlp); | |
740 | assert(e->swap_status == SWAPOUT_OPENING); | |
741 | if (fd < 0) { | |
742 | debug(20, 0) ("storeSwapoutFileOpened: Unable to open swapfile: %s\n", | |
743 | swapfilename); | |
744 | storeDirMapBitReset(e->swap_file_number); | |
745 | e->swap_file_number = -1; | |
746 | e->swap_status = oldswapstatus; | |
747 | xfree(swapfilename); | |
090089c4 | 748 | return; |
8350fe9b | 749 | } |
750 | mem = e->mem_obj; | |
751 | mem->swapout.fd = (short) fd; | |
752 | e->swap_status = SWAPOUT_WRITING; | |
753 | debug(20, 5) ("storeSwapoutFileOpened: Begin SwapOut '%s' to FD %d FILE %s.\n", | |
9fb13bb6 | 754 | mem->url, fd, swapfilename); |
8350fe9b | 755 | xfree(swapfilename); |
756 | debug(20, 5) ("swap_file_number=%08X\n", e->swap_file_number); | |
8350fe9b | 757 | storeCheckSwapOut(e); |
758 | } | |
759 | ||
760 | /* start swapping object to disk */ | |
761 | static void | |
762 | storeSwapOutStart(StoreEntry * e) | |
763 | { | |
764 | swapout_ctrl_t *ctrlp; | |
765 | LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); | |
766 | storeLockObject(e); | |
767 | if ((e->swap_file_number = storeGetUnusedFileno()) < 0) | |
768 | e->swap_file_number = storeDirMapAllocate(); | |
769 | storeSwapFullPath(e->swap_file_number, swapfilename); | |
770 | ctrlp = xmalloc(sizeof(swapout_ctrl_t)); | |
771 | ctrlp->swapfilename = xstrdup(swapfilename); | |
772 | ctrlp->e = e; | |
773 | ctrlp->oldswapstatus = e->swap_status; | |
774 | e->swap_status = SWAPOUT_OPENING; | |
775 | file_open(swapfilename, | |
776 | O_WRONLY | O_CREAT | O_TRUNC, | |
777 | storeSwapoutFileOpened, | |
778 | ctrlp); | |
779 | } | |
780 | ||
781 | static void | |
79d39a72 | 782 | storeSwapOutHandle(int fdnotused, int flag, size_t len, void *data) |
8350fe9b | 783 | { |
784 | StoreEntry *e = data; | |
785 | MemObject *mem = e->mem_obj; | |
9fb13bb6 | 786 | debug(20, 3) ("storeSwapOutHandle: '%s', len=%d\n", storeKeyText(e->key), (int) len); |
8350fe9b | 787 | assert(mem != NULL); |
788 | if (flag < 0) { | |
789 | debug(20, 1) ("storeSwapOutHandle: SwapOut failure (err code = %d).\n", | |
790 | flag); | |
791 | e->swap_status = SWAPOUT_NONE; | |
8350fe9b | 792 | if (e->swap_file_number != -1) { |
793 | storePutUnusedFileno(e->swap_file_number); | |
794 | e->swap_file_number = -1; | |
795 | } | |
8350fe9b | 796 | if (flag == DISK_NO_SPACE_LEFT) { |
797 | /* reduce the swap_size limit to the current size. */ | |
798 | Config.Swap.maxSize = store_swap_size; | |
799 | storeConfigure(); | |
800 | } | |
6e86c3e8 | 801 | storeReleaseRequest(e); |
802 | storeSwapOutFileClose(e); | |
8350fe9b | 803 | return; |
804 | } | |
8c123b71 | 805 | mem->swapout.done_offset += len; |
806 | if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len) { | |
8350fe9b | 807 | storeCheckSwapOut(e); |
808 | return; | |
809 | } | |
810 | /* swapping complete */ | |
8350fe9b | 811 | debug(20, 5) ("storeSwapOutHandle: SwapOut complete: '%s' to %s.\n", |
9fb13bb6 | 812 | mem->url, storeSwapFullPath(e->swap_file_number, NULL)); |
20cba4b4 | 813 | e->swap_status = SWAPOUT_DONE; |
814 | storeDirUpdateSwapSize(e->swap_file_number, mem->swapout.done_offset, 1); | |
8350fe9b | 815 | HTTPCacheInfo->proto_newobject(HTTPCacheInfo, |
816 | mem->request->protocol, | |
817 | e->object_len, | |
818 | FALSE); | |
20cba4b4 | 819 | if (storeCheckCachable(e)) { |
820 | storeLog(STORE_LOG_SWAPOUT, e); | |
821 | storeDirSwapLog(e); | |
822 | } | |
823 | /* Note, we don't otherwise call storeReleaseRequest() here because | |
824 | * storeCheckCachable() does it for is if necessary */ | |
6e86c3e8 | 825 | storeSwapOutFileClose(e); |
8350fe9b | 826 | } |
827 | ||
828 | static void | |
829 | storeCheckSwapOut(StoreEntry * e) | |
830 | { | |
831 | MemObject *mem = e->mem_obj; | |
8350fe9b | 832 | off_t lowest_offset; |
8c123b71 | 833 | off_t new_mem_lo; |
834 | size_t swapout_size; | |
c6ac1aae | 835 | char *swap_buf; |
79d39a72 | 836 | ssize_t swap_buf_len; |
dbfda5cb | 837 | int x; |
8350fe9b | 838 | assert(mem != NULL); |
839 | /* should we swap something out to disk? */ | |
9fb13bb6 | 840 | debug(20, 3) ("storeCheckSwapOut: %s\n", mem->url); |
8350fe9b | 841 | debug(20, 3) ("storeCheckSwapOut: store_status = %s\n", |
842 | storeStatusStr[e->store_status]); | |
6e86c3e8 | 843 | if (e->store_status == STORE_ABORTED) { |
844 | assert(BIT_TEST(e->flag, RELEASE_REQUEST)); | |
845 | storeSwapOutFileClose(e); | |
8350fe9b | 846 | return; |
6e86c3e8 | 847 | } |
8c123b71 | 848 | debug(20, 3) ("storeCheckSwapOut: mem->inmem_lo = %d\n", |
849 | (int) mem->inmem_lo); | |
850 | debug(20, 3) ("storeCheckSwapOut: mem->inmem_hi = %d\n", | |
851 | (int) mem->inmem_hi); | |
c6ac1aae | 852 | debug(20, 3) ("storeCheckSwapOut: swapout.queue_offset = %d\n", |
853 | (int) mem->swapout.queue_offset); | |
854 | debug(20, 3) ("storeCheckSwapOut: swapout.done_offset = %d\n", | |
855 | (int) mem->swapout.done_offset); | |
8c123b71 | 856 | assert(mem->inmem_hi >= mem->swapout.queue_offset); |
857 | swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); | |
8350fe9b | 858 | lowest_offset = storeLowestMemReaderOffset(e); |
ea0dd42b | 859 | debug(20, 3) ("storeCheckSwapOut: lowest_offset = %d\n", |
860 | (int) lowest_offset); | |
8c123b71 | 861 | assert(lowest_offset >= mem->inmem_lo); |
862 | ||
863 | new_mem_lo = lowest_offset; | |
864 | if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) { | |
865 | memFreeDataUpto(mem->data, new_mem_lo); | |
866 | mem->inmem_lo = new_mem_lo; | |
8350fe9b | 867 | return; |
8c123b71 | 868 | } |
869 | if (mem->swapout.queue_offset < new_mem_lo) | |
870 | new_mem_lo = mem->swapout.queue_offset; | |
871 | memFreeDataUpto(mem->data, new_mem_lo); | |
872 | mem->inmem_lo = new_mem_lo; | |
873 | ||
874 | swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); | |
875 | debug(20, 3) ("storeCheckSwapOut: swapout_size = %d\n", | |
876 | (int) swapout_size); | |
877 | if (swapout_size == 0) | |
878 | return; | |
879 | if (e->store_status == STORE_PENDING && swapout_size < VM_WINDOW_SZ) | |
8350fe9b | 880 | return; /* wait for a full block */ |
881 | /* Ok, we have stuff to swap out. Is there a swapout.fd open? */ | |
882 | if (e->swap_status == SWAPOUT_NONE) { | |
883 | assert(mem->swapout.fd == -1); | |
8c123b71 | 884 | if (storeCheckCachable(e)) |
6a54c60e | 885 | storeSwapOutStart(e); |
8c123b71 | 886 | /* else ENTRY_CACHABLE will be cleared and we'll never get |
6a54c60e | 887 | * here again */ |
8350fe9b | 888 | return; |
889 | } | |
890 | if (e->swap_status == SWAPOUT_OPENING) | |
891 | return; | |
892 | assert(mem->swapout.fd > -1); | |
8c123b71 | 893 | if (swapout_size > SWAP_BUF) |
894 | swapout_size = SWAP_BUF; | |
c6ac1aae | 895 | swap_buf = get_free_8k_page(); |
dbfda5cb | 896 | swap_buf_len = memCopy(mem->data, |
8c123b71 | 897 | mem->swapout.queue_offset, |
c6ac1aae | 898 | swap_buf, |
dbfda5cb | 899 | swapout_size); |
900 | if (swap_buf_len < 0) { | |
9fb13bb6 | 901 | debug(20, 1) ("memCopy returned %d for '%s'\n", swap_buf_len, storeKeyText(e->key)); |
dbfda5cb | 902 | /* XXX This is probably wrong--we should storeRelease()? */ |
8350fe9b | 903 | storeDirMapBitReset(e->swap_file_number); |
904 | safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); | |
905 | e->swap_file_number = -1; | |
906 | e->swap_status = SWAPOUT_NONE; | |
c6ac1aae | 907 | put_free_8k_page(swap_buf); |
bb5f11c4 | 908 | storeSwapOutFileClose(e); |
8350fe9b | 909 | return; |
910 | } | |
c6ac1aae | 911 | debug(20, 3) ("storeCheckSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); |
912 | assert(swap_buf_len > 0); | |
8350fe9b | 913 | debug(20, 3) ("storeCheckSwapOut: swapping out %d bytes from %d\n", |
c6ac1aae | 914 | swap_buf_len, mem->swapout.queue_offset); |
520452b6 | 915 | mem->swapout.queue_offset += swap_buf_len; |
8350fe9b | 916 | x = file_write(mem->swapout.fd, |
c6ac1aae | 917 | swap_buf, |
918 | swap_buf_len, | |
8350fe9b | 919 | storeSwapOutHandle, |
920 | e, | |
c6ac1aae | 921 | put_free_8k_page); |
8350fe9b | 922 | assert(x == DISK_OK); |
090089c4 | 923 | } |
924 | ||
925 | /* Append incoming data from a primary server to an entry. */ | |
b8d8561b | 926 | void |
b8014333 | 927 | storeAppend(StoreEntry * e, const char *buf, int len) |
090089c4 | 928 | { |
3a1c3e2f | 929 | MemObject *mem = e->mem_obj; |
930 | assert(mem != NULL); | |
931 | assert(len >= 0); | |
090089c4 | 932 | if (len) { |
9fb13bb6 | 933 | debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", len, storeKeyText(e->key)); |
38792624 | 934 | storeGetMemSpace(len); |
3a1c3e2f | 935 | memAppend(mem->data, buf, len); |
8350fe9b | 936 | mem->inmem_hi += len; |
090089c4 | 937 | } |
aadcfbad | 938 | if (e->store_status != STORE_ABORTED && !BIT_TEST(e->flag, DELAY_SENDING)) |
090089c4 | 939 | InvokeHandlers(e); |
8350fe9b | 940 | storeCheckSwapOut(e); |
090089c4 | 941 | } |
942 | ||
24382924 | 943 | #ifdef __STDC__ |
b8d8561b | 944 | void |
fe4e214f | 945 | storeAppendPrintf(StoreEntry * e, const char *fmt,...) |
c30c5a73 | 946 | { |
15c05bb0 | 947 | va_list args; |
95d659f0 | 948 | LOCAL_ARRAY(char, buf, 4096); |
15c05bb0 | 949 | va_start(args, fmt); |
c30c5a73 | 950 | #else |
b8d8561b | 951 | void |
952 | storeAppendPrintf(va_alist) | |
15c05bb0 | 953 | va_dcl |
954 | { | |
955 | va_list args; | |
956 | StoreEntry *e = NULL; | |
0ee4272b | 957 | const char *fmt = NULL; |
95d659f0 | 958 | LOCAL_ARRAY(char, buf, 4096); |
15c05bb0 | 959 | va_start(args); |
960 | e = va_arg(args, StoreEntry *); | |
961 | fmt = va_arg(args, char *); | |
c30c5a73 | 962 | #endif |
15c05bb0 | 963 | buf[0] = '\0'; |
8350fe9b | 964 | vsnprintf(buf, 4096, fmt, args); |
15c05bb0 | 965 | storeAppend(e, buf, strlen(buf)); |
966 | va_end(args); | |
c30c5a73 | 967 | } |
968 | ||
090089c4 | 969 | /* start swapping in */ |
95c4b18f | 970 | void |
582b6456 | 971 | storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) |
090089c4 | 972 | { |
0a0bf5db | 973 | swapin_ctrl_t *ctrlp; |
8350fe9b | 974 | assert(e->mem_status == NOT_IN_MEMORY); |
311ea387 | 975 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
97a88399 | 976 | if (storeDirMapBitTest(e->swap_file_number)) { |
311ea387 | 977 | /* someone took our file while we weren't looking */ |
8350fe9b | 978 | callback(-1, callback_data); |
311ea387 | 979 | return; |
980 | } | |
981 | } | |
bbed1a99 | 982 | assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); |
95c4b18f | 983 | assert(e->swap_file_number >= 0); |
8350fe9b | 984 | assert(e->mem_obj != NULL); |
0a0bf5db | 985 | ctrlp = xmalloc(sizeof(swapin_ctrl_t)); |
986 | ctrlp->e = e; | |
987 | ctrlp->callback = callback; | |
988 | ctrlp->callback_data = callback_data; | |
95c4b18f | 989 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
35ac11a3 | 990 | storeSwapInValidateComplete(ctrlp); |
95c4b18f | 991 | else |
992 | storeValidate(e, storeSwapInValidateComplete, ctrlp); | |
0a0bf5db | 993 | } |
090089c4 | 994 | |
0a0bf5db | 995 | |
996 | static void | |
35ac11a3 | 997 | storeSwapInValidateComplete(void *data) |
0a0bf5db | 998 | { |
999 | swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; | |
0a0bf5db | 1000 | StoreEntry *e; |
1001 | e = ctrlp->e; | |
95c4b18f | 1002 | assert(e->mem_status == NOT_IN_MEMORY); |
0a0bf5db | 1003 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
fc07a0e4 | 1004 | /* Invoke a store abort that should free the memory object */ |
8350fe9b | 1005 | (ctrlp->callback) (-1, ctrlp->callback_data); |
0a0bf5db | 1006 | xfree(ctrlp); |
1007 | return; | |
090089c4 | 1008 | } |
95c4b18f | 1009 | ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); |
8350fe9b | 1010 | file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp); |
0a0bf5db | 1011 | } |
1012 | ||
1013 | static void | |
8350fe9b | 1014 | storeSwapInFileOpened(void *data, int fd) |
0a0bf5db | 1015 | { |
0a0bf5db | 1016 | swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; |
1017 | StoreEntry *e = ctrlp->e; | |
9fb13bb6 | 1018 | MemObject *mem = e->mem_obj; |
1019 | assert(mem != NULL); | |
95c4b18f | 1020 | assert(e->mem_status == NOT_IN_MEMORY); |
be9ce462 | 1021 | assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); |
0a0bf5db | 1022 | if (fd < 0) { |
9fb13bb6 | 1023 | debug(20, 0) ("storeSwapInStartComplete: Failed for '%s'\n", mem->url); |
0a0bf5db | 1024 | /* Invoke a store abort that should free the memory object */ |
8350fe9b | 1025 | (ctrlp->callback) (-1, ctrlp->callback_data); |
0a0bf5db | 1026 | xfree(ctrlp->path); |
1027 | xfree(ctrlp); | |
1028 | return; | |
1029 | } | |
a3d5953d | 1030 | debug(20, 5) ("storeSwapInStart: initialized swap file '%s' for '%s'\n", |
9fb13bb6 | 1031 | ctrlp->path, mem->url); |
8350fe9b | 1032 | (ctrlp->callback) (fd, ctrlp->callback_data); |
0a0bf5db | 1033 | xfree(ctrlp->path); |
1034 | xfree(ctrlp); | |
090089c4 | 1035 | } |
1036 | ||
090089c4 | 1037 | /* recreate meta data from disk image in swap directory */ |
3facc31e | 1038 | /* Add one swap file at a time from disk storage */ |
48f44632 | 1039 | static void |
1040 | storeDoRebuildFromDisk(void *data) | |
3facc31e | 1041 | { |
5608850b | 1042 | struct storeRebuildState *RB = data; |
429fdbec | 1043 | LOCAL_ARRAY(char, swapfile, MAXPATHLEN); |
9fb13bb6 | 1044 | LOCAL_ARRAY(char, keytext, MAX_URL); |
090089c4 | 1045 | StoreEntry *e = NULL; |
090089c4 | 1046 | time_t expires; |
1047 | time_t timestamp; | |
919877b6 | 1048 | time_t lastref; |
f5469bce | 1049 | time_t lastmod; |
1050 | int scan1; | |
1051 | int scan2; | |
1052 | int scan3; | |
1053 | int scan4; | |
c54e9052 | 1054 | int scan5; |
1055 | int scan6; | |
919877b6 | 1056 | int scan7; |
3facc31e | 1057 | off_t size; |
090089c4 | 1058 | int sfileno = 0; |
30a4f2a8 | 1059 | int count; |
f5469bce | 1060 | int x; |
5608850b | 1061 | struct _rebuild_dir *d; |
1062 | struct _rebuild_dir **D; | |
429fdbec | 1063 | int used; /* is swapfile already in use? */ |
1064 | int newer; /* is the log entry newer than current entry? */ | |
9fb13bb6 | 1065 | const cache_key *key; |
429fdbec | 1066 | |
30a4f2a8 | 1067 | /* load a number of objects per invocation */ |
5608850b | 1068 | if ((d = RB->rebuild_dir) == NULL) { |
3e347513 | 1069 | storeRebuiltFromDisk(RB); |
1070 | return; | |
5608850b | 1071 | } |
1072 | for (count = 0; count < d->speed; count++) { | |
1073 | if (fgets(RB->line_in, RB->line_in_sz, d->log) == NULL) { | |
a3d5953d | 1074 | debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); |
3e347513 | 1075 | fclose(d->log); |
1076 | d->log = NULL; | |
1077 | storeDirCloseTmpSwapLog(d->dirn); | |
1078 | RB->rebuild_dir = d->next; | |
1079 | safe_free(d); | |
1080 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); | |
1081 | return; | |
48f44632 | 1082 | } |
5608850b | 1083 | if ((++RB->linecount & 0x3FFF) == 0) |
a3d5953d | 1084 | debug(20, 1) (" %7d Lines read so far.\n", RB->linecount); |
1085 | debug(20, 9) ("line_in: %s", RB->line_in); | |
5608850b | 1086 | if (RB->line_in[0] == '\0') |
1087 | continue; | |
1088 | if (RB->line_in[0] == '\n') | |
1089 | continue; | |
1090 | if (RB->line_in[0] == '#') | |
1091 | continue; | |
9fb13bb6 | 1092 | keytext[0] = '\0'; |
f5469bce | 1093 | sfileno = 0; |
1094 | scan1 = 0; | |
1095 | scan2 = 0; | |
30a4f2a8 | 1096 | scan3 = 0; |
f5469bce | 1097 | scan4 = 0; |
919877b6 | 1098 | scan5 = 0; |
1099 | scan6 = 0; | |
1100 | scan7 = 0; | |
1101 | x = sscanf(RB->line_in, "%x %x %x %x %x %d %d %x %s", | |
0be3ec60 | 1102 | &sfileno, /* swap_file_number */ |
1103 | &scan1, /* timestamp */ | |
919877b6 | 1104 | &scan2, /* lastref */ |
1105 | &scan3, /* expires */ | |
1106 | &scan4, /* last modified */ | |
1107 | &scan5, /* size */ | |
1108 | &scan6, /* refcount */ | |
1109 | &scan7, /* flags */ | |
9fb13bb6 | 1110 | keytext); /* key */ |
994bbf93 | 1111 | if (x < 1) { |
1112 | RB->invalid++; | |
30a4f2a8 | 1113 | continue; |
994bbf93 | 1114 | } |
429fdbec | 1115 | storeSwapFullPath(sfileno, swapfile); |
919877b6 | 1116 | if (x != 9) { |
994bbf93 | 1117 | RB->invalid++; |
429fdbec | 1118 | continue; |
994bbf93 | 1119 | } |
1120 | if (sfileno < 0) { | |
1121 | RB->invalid++; | |
0a0bf5db | 1122 | continue; |
994bbf93 | 1123 | } |
919877b6 | 1124 | if (BIT_TEST(scan7, KEY_PRIVATE)) { |
a47b9029 | 1125 | RB->badflags++; |
1126 | continue; | |
1127 | } | |
c932b107 | 1128 | sfileno = storeDirProperFileno(d->dirn, sfileno); |
f5469bce | 1129 | timestamp = (time_t) scan1; |
919877b6 | 1130 | lastref = (time_t) scan2; |
1131 | expires = (time_t) scan3; | |
1132 | lastmod = (time_t) scan4; | |
1133 | size = (off_t) scan5; | |
30a4f2a8 | 1134 | |
9fb13bb6 | 1135 | key = storeKeyScan(keytext); |
1136 | e = storeGet(key); | |
429fdbec | 1137 | used = storeDirMapBitTest(sfileno); |
1138 | /* If this URL already exists in the cache, does the swap log | |
9042745d | 1139 | * appear to have a newer entry? Compare 'lastref' from the |
1140 | * swap log to e->lastref. */ | |
1141 | newer = e ? (lastref > e->lastref ? 1 : 0) : 0; | |
429fdbec | 1142 | if (used && !newer) { |
1143 | /* log entry is old, ignore it */ | |
1144 | RB->clashcount++; | |
1145 | continue; | |
1146 | } else if (used && e && e->swap_file_number == sfileno) { | |
1147 | /* swapfile taken, same URL, newer, update meta */ | |
1148 | e->lastref = timestamp; | |
1149 | e->timestamp = timestamp; | |
1150 | e->expires = expires; | |
1151 | e->lastmod = lastmod; | |
919877b6 | 1152 | e->flag |= (u_num32) scan6; |
1153 | e->refcount += (u_num32) scan7; | |
429fdbec | 1154 | continue; |
1155 | } else if (used) { | |
1156 | /* swapfile in use, not by this URL, log entry is newer */ | |
1157 | /* This is sorta bad: the log entry should NOT be newer at this | |
1158 | * point. If the log is dirty, the filesize check should have | |
1159 | * caught this. If the log is clean, there should never be a | |
1160 | * newer entry. */ | |
a3d5953d | 1161 | debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", |
30a4f2a8 | 1162 | sfileno); |
429fdbec | 1163 | /* I'm tempted to remove the swapfile here just to be safe, |
1164 | * but there is a bad race condition in the NOVM version if | |
1165 | * the swapfile has recently been opened for writing, but | |
1166 | * not yet opened for reading. Because we can't map | |
1167 | * swapfiles back to StoreEntrys, we don't know the state | |
1168 | * of the entry using that file. */ | |
ccbc48f5 | 1169 | /* We'll assume the existing entry is valid, probably because |
365e5b34 | 1170 | * were in a slow rebuild and the the swap file number got taken |
1171 | * and the validation procedure hasn't run. */ | |
ccbc48f5 | 1172 | assert(RB->need_to_validate); |
5608850b | 1173 | RB->clashcount++; |
30a4f2a8 | 1174 | continue; |
429fdbec | 1175 | } else if (e) { |
1176 | /* URL already exists, this swapfile not being used */ | |
1177 | /* junk old, load new */ | |
1178 | storeRelease(e); /* release old entry */ | |
1179 | RB->dupcount++; | |
1180 | } else { | |
1181 | /* URL doesnt exist, swapfile not in use */ | |
1182 | /* load new */ | |
1183 | (void) 0; | |
30a4f2a8 | 1184 | } |
1185 | /* update store_swap_size */ | |
5608850b | 1186 | RB->objcount++; |
9fb13bb6 | 1187 | e = storeAddDiskRestore(key, |
30a4f2a8 | 1188 | sfileno, |
1189 | (int) size, | |
1190 | expires, | |
f5469bce | 1191 | timestamp, |
919877b6 | 1192 | lastref, |
311ea387 | 1193 | lastmod, |
919877b6 | 1194 | (u_num32) scan6, /* refcount */ |
1195 | (u_num32) scan7, /* flags */ | |
311ea387 | 1196 | d->clean); |
5608850b | 1197 | storeDirSwapLog(e); |
c943f331 | 1198 | } |
5608850b | 1199 | RB->rebuild_dir = d->next; |
76fefb77 | 1200 | for (D = &RB->rebuild_dir; *D; D = &(*D)->next); |
5608850b | 1201 | *D = d; |
1202 | d->next = NULL; | |
1203 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); | |
3facc31e | 1204 | } |
1205 | ||
0a0bf5db | 1206 | static void |
79d39a72 | 1207 | storeCleanup(void *datanotused) |
0a0bf5db | 1208 | { |
1209 | static storeCleanList *list = NULL; | |
1210 | storeCleanList *curr; | |
1211 | static int bucketnum = -1; | |
1212 | static int validnum = 0; | |
1213 | StoreEntry *e; | |
1214 | hash_link *link_ptr = NULL; | |
1215 | if (list == NULL) { | |
9fb13bb6 | 1216 | if (++bucketnum >= store_hash_buckets) { |
a3d5953d | 1217 | debug(20, 1) (" Completed Validation Procedure\n"); |
1218 | debug(20, 1) (" Validated %d Entries\n", validnum); | |
8a31d584 | 1219 | debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); |
f182d1c5 | 1220 | store_rebuilding = 0; |
0a0bf5db | 1221 | return; |
1222 | } | |
1223 | link_ptr = hash_get_bucket(store_table, bucketnum); | |
1224 | for (; link_ptr; link_ptr = link_ptr->next) { | |
1225 | e = (StoreEntry *) link_ptr; | |
35ac11a3 | 1226 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1227 | continue; | |
1228 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) | |
1229 | continue; | |
311ea387 | 1230 | curr = xcalloc(1, sizeof(storeCleanList)); |
9fb13bb6 | 1231 | curr->key = storeKeyDup(e->key); |
0a0bf5db | 1232 | curr->next = list; |
1233 | list = curr; | |
1234 | } | |
1235 | } | |
1236 | if (list == NULL) { | |
1237 | eventAdd("storeCleanup", storeCleanup, NULL, 0); | |
1238 | return; | |
1239 | } | |
1240 | curr = list; | |
1241 | list = list->next; | |
1242 | e = (StoreEntry *) hash_lookup(store_table, curr->key); | |
c340827e | 1243 | if (e && !BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
365e5b34 | 1244 | storeLockObject(e); |
3e347513 | 1245 | storeValidate(e, storeCleanupComplete, e); |
a3d5953d | 1246 | if ((++validnum & 0xFFF) == 0) |
1247 | debug(20, 1) (" %7d Entries Validated so far.\n", validnum); | |
35ac11a3 | 1248 | assert(validnum <= meta_data.store_entries); |
95c4b18f | 1249 | } |
9fb13bb6 | 1250 | storeKeyFree(curr->key); |
0a0bf5db | 1251 | xfree(curr); |
1252 | eventAdd("storeCleanup", storeCleanup, NULL, 0); | |
1253 | } | |
1254 | ||
0a0bf5db | 1255 | static void |
35ac11a3 | 1256 | storeCleanupComplete(void *data) |
0a0bf5db | 1257 | { |
1258 | StoreEntry *e = data; | |
35ac11a3 | 1259 | storeUnlockObject(e); |
0a0bf5db | 1260 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1261 | storeRelease(e); | |
1262 | } | |
1263 | ||
1264 | static void | |
1265 | storeValidate(StoreEntry * e, VCB callback, void *callback_data) | |
1266 | { | |
1267 | valid_ctrl_t *ctrlp; | |
1268 | char *path; | |
1269 | struct stat *sb; | |
606bd3b5 | 1270 | #if !USE_ASYNC_IO |
95c4b18f | 1271 | int x; |
606bd3b5 | 1272 | #endif |
d7b78fbb | 1273 | assert(!BIT_TEST(e->flag, ENTRY_VALIDATED)); |
0a0bf5db | 1274 | if (e->swap_file_number < 0) { |
d3f89c29 | 1275 | BIT_CLR(e->flag, ENTRY_VALIDATED); |
35ac11a3 | 1276 | callback(callback_data); |
0a0bf5db | 1277 | return; |
1278 | } | |
1279 | path = storeSwapFullPath(e->swap_file_number, NULL); | |
1280 | sb = xmalloc(sizeof(struct stat)); | |
1281 | ctrlp = xmalloc(sizeof(valid_ctrl_t)); | |
1282 | ctrlp->sb = sb; | |
1283 | ctrlp->e = e; | |
1284 | ctrlp->callback = callback; | |
1285 | ctrlp->callback_data = callback_data; | |
1286 | #if USE_ASYNC_IO | |
1287 | aioStat(path, sb, storeValidateComplete, ctrlp); | |
1288 | #else | |
95c4b18f | 1289 | /* When evaluating the actual arguments in a function call, the order |
1290 | * in which the arguments and the function expression are evaluated is | |
1291 | * not specified; */ | |
1292 | x = stat(path, sb); | |
1293 | storeValidateComplete(ctrlp, x, errno); | |
0a0bf5db | 1294 | #endif |
1295 | return; | |
1296 | } | |
1297 | ||
1298 | static void | |
1299 | storeValidateComplete(void *data, int retcode, int errcode) | |
1300 | { | |
1301 | valid_ctrl_t *ctrlp = data; | |
1302 | struct stat *sb = ctrlp->sb; | |
1303 | StoreEntry *e = ctrlp->e; | |
1304 | char *path; | |
1305 | if (retcode < 0 && errcode == EWOULDBLOCK) { | |
1306 | path = storeSwapFullPath(e->swap_file_number, NULL); | |
1307 | retcode = stat(path, sb); | |
1308 | } | |
9e4ad609 | 1309 | if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { |
d3f89c29 | 1310 | BIT_CLR(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 1311 | } else { |
0a0bf5db | 1312 | BIT_SET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 1313 | storeDirMapBitSet(e->swap_file_number); |
116ca1e0 | 1314 | storeDirUpdateSwapSize(e->swap_file_number, e->object_len, 1); |
9e4ad609 | 1315 | } |
0a0bf5db | 1316 | errno = errcode; |
35ac11a3 | 1317 | ctrlp->callback(ctrlp->callback_data); |
0a0bf5db | 1318 | xfree(sb); |
1319 | xfree(ctrlp); | |
1320 | } | |
1321 | ||
3facc31e | 1322 | /* meta data recreated from disk image in swap directory */ |
b8d8561b | 1323 | static void |
5608850b | 1324 | storeRebuiltFromDisk(struct storeRebuildState *data) |
3facc31e | 1325 | { |
1326 | time_t r; | |
1327 | time_t stop; | |
97c03d3c | 1328 | stop = squid_curtime; |
3facc31e | 1329 | r = stop - data->start; |
a3d5953d | 1330 | debug(20, 1) ("Finished rebuilding storage from disk image.\n"); |
1331 | debug(20, 1) (" %7d Lines read from previous logfile.\n", data->linecount); | |
994bbf93 | 1332 | debug(20, 1) (" %7d Invalid lines.\n", data->invalid); |
a47b9029 | 1333 | debug(20, 1) (" %7d With invalid flags.\n", data->badflags); |
a3d5953d | 1334 | debug(20, 1) (" %7d Objects loaded.\n", data->objcount); |
1335 | debug(20, 1) (" %7d Objects expired.\n", data->expcount); | |
1336 | debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); | |
1337 | debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); | |
1338 | debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", | |
3facc31e | 1339 | r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); |
76f87348 | 1340 | if (data->need_to_validate && data->linecount) { |
a3d5953d | 1341 | debug(20, 1) ("Beginning Validation Procedure\n"); |
0a0bf5db | 1342 | eventAdd("storeCleanup", storeCleanup, NULL, 0); |
f182d1c5 | 1343 | } else { |
8a31d584 | 1344 | debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); |
a3d5953d | 1345 | store_rebuilding = 0; |
0a0bf5db | 1346 | } |
f182d1c5 | 1347 | safe_free(data->line_in); |
1348 | safe_free(data); | |
090089c4 | 1349 | } |
1350 | ||
24382924 | 1351 | static void |
b8d8561b | 1352 | storeStartRebuildFromDisk(void) |
3facc31e | 1353 | { |
3facc31e | 1354 | int i; |
5608850b | 1355 | struct storeRebuildState *RB; |
1356 | struct _rebuild_dir *d; | |
1357 | FILE *fp; | |
1358 | int clean; | |
1359 | RB = xcalloc(1, sizeof(struct storeRebuildState)); | |
1360 | RB->start = squid_curtime; | |
f1dc9b30 | 1361 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
5608850b | 1362 | fp = storeDirOpenTmpSwapLog(i, &clean); |
1363 | if (fp == NULL) | |
1364 | continue; | |
1365 | d = xcalloc(1, sizeof(struct _rebuild_dir)); | |
1366 | d->dirn = i; | |
1367 | d->log = fp; | |
1368 | d->clean = clean; | |
1369 | d->speed = opt_foreground_rebuild ? 1 << 30 : 50; | |
1370 | d->next = RB->rebuild_dir; | |
1371 | RB->rebuild_dir = d; | |
1372 | if (!clean) | |
f182d1c5 | 1373 | RB->need_to_validate = 1; |
a3d5953d | 1374 | debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n", |
5608850b | 1375 | i, clean ? "CLEAN" : "DIRTY"); |
1376 | } | |
1377 | RB->line_in_sz = 4096; | |
1378 | RB->line_in = xcalloc(1, RB->line_in_sz); | |
30a4f2a8 | 1379 | if (opt_foreground_rebuild) { |
5608850b | 1380 | storeDoRebuildFromDisk(RB); |
30a4f2a8 | 1381 | } else { |
5608850b | 1382 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); |
30a4f2a8 | 1383 | } |
3facc31e | 1384 | } |
090089c4 | 1385 | |
b8d8561b | 1386 | static int |
8350fe9b | 1387 | storeCheckCachable(StoreEntry * e) |
6602e70e | 1388 | { |
a7e59001 | 1389 | if (e->method != METHOD_GET) { |
8350fe9b | 1390 | debug(20, 2) ("storeCheckCachable: NO: non-GET method\n"); |
1c481e00 | 1391 | } else if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) { |
8350fe9b | 1392 | debug(20, 2) ("storeCheckCachable: NO: not cachable\n"); |
6602e70e | 1393 | } else if (BIT_TEST(e->flag, RELEASE_REQUEST)) { |
8350fe9b | 1394 | debug(20, 2) ("storeCheckCachable: NO: release requested\n"); |
83d453f3 | 1395 | } else if (e->store_status == STORE_OK && !storeEntryValidLength(e)) { |
8350fe9b | 1396 | debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n"); |
79b5cc5f | 1397 | } else if (BIT_TEST(e->flag, ENTRY_NEGCACHED)) { |
8350fe9b | 1398 | debug(20, 2) ("storeCheckCachable: NO: negative cached\n"); |
3e98df20 | 1399 | return 0; /* avoid release call below */ |
8350fe9b | 1400 | } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { |
1401 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); | |
89e079ed | 1402 | } else if (BIT_TEST(e->flag, KEY_PRIVATE)) { |
8350fe9b | 1403 | debug(20, 3) ("storeCheckCachable: NO: private key\n"); |
d4432957 | 1404 | } else { |
6602e70e | 1405 | return 1; |
d4432957 | 1406 | } |
2daae136 | 1407 | storeReleaseRequest(e); |
d3f89c29 | 1408 | BIT_CLR(e->flag, ENTRY_CACHABLE); |
6602e70e | 1409 | return 0; |
1410 | } | |
1411 | ||
090089c4 | 1412 | /* Complete transfer into the local cache. */ |
b8d8561b | 1413 | void |
1414 | storeComplete(StoreEntry * e) | |
090089c4 | 1415 | { |
9fb13bb6 | 1416 | debug(20, 3) ("storeComplete: '%s'\n", storeKeyText(e->key)); |
8350fe9b | 1417 | e->object_len = e->mem_obj->inmem_hi; |
234967c9 | 1418 | e->store_status = STORE_OK; |
8350fe9b | 1419 | assert(e->mem_status == NOT_IN_MEMORY); |
3a1c3e2f | 1420 | InvokeHandlers(e); |
8c123b71 | 1421 | storeCheckSwapOut(e); |
090089c4 | 1422 | } |
1423 | ||
1424 | /* | |
474cac1b | 1425 | * Someone wants to abort this transfer. Set the reason in the |
1426 | * request structure, call the server-side callback and mark the | |
1427 | * entry for releasing | |
090089c4 | 1428 | */ |
b8d8561b | 1429 | void |
9b312a19 | 1430 | storeAbort(StoreEntry * e, int cbflag) |
090089c4 | 1431 | { |
3e98df20 | 1432 | MemObject *mem = e->mem_obj; |
8f39b81d | 1433 | assert(e->store_status == STORE_PENDING); |
1434 | assert(mem != NULL); | |
9fb13bb6 | 1435 | debug(20, 6) ("storeAbort: %s\n", storeKeyText(e->key)); |
79b5cc5f | 1436 | storeNegativeCache(e); |
474cac1b | 1437 | storeReleaseRequest(e); |
234967c9 | 1438 | e->store_status = STORE_ABORTED; |
8350fe9b | 1439 | storeSetMemStatus(e, NOT_IN_MEMORY); |
090089c4 | 1440 | /* No DISK swap for negative cached object */ |
8350fe9b | 1441 | e->swap_status = SWAPOUT_NONE; |
090089c4 | 1442 | /* Count bytes faulted through cache but not moved to disk */ |
a528eb87 | 1443 | HTTPCacheInfo->proto_touchobject(HTTPCacheInfo, |
3e98df20 | 1444 | mem->request ? mem->request->protocol : PROTO_NONE, |
8350fe9b | 1445 | mem->inmem_hi); |
090089c4 | 1446 | /* We assign an object length here--The only other place we assign the |
1447 | * object length is in storeComplete() */ | |
8350fe9b | 1448 | e->object_len = mem->inmem_hi; |
474cac1b | 1449 | /* Notify the server side */ |
1450 | if (cbflag && mem->abort.callback) { | |
bfcaf585 | 1451 | mem->abort.callback(mem->abort.data); |
1452 | mem->abort.callback = NULL; | |
1453 | } | |
474cac1b | 1454 | /* Notify the client side */ |
090089c4 | 1455 | InvokeHandlers(e); |
6e86c3e8 | 1456 | /* Do we need to close the swapout file? */ |
1457 | /* Not if we never started swapping out */ | |
1458 | if (e->swap_file_number == -1) | |
1459 | return; | |
1460 | /* not if a disk write is queued, the handler will close up */ | |
1461 | if (mem->swapout.queue_offset > mem->swapout.done_offset) | |
1462 | return; | |
1463 | /* we do */ | |
1464 | storeSwapOutFileClose(e); | |
090089c4 | 1465 | } |
1466 | ||
090089c4 | 1467 | /* get the first entry in the storage */ |
b8d8561b | 1468 | StoreEntry * |
1469 | storeGetFirst(void) | |
090089c4 | 1470 | { |
c3c5b744 | 1471 | return ((StoreEntry *) hash_first(store_table)); |
090089c4 | 1472 | } |
1473 | ||
1474 | ||
1475 | /* get the next entry in the storage for a given search pointer */ | |
b8d8561b | 1476 | StoreEntry * |
1477 | storeGetNext(void) | |
090089c4 | 1478 | { |
c3c5b744 | 1479 | return ((StoreEntry *) hash_next(store_table)); |
090089c4 | 1480 | } |
1481 | ||
090089c4 | 1482 | |
090089c4 | 1483 | /* Clear Memory storage to accommodate the given object len */ |
38792624 | 1484 | static void |
d4432957 | 1485 | storeGetMemSpace(int size) |
090089c4 | 1486 | { |
b32508fb | 1487 | StoreEntry *e = NULL; |
20cba4b4 | 1488 | int released = 0; |
b32508fb | 1489 | static time_t last_check = 0; |
e954773d | 1490 | int pages_needed; |
b93bcace | 1491 | dlink_node *m; |
79d39a72 | 1492 | dlink_node *prev = NULL; |
b32508fb | 1493 | if (squid_curtime == last_check) |
38792624 | 1494 | return; |
b32508fb | 1495 | last_check = squid_curtime; |
e954773d | 1496 | pages_needed = (size / SM_PAGE_SIZE) + 1; |
61cd203b | 1497 | if (sm_stats.n_pages_in_use + pages_needed < store_pages_high) |
38792624 | 1498 | return; |
311ea387 | 1499 | if (store_rebuilding) |
38792624 | 1500 | return; |
a3d5953d | 1501 | debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed); |
b93bcace | 1502 | for (m = inmem_list.tail; m; m = prev) { |
8350fe9b | 1503 | prev = m->prev; |
b93bcace | 1504 | e = m->data; |
8350fe9b | 1505 | if (storeEntryLocked(e)) |
1506 | continue; | |
20cba4b4 | 1507 | released++; |
1508 | storeRelease(e); | |
8350fe9b | 1509 | if (sm_stats.n_pages_in_use + pages_needed < store_pages_low) |
1510 | break; | |
1511 | } | |
5e3d4fef | 1512 | debug(20, 3) ("storeGetMemSpace stats:\n"); |
1513 | debug(20, 3) (" %6d HOT objects\n", meta_data.hot_vm); | |
20cba4b4 | 1514 | debug(20, 3) (" %6d were released\n", released); |
090089c4 | 1515 | } |
1516 | ||
090089c4 | 1517 | /* The maximum objects to scan for maintain storage space */ |
fcefe642 | 1518 | #define MAINTAIN_MAX_SCAN 1024 |
1519 | #define MAINTAIN_MAX_REMOVE 64 | |
090089c4 | 1520 | |
fcefe642 | 1521 | /* |
1522 | * This routine is to be called by main loop in main.c. | |
1523 | * It removes expired objects on only one bucket for each time called. | |
1524 | * returns the number of objects removed | |
1525 | * | |
1526 | * This should get called 1/s from main(). | |
1527 | */ | |
679ac4f0 | 1528 | void |
79d39a72 | 1529 | storeMaintainSwapSpace(void *datanotused) |
090089c4 | 1530 | { |
b93bcace | 1531 | dlink_node *m; |
79d39a72 | 1532 | dlink_node *prev = NULL; |
a1b0d7cf | 1533 | StoreEntry *e = NULL; |
090089c4 | 1534 | int scanned = 0; |
090089c4 | 1535 | int locked = 0; |
679ac4f0 | 1536 | int expired = 0; |
20cba4b4 | 1537 | int max_scan; |
1538 | int max_remove; | |
ec14f197 | 1539 | static time_t last_warn_time = 0; |
fcefe642 | 1540 | eventAdd("storeMaintainSwapSpace", storeMaintainSwapSpace, NULL, 1); |
1541 | /* We can't delete objects while rebuilding swap */ | |
1542 | if (store_rebuilding) | |
679ac4f0 | 1543 | return; |
20cba4b4 | 1544 | if (store_swap_size < store_swap_high) { |
1545 | max_scan = 100; | |
1546 | max_remove = 10; | |
1547 | } else { | |
1548 | max_scan = 500; | |
1549 | max_remove = 50; | |
1550 | } | |
fcefe642 | 1551 | debug(20, 3) ("storeMaintainSwapSpace\n"); |
b93bcace | 1552 | for (m = all_list.tail; m; m = prev) { |
1553 | prev = m->prev; | |
1554 | e = m->data; | |
1555 | if (storeEntryLocked(e)) { | |
1556 | locked++; | |
fcefe642 | 1557 | } else if (storeCheckExpired(e, 1)) { |
b93bcace | 1558 | expired++; |
1559 | storeRelease(e); | |
090089c4 | 1560 | } |
20cba4b4 | 1561 | if (expired > max_remove) |
b93bcace | 1562 | break; |
20cba4b4 | 1563 | if (++scanned > max_scan) |
b93bcace | 1564 | break; |
090089c4 | 1565 | } |
fcefe642 | 1566 | debug(20, 3) ("storeMaintainSwapSpace stats:\n"); |
1567 | debug(20, 3) (" %6d objects\n", meta_data.store_entries); | |
1568 | debug(20, 3) (" %6d were scanned\n", scanned); | |
1569 | debug(20, 3) (" %6d were locked\n", locked); | |
1570 | debug(20, 3) (" %6d were expired\n", expired); | |
fcefe642 | 1571 | if (store_swap_size < Config.Swap.maxSize) |
b93bcace | 1572 | return; |
1573 | if (squid_curtime - last_warn_time < 10) | |
1574 | return; | |
1575 | debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", | |
1576 | store_swap_size, Config.Swap.maxSize); | |
1577 | last_warn_time = squid_curtime; | |
090089c4 | 1578 | } |
1579 | ||
1580 | ||
1581 | /* release an object from a cache */ | |
cc61958c | 1582 | /* return number of objects released. */ |
b8d8561b | 1583 | int |
1584 | storeRelease(StoreEntry * e) | |
090089c4 | 1585 | { |
9fb13bb6 | 1586 | debug(20, 3) ("storeRelease: Releasing: '%s'\n", storeKeyText(e->key)); |
090089c4 | 1587 | /* If, for any reason we can't discard this object because of an |
1588 | * outstanding request, mark it for pending release */ | |
1589 | if (storeEntryLocked(e)) { | |
9174e204 | 1590 | storeExpireNow(e); |
a3d5953d | 1591 | debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n"); |
2daae136 | 1592 | storeReleaseRequest(e); |
cc61958c | 1593 | return 0; |
090089c4 | 1594 | } |
311ea387 | 1595 | if (store_rebuilding) { |
a3d5953d | 1596 | debug(20, 2) ("storeRelease: Delaying release until store is rebuilt: '%s'\n", |
9fb13bb6 | 1597 | storeUrl(e)); |
30a4f2a8 | 1598 | storeExpireNow(e); |
1599 | storeSetPrivateKey(e); | |
35ac11a3 | 1600 | BIT_SET(e->flag, RELEASE_REQUEST); |
cc61958c | 1601 | return 0; |
30a4f2a8 | 1602 | } |
43fee0b8 | 1603 | if (e->swap_file_number > -1) { |
9e4ad609 | 1604 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1605 | storePutUnusedFileno(e->swap_file_number); | |
c932b107 | 1606 | storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); |
090089c4 | 1607 | e->swap_file_number = -1; |
090089c4 | 1608 | } |
8350fe9b | 1609 | storeSetMemStatus(e, NOT_IN_MEMORY); |
f7a5493b | 1610 | storeHashDelete(e); |
147d3115 | 1611 | storeLog(STORE_LOG_RELEASE, e); |
30a4f2a8 | 1612 | destroy_StoreEntry(e); |
cc61958c | 1613 | return 1; |
090089c4 | 1614 | } |
1615 | ||
090089c4 | 1616 | /* return 1 if a store entry is locked */ |
24382924 | 1617 | static int |
fe4e214f | 1618 | storeEntryLocked(const StoreEntry * e) |
090089c4 | 1619 | { |
30a4f2a8 | 1620 | if (e->lock_count) |
1621 | return 1; | |
8350fe9b | 1622 | if (e->swap_status == SWAPOUT_WRITING) |
30a4f2a8 | 1623 | return 1; |
a1e47288 | 1624 | if (e->store_status == STORE_PENDING) |
1625 | return 1; | |
365cb147 | 1626 | if (BIT_TEST(e->flag, ENTRY_SPECIAL)) |
1627 | return 1; | |
30a4f2a8 | 1628 | return 0; |
090089c4 | 1629 | } |
1630 | ||
090089c4 | 1631 | /* check if there is any client waiting for this object at all */ |
1632 | /* return 1 if there is at least one client */ | |
b8d8561b | 1633 | int |
fe4e214f | 1634 | storeClientWaiting(const StoreEntry * e) |
090089c4 | 1635 | { |
6e40f263 | 1636 | MemObject *mem = e->mem_obj; |
43329771 | 1637 | store_client *sc; |
1638 | for (sc = mem->clients; sc; sc = sc->next) { | |
1639 | if (sc->callback_data != NULL) | |
1640 | return 1; | |
090089c4 | 1641 | } |
1642 | return 0; | |
1643 | } | |
1644 | ||
43329771 | 1645 | static store_client * |
382d851a | 1646 | storeClientListSearch(const MemObject * mem, void *data) |
090089c4 | 1647 | { |
43329771 | 1648 | store_client *sc; |
1649 | for (sc = mem->clients; sc; sc = sc->next) { | |
1650 | if (sc->callback_data == data) | |
1651 | break; | |
090089c4 | 1652 | } |
43329771 | 1653 | return sc; |
090089c4 | 1654 | } |
1655 | ||
1656 | /* add client with fd to client list */ | |
43329771 | 1657 | void |
fe96bbe6 | 1658 | storeClientListAdd(StoreEntry * e, void *data) |
090089c4 | 1659 | { |
6e40f263 | 1660 | MemObject *mem = e->mem_obj; |
43329771 | 1661 | store_client **T; |
1662 | store_client *sc; | |
9fb13bb6 | 1663 | assert(mem); |
43329771 | 1664 | if (storeClientListSearch(mem, data) != NULL) |
1665 | return; | |
1666 | mem->nclients++; | |
1667 | sc = xcalloc(1, sizeof(store_client)); | |
5d86029a | 1668 | cbdataAdd(sc); /* sc is callback_data for file_read */ |
5f0e9628 | 1669 | sc->callback_data = data; |
1670 | sc->seen_offset = 0; | |
1671 | sc->copy_offset = 0; | |
1672 | sc->swapin_fd = -1; | |
1673 | sc->mem = mem; | |
43329771 | 1674 | if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) |
1675 | sc->type = STORE_MEM_CLIENT; | |
1676 | else | |
1677 | sc->type = STORE_DISK_CLIENT; | |
1678 | for (T = &mem->clients; *T; T = &(*T)->next); | |
1679 | *T = sc; | |
090089c4 | 1680 | } |
1681 | ||
dbfda5cb | 1682 | /* copy bytes requested by the client */ |
d89d1fb6 | 1683 | void |
38792624 | 1684 | storeClientCopy(StoreEntry * e, |
fe96bbe6 | 1685 | off_t seen_offset, |
1686 | off_t copy_offset, | |
d89d1fb6 | 1687 | size_t size, |
38792624 | 1688 | char *buf, |
d89d1fb6 | 1689 | STCB * callback, |
382d851a | 1690 | void *data) |
38792624 | 1691 | { |
43329771 | 1692 | store_client *sc; |
d4b1b930 | 1693 | static int recurse_detect = 0; |
b974bc9d | 1694 | assert(e->store_status != STORE_ABORTED); |
f182d1c5 | 1695 | assert(recurse_detect < 3); /* could == 1 for IMS not modified's */ |
8350fe9b | 1696 | debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", |
9fb13bb6 | 1697 | storeKeyText(e->key), |
86101e40 | 1698 | (int) seen_offset, |
1699 | (int) copy_offset, | |
1700 | (int) size, | |
1701 | callback, | |
1702 | data); | |
dbfda5cb | 1703 | sc = storeClientListSearch(e->mem_obj, data); |
8350fe9b | 1704 | assert(sc != NULL); |
edbaeece | 1705 | assert(sc->callback == NULL); |
fe96bbe6 | 1706 | sc->copy_offset = copy_offset; |
1707 | sc->seen_offset = seen_offset; | |
8350fe9b | 1708 | sc->callback = callback; |
1709 | sc->copy_buf = buf; | |
1710 | sc->copy_size = size; | |
1711 | sc->copy_offset = copy_offset; | |
dbfda5cb | 1712 | storeClientCopy2(e, sc); |
1713 | recurse_detect--; | |
1714 | } | |
1715 | ||
1716 | static void | |
1717 | storeClientCopy2(StoreEntry * e, store_client * sc) | |
1718 | { | |
1719 | STCB *callback = sc->callback; | |
1720 | MemObject *mem = e->mem_obj; | |
1721 | size_t sz; | |
9fb13bb6 | 1722 | debug(20, 3) ("storeClientCopy2: %s\n", storeKeyText(e->key)); |
b93bcace | 1723 | assert(callback != NULL); |
1724 | if (e->store_status == STORE_ABORTED) { | |
1725 | sc->callback = NULL; | |
1726 | callback(sc->callback_data, sc->copy_buf, 0); | |
1727 | return; | |
1728 | } | |
1729 | assert(sc->seen_offset <= mem->inmem_hi || e->store_status == STORE_OK); | |
dbfda5cb | 1730 | if (e->store_status == STORE_PENDING && sc->seen_offset == mem->inmem_hi) { |
fe96bbe6 | 1731 | /* client has already seen this, wait for more */ |
dbfda5cb | 1732 | debug(20, 3) ("storeClientCopy2: Waiting for more\n"); |
d89d1fb6 | 1733 | return; |
090089c4 | 1734 | } |
dbfda5cb | 1735 | if (sc->copy_offset >= mem->inmem_lo && mem->inmem_lo < mem->inmem_hi) { |
8350fe9b | 1736 | /* What the client wants is in memory */ |
dbfda5cb | 1737 | debug(20, 3) ("storeClientCopy2: Copying from memory\n"); |
1738 | sz = memCopy(mem->data, sc->copy_offset, sc->copy_buf, sc->copy_size); | |
8350fe9b | 1739 | sc->callback = NULL; |
dbfda5cb | 1740 | callback(sc->callback_data, sc->copy_buf, sz); |
8350fe9b | 1741 | } else if (sc->swapin_fd < 0) { |
dbfda5cb | 1742 | debug(20, 3) ("storeClientCopy2: Need to open swap in file\n"); |
43329771 | 1743 | assert(sc->type == STORE_DISK_CLIENT); |
8350fe9b | 1744 | /* gotta open the swapin file */ |
dbfda5cb | 1745 | assert(sc->copy_offset == 0); |
8350fe9b | 1746 | storeSwapInStart(e, storeClientCopyFileOpened, sc); |
1747 | } else { | |
1748 | debug(20, 3) ("storeClientCopy: reading from disk FD %d\n", | |
1749 | sc->swapin_fd); | |
43329771 | 1750 | assert(sc->type == STORE_DISK_CLIENT); |
8350fe9b | 1751 | storeClientCopyFileRead(sc); |
1752 | } | |
090089c4 | 1753 | } |
1754 | ||
8350fe9b | 1755 | static void |
1756 | storeClientCopyFileOpened(int fd, void *data) | |
1757 | { | |
43329771 | 1758 | store_client *sc = data; |
1759 | STCB *callback = sc->callback; | |
8350fe9b | 1760 | if (fd < 0) { |
edbaeece | 1761 | debug(20, 3) ("storeClientCopyFileOpened: failed\n"); |
43329771 | 1762 | sc->callback = NULL; |
1763 | callback(sc->callback_data, sc->copy_buf, -1); | |
8350fe9b | 1764 | return; |
1765 | } | |
1766 | sc->swapin_fd = fd; | |
1767 | storeClientCopyFileRead(sc); | |
1768 | } | |
1769 | ||
1770 | static void | |
5d86029a | 1771 | storeClientCopyFileRead(store_client * sc) |
8350fe9b | 1772 | { |
5d86029a | 1773 | assert(sc->callback != NULL); |
8350fe9b | 1774 | file_read(sc->swapin_fd, |
1775 | sc->copy_buf, | |
1776 | sc->copy_size, | |
1777 | sc->copy_offset, | |
1778 | storeClientCopyHandleRead, | |
1779 | sc); | |
1780 | } | |
1781 | ||
1782 | static void | |
79d39a72 | 1783 | storeClientCopyHandleRead(int fd, const char *buf, int len, int flagnotused, void *data) |
8350fe9b | 1784 | { |
43329771 | 1785 | store_client *sc = data; |
5f0e9628 | 1786 | MemObject *mem = sc->mem; |
43329771 | 1787 | STCB *callback = sc->callback; |
06e8899b | 1788 | assert(sc->callback != NULL); |
5f0e9628 | 1789 | debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); |
43329771 | 1790 | if (sc->copy_offset == 0 && len > 0 && mem != NULL) |
6a54c60e | 1791 | httpParseReplyHeaders(buf, mem->reply); |
43329771 | 1792 | sc->callback = NULL; |
1793 | callback(sc->callback_data, sc->copy_buf, len); | |
8350fe9b | 1794 | } |
1795 | ||
24382924 | 1796 | static int |
fe4e214f | 1797 | storeEntryValidLength(const StoreEntry * e) |
6602e70e | 1798 | { |
ffe4a367 | 1799 | int diff; |
2285407f | 1800 | int hdr_sz; |
1801 | int content_length; | |
ffe4a367 | 1802 | |
2285407f | 1803 | if (e->mem_obj == NULL) |
ffe4a367 | 1804 | fatal_dump("storeEntryValidLength: NULL mem_obj"); |
1805 | ||
2285407f | 1806 | hdr_sz = e->mem_obj->reply->hdr_sz; |
147d3115 | 1807 | content_length = e->mem_obj->reply->content_length; |
2285407f | 1808 | |
9fb13bb6 | 1809 | debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", storeKeyText(e->key)); |
a3d5953d | 1810 | debug(20, 5) ("storeEntryValidLength: object_len = %d\n", e->object_len); |
1811 | debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", hdr_sz); | |
1812 | debug(20, 5) ("storeEntryValidLength: content_length = %d\n", content_length); | |
ffe4a367 | 1813 | |
2285407f | 1814 | if (content_length == 0) { |
a3d5953d | 1815 | debug(20, 5) ("storeEntryValidLength: Zero content length; assume valid; '%s'\n", |
9fb13bb6 | 1816 | storeKeyText(e->key)); |
ffe4a367 | 1817 | return 1; |
1818 | } | |
2285407f | 1819 | if (hdr_sz == 0) { |
a3d5953d | 1820 | debug(20, 5) ("storeEntryValidLength: Zero header size; assume valid; '%s'\n", |
9fb13bb6 | 1821 | storeKeyText(e->key)); |
ffe4a367 | 1822 | return 1; |
1823 | } | |
2285407f | 1824 | diff = hdr_sz + content_length - e->object_len; |
ffe4a367 | 1825 | if (diff != 0) { |
a3d5953d | 1826 | debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", |
ffe4a367 | 1827 | diff < 0 ? -diff : diff, |
1828 | diff < 0 ? "small" : "big", | |
9fb13bb6 | 1829 | storeKeyText(e->key)); |
ffe4a367 | 1830 | return 0; |
1831 | } | |
1832 | return 1; | |
1833 | } | |
6602e70e | 1834 | |
66cedb85 | 1835 | static void |
1836 | storeInitHashValues(void) | |
1837 | { | |
1838 | int i; | |
a7e59001 | 1839 | /* Calculate size of hash table (maximum currently 64k buckets). */ |
38792624 | 1840 | i = Config.Swap.maxSize / Config.Store.avgObjectSize; |
86101e40 | 1841 | debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n", |
66cedb85 | 1842 | Config.Swap.maxSize, i); |
38792624 | 1843 | i /= Config.Store.objectsPerBucket; |
a3d5953d | 1844 | debug(20, 1) ("Target number of buckets: %d\n", i); |
66cedb85 | 1845 | /* ideally the full scan period should be configurable, for the |
1846 | * moment it remains at approximately 24 hours. */ | |
9fb13bb6 | 1847 | store_hash_buckets = storeKeyHashBuckets(i); |
24ffafb4 | 1848 | store_maintain_rate = 86400 / store_hash_buckets; |
9fb13bb6 | 1849 | assert(store_maintain_rate > 0); |
a3d5953d | 1850 | debug(20, 1) ("Using %d Store buckets, maintain %d bucket%s every %d second%s\n", |
9fb13bb6 | 1851 | store_hash_buckets, |
66cedb85 | 1852 | store_maintain_buckets, |
1853 | store_maintain_buckets == 1 ? null_string : "s", | |
1854 | store_maintain_rate, | |
1855 | store_maintain_rate == 1 ? null_string : "s"); | |
a47b9029 | 1856 | debug(20, 1) ("Max Mem size: %d KB\n", Config.Mem.maxSize >> 10); |
1857 | debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize); | |
66cedb85 | 1858 | } |
1859 | ||
b8d8561b | 1860 | void |
1861 | storeInit(void) | |
c943f331 | 1862 | { |
234967c9 | 1863 | char *fname = NULL; |
66cedb85 | 1864 | storeInitHashValues(); |
9fb13bb6 | 1865 | store_table = hash_create(storeKeyHashCmp, |
1866 | store_hash_buckets, storeKeyHashHash); | |
b6f794d6 | 1867 | if (strcmp((fname = Config.Log.store), "none") == 0) |
234967c9 | 1868 | storelog_fd = -1; |
1869 | else | |
f17936ab | 1870 | storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); |
234967c9 | 1871 | if (storelog_fd < 0) |
a3d5953d | 1872 | debug(20, 1) ("Store logging disabled\n"); |
e7a22b88 | 1873 | storeVerifySwapDirs(); |
1874 | storeDirOpenSwapLogs(); | |
b6f794d6 | 1875 | if (!opt_zap_disk_store) |
3facc31e | 1876 | storeStartRebuildFromDisk(); |
30a4f2a8 | 1877 | else |
311ea387 | 1878 | store_rebuilding = 0; |
b93bcace | 1879 | all_list.head = all_list.tail = NULL; |
1880 | inmem_list.head = inmem_list.tail = NULL; | |
b1c0cc67 | 1881 | } |
c943f331 | 1882 | |
b8d8561b | 1883 | void |
1884 | storeConfigure(void) | |
b1c0cc67 | 1885 | { |
e954773d | 1886 | int store_mem_high = 0; |
1887 | int store_mem_low = 0; | |
b6f794d6 | 1888 | store_mem_high = (long) (Config.Mem.maxSize / 100) * |
1889 | Config.Mem.highWaterMark; | |
1890 | store_mem_low = (long) (Config.Mem.maxSize / 100) * | |
1891 | Config.Mem.lowWaterMark; | |
090089c4 | 1892 | |
b1c0cc67 | 1893 | store_swap_high = (long) (((float) Config.Swap.maxSize * |
1894 | (float) Config.Swap.highWaterMark) / (float) 100); | |
1895 | store_swap_low = (long) (((float) Config.Swap.maxSize * | |
1896 | (float) Config.Swap.lowWaterMark) / (float) 100); | |
e954773d | 1897 | |
1898 | store_pages_high = store_mem_high / SM_PAGE_SIZE; | |
1899 | store_pages_low = store_mem_low / SM_PAGE_SIZE; | |
090089c4 | 1900 | } |
1901 | ||
b8d8561b | 1902 | int |
9fb13bb6 | 1903 | urlcmp(const void *url1, const void *url2) |
090089c4 | 1904 | { |
1905 | if (!url1 || !url2) | |
6eb42cae | 1906 | fatal_dump("urlcmp: Got a NULL url pointer."); |
090089c4 | 1907 | return (strcmp(url1, url2)); |
1908 | } | |
1909 | ||
090089c4 | 1910 | /* |
5608850b | 1911 | * storeWriteCleanLogs |
090089c4 | 1912 | * |
1913 | * Writes a "clean" swap log file from in-memory metadata. | |
1914 | */ | |
b8d8561b | 1915 | int |
e102ebda | 1916 | storeWriteCleanLogs(int reopen) |
090089c4 | 1917 | { |
1918 | StoreEntry *e = NULL; | |
9e6c462c | 1919 | int *fd; |
5608850b | 1920 | char *line; |
090089c4 | 1921 | int n = 0; |
1922 | time_t start, stop, r; | |
10389b74 | 1923 | struct stat sb; |
9e6c462c | 1924 | char **cur; |
1925 | char **new; | |
1926 | char **cln; | |
5608850b | 1927 | int dirn; |
30a4f2a8 | 1928 | if (store_rebuilding) { |
a3d5953d | 1929 | debug(20, 1) ("Not currently OK to rewrite swap log.\n"); |
1930 | debug(20, 1) ("storeWriteCleanLogs: Operation aborted.\n"); | |
20cba4b4 | 1931 | storeDirCloseSwapLogs(); |
090089c4 | 1932 | return 0; |
1933 | } | |
a3d5953d | 1934 | debug(20, 1) ("storeWriteCleanLogs: Starting...\n"); |
97c03d3c | 1935 | start = squid_curtime; |
f1dc9b30 | 1936 | fd = xcalloc(Config.cacheSwap.n_configured, sizeof(int)); |
1937 | cur = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
1938 | new = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
1939 | cln = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
1940 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { | |
3e347513 | 1941 | fd[dirn] = -1; |
1942 | cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL)); | |
1943 | new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean")); | |
1944 | cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean")); | |
1945 | safeunlink(new[dirn], 1); | |
1946 | safeunlink(cln[dirn], 1); | |
f17936ab | 1947 | fd[dirn] = file_open(new[dirn], |
9e4ad609 | 1948 | O_WRONLY | O_CREAT | O_TRUNC, |
1949 | NULL, | |
1950 | NULL); | |
3e347513 | 1951 | if (fd[dirn] < 0) { |
a3d5953d | 1952 | debug(50, 0) ("storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); |
5608850b | 1953 | continue; |
1954 | } | |
4ac29a5b | 1955 | #if HAVE_FCHMOD |
3e347513 | 1956 | if (stat(cur[dirn], &sb) == 0) |
5608850b | 1957 | fchmod(fd[dirn], sb.st_mode); |
4ac29a5b | 1958 | #endif |
5608850b | 1959 | } |
1960 | line = xcalloc(1, 16384); | |
090089c4 | 1961 | for (e = storeGetFirst(); e; e = storeGetNext()) { |
090089c4 | 1962 | if (e->swap_file_number < 0) |
1963 | continue; | |
8350fe9b | 1964 | if (e->swap_status != SWAPOUT_DONE) |
090089c4 | 1965 | continue; |
1966 | if (e->object_len <= 0) | |
1967 | continue; | |
3ba50d75 | 1968 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
0a21bd84 | 1969 | continue; |
1970 | if (BIT_TEST(e->flag, KEY_PRIVATE)) | |
1971 | continue; | |
9e975e4e | 1972 | dirn = storeDirNumber(e->swap_file_number); |
1973 | assert(dirn < Config.cacheSwap.n_configured); | |
5608850b | 1974 | if (fd[dirn] < 0) |
1975 | continue; | |
042461c3 | 1976 | snprintf(line, 16384, "%08x %08x %08x %08x %08x %9d %6d %08x %s\n", |
f5469bce | 1977 | (int) e->swap_file_number, |
1978 | (int) e->timestamp, | |
919877b6 | 1979 | (int) e->lastref, |
f5469bce | 1980 | (int) e->expires, |
1981 | (int) e->lastmod, | |
1982 | e->object_len, | |
994bbf93 | 1983 | e->refcount, |
1984 | e->flag, | |
9fb13bb6 | 1985 | storeKeyText(e->key)); |
5608850b | 1986 | if (write(fd[dirn], line, strlen(line)) < 0) { |
a3d5953d | 1987 | debug(50, 0) ("storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); |
1988 | debug(20, 0) ("storeWriteCleanLogs: Current swap logfile not replaced.\n"); | |
5608850b | 1989 | file_close(fd[dirn]); |
1990 | fd[dirn] = -1; | |
1991 | safeunlink(cln[dirn], 0); | |
1992 | continue; | |
30a4f2a8 | 1993 | } |
5608850b | 1994 | if ((++n & 0x3FFF) == 0) { |
a3d5953d | 1995 | debug(20, 1) (" %7d lines written so far.\n", n); |
090089c4 | 1996 | } |
1997 | } | |
5608850b | 1998 | safe_free(line); |
f1dc9b30 | 1999 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { |
3e347513 | 2000 | file_close(fd[dirn]); |
2001 | fd[dirn] = -1; | |
2002 | if (rename(new[dirn], cur[dirn]) < 0) { | |
a3d5953d | 2003 | debug(50, 0) ("storeWriteCleanLogs: rename failed: %s\n", |
3e347513 | 2004 | xstrerror()); |
2005 | } | |
5608850b | 2006 | } |
2007 | storeDirCloseSwapLogs(); | |
e102ebda | 2008 | if (reopen) |
2009 | storeDirOpenSwapLogs(); | |
97c03d3c | 2010 | stop = squid_curtime; |
090089c4 | 2011 | r = stop - start; |
a3d5953d | 2012 | debug(20, 1) (" Finished. Wrote %d lines.\n", n); |
2013 | debug(20, 1) (" Took %d seconds (%6.1lf lines/sec).\n", | |
090089c4 | 2014 | r > 0 ? r : 0, (double) n / (r > 0 ? r : 1)); |
0a0bf5db | 2015 | /* touch a timestamp file if we're not still validating */ |
f1dc9b30 | 2016 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { |
f182d1c5 | 2017 | if (!store_rebuilding) |
f17936ab | 2018 | file_close(file_open(cln[dirn], |
3e347513 | 2019 | O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL)); |
5608850b | 2020 | safe_free(cur[dirn]); |
2021 | safe_free(new[dirn]); | |
2022 | safe_free(cln[dirn]); | |
0a0bf5db | 2023 | } |
9e6c462c | 2024 | safe_free(cur); |
2025 | safe_free(new); | |
2026 | safe_free(cln); | |
2027 | safe_free(fd); | |
090089c4 | 2028 | return n; |
2029 | } | |
2030 | ||
b8d8561b | 2031 | int |
fe4e214f | 2032 | storePendingNClients(const StoreEntry * e) |
090089c4 | 2033 | { |
2034 | int npend = 0; | |
52d4522b | 2035 | MemObject *mem = e->mem_obj; |
43329771 | 2036 | store_client *sc; |
2037 | store_client *nx = NULL; | |
52d4522b | 2038 | if (mem == NULL) |
090089c4 | 2039 | return 0; |
5d86029a | 2040 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 2041 | nx = sc->next; |
2042 | if (sc->callback_data == NULL) | |
52d4522b | 2043 | continue; |
b012353a | 2044 | npend++; |
090089c4 | 2045 | } |
2046 | return npend; | |
2047 | } | |
d8b45066 | 2048 | |
b8d8561b | 2049 | void |
2050 | storeRotateLog(void) | |
d8b45066 | 2051 | { |
2052 | char *fname = NULL; | |
2053 | int i; | |
95d659f0 | 2054 | LOCAL_ARRAY(char, from, MAXPATHLEN); |
2055 | LOCAL_ARRAY(char, to, MAXPATHLEN); | |
88738790 | 2056 | #ifdef S_ISREG |
2edc2504 | 2057 | struct stat sb; |
88738790 | 2058 | #endif |
d8b45066 | 2059 | |
234967c9 | 2060 | if (storelog_fd > -1) { |
2061 | file_close(storelog_fd); | |
2062 | storelog_fd = -1; | |
2063 | } | |
b6f794d6 | 2064 | if ((fname = Config.Log.store) == NULL) |
2d25dc1a | 2065 | return; |
234967c9 | 2066 | if (strcmp(fname, "none") == 0) |
2067 | return; | |
71d95578 | 2068 | #ifdef S_ISREG |
2edc2504 | 2069 | if (stat(fname, &sb) == 0) |
2070 | if (S_ISREG(sb.st_mode) == 0) | |
2071 | return; | |
71d95578 | 2072 | #endif |
234967c9 | 2073 | |
a3d5953d | 2074 | debug(20, 1) ("storeRotateLog: Rotating.\n"); |
d8b45066 | 2075 | |
2076 | /* Rotate numbers 0 through N up one */ | |
b6f794d6 | 2077 | for (i = Config.Log.rotateNumber; i > 1;) { |
2d25dc1a | 2078 | i--; |
8350fe9b | 2079 | snprintf(from, MAXPATHLEN, "%s.%d", fname, i - 1); |
042461c3 | 2080 | snprintf(to, MAXPATHLEN, "%s.%d", fname, i); |
2d25dc1a | 2081 | rename(from, to); |
d8b45066 | 2082 | } |
2083 | /* Rotate the current log to .0 */ | |
b6f794d6 | 2084 | if (Config.Log.rotateNumber > 0) { |
8350fe9b | 2085 | snprintf(to, MAXPATHLEN, "%s.%d", fname, 0); |
2d25dc1a | 2086 | rename(fname, to); |
d8b45066 | 2087 | } |
f17936ab | 2088 | storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); |
234967c9 | 2089 | if (storelog_fd < 0) { |
a3d5953d | 2090 | debug(50, 0) ("storeRotateLog: %s: %s\n", fname, xstrerror()); |
2091 | debug(20, 1) ("Store logging disabled\n"); | |
234967c9 | 2092 | } |
d8b45066 | 2093 | } |
30a4f2a8 | 2094 | |
56f29785 | 2095 | static int |
8350fe9b | 2096 | storeKeepInMemory(const StoreEntry * e) |
56f29785 | 2097 | { |
8350fe9b | 2098 | MemObject *mem = e->mem_obj; |
2099 | if (mem == NULL) | |
56f29785 | 2100 | return 0; |
8350fe9b | 2101 | if (mem->data == NULL) |
2102 | return 0; | |
2103 | return mem->inmem_lo == 0; | |
56f29785 | 2104 | } |
2105 | ||
b8d8561b | 2106 | static int |
429fdbec | 2107 | storeCheckExpired(const StoreEntry * e, int check_lru_age) |
620da955 | 2108 | { |
2109 | if (storeEntryLocked(e)) | |
2110 | return 0; | |
7cc1830f | 2111 | if (BIT_TEST(e->flag, ENTRY_NEGCACHED) && squid_curtime >= e->expires) |
2112 | return 1; | |
429fdbec | 2113 | if (!check_lru_age) |
2114 | return 0; | |
fcefe642 | 2115 | if (squid_curtime - e->lastref > storeExpiredReferenceAge()) |
66cedb85 | 2116 | return 1; |
66cedb85 | 2117 | return 0; |
620da955 | 2118 | } |
2119 | ||
db9ccd5a | 2120 | /* |
2121 | * storeExpiredReferenceAge | |
2122 | * | |
429fdbec | 2123 | * The LRU age is scaled exponentially between 1 minute and |
2124 | * Config.referenceAge , when store_swap_low < store_swap_size < | |
2125 | * store_swap_high. This keeps store_swap_size within the low and high | |
2126 | * water marks. If the cache is very busy then store_swap_size stays | |
2127 | * closer to the low water mark, if it is not busy, then it will stay | |
2128 | * near the high water mark. The LRU age value can be examined on the | |
2129 | * cachemgr 'info' page. | |
fbdfb3cf | 2130 | */ |
35feb4aa | 2131 | time_t |
58104eab | 2132 | storeExpiredReferenceAge(void) |
2133 | { | |
58104eab | 2134 | double x; |
2135 | double z; | |
2136 | time_t age; | |
429fdbec | 2137 | x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low); |
2138 | x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; | |
2139 | z = pow((double) Config.referenceAge, x); | |
58104eab | 2140 | age = (time_t) (z * 60.0); |
2141 | if (age < 60) | |
2142 | age = 60; | |
2143 | else if (age > 31536000) | |
2144 | age = 31536000; | |
2145 | return age; | |
0a362c34 | 2146 | } |
2edc2504 | 2147 | |
b8d8561b | 2148 | void |
2149 | storeCloseLog(void) | |
457bedbd | 2150 | { |
15be2259 | 2151 | if (storelog_fd >= 0) |
40fcf5d4 | 2152 | file_close(storelog_fd); |
457bedbd | 2153 | } |
79b5cc5f | 2154 | |
b8d8561b | 2155 | void |
2156 | storeNegativeCache(StoreEntry * e) | |
79b5cc5f | 2157 | { |
ff5731b1 | 2158 | e->expires = squid_curtime + Config.negativeTtl; |
2159 | BIT_SET(e->flag, ENTRY_NEGCACHED); | |
79b5cc5f | 2160 | } |
0a21bd84 | 2161 | |
2162 | void | |
2163 | storeFreeMemory(void) | |
2164 | { | |
2165 | StoreEntry *e; | |
df92bd2a | 2166 | StoreEntry **list; |
2167 | int i = 0; | |
ea6b05e3 | 2168 | int j; |
56e15c50 | 2169 | list = xcalloc(meta_data.store_entries, sizeof(StoreEntry *)); |
df92bd2a | 2170 | e = (StoreEntry *) hash_first(store_table); |
2171 | while (e && i < meta_data.store_entries) { | |
2172 | *(list + i) = e; | |
2173 | i++; | |
2174 | e = (StoreEntry *) hash_next(store_table); | |
2175 | } | |
ea6b05e3 | 2176 | for (j = 0; j < i; j++) |
2177 | destroy_StoreEntry(*(list + j)); | |
df92bd2a | 2178 | xfree(list); |
0a21bd84 | 2179 | hashFreeMemory(store_table); |
afe95a7e | 2180 | store_table = NULL; |
0a21bd84 | 2181 | } |
a7e59001 | 2182 | |
2183 | int | |
2184 | expiresMoreThan(time_t expires, time_t when) | |
2185 | { | |
48f44632 | 2186 | if (expires < 0) /* No Expires given */ |
2187 | return 1; | |
2188 | return (expires > (squid_curtime + when)); | |
a7e59001 | 2189 | } |
fe54d06d | 2190 | |
2191 | int | |
2192 | storeEntryValidToSend(StoreEntry * e) | |
2193 | { | |
2194 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) | |
2195 | return 0; | |
2196 | if (BIT_TEST(e->flag, ENTRY_NEGCACHED)) | |
44f78c24 | 2197 | if (e->expires <= squid_curtime) |
fe54d06d | 2198 | return 0; |
2199 | if (e->store_status == STORE_ABORTED) | |
2200 | return 0; | |
2201 | return 1; | |
2202 | } | |
62663274 | 2203 | |
ca98227c | 2204 | void |
2205 | storeTimestampsSet(StoreEntry * entry) | |
2206 | { | |
2207 | time_t served_date = -1; | |
2208 | struct _http_reply *reply = entry->mem_obj->reply; | |
2209 | served_date = reply->date > -1 ? reply->date : squid_curtime; | |
2210 | entry->expires = reply->expires; | |
2211 | if (reply->last_modified > -1) | |
2212 | entry->lastmod = reply->last_modified; | |
2213 | else | |
2214 | entry->lastmod = served_date; | |
2215 | entry->timestamp = served_date; | |
2216 | } | |
429fdbec | 2217 | |
2218 | #define FILENO_STACK_SIZE 128 | |
2219 | static int fileno_stack[FILENO_STACK_SIZE]; | |
429fdbec | 2220 | |
2221 | static int | |
2222 | storeGetUnusedFileno(void) | |
2223 | { | |
9e4ad609 | 2224 | int fn; |
429fdbec | 2225 | if (fileno_stack_count < 1) |
2226 | return -1; | |
9e4ad609 | 2227 | fn = fileno_stack[--fileno_stack_count]; |
2228 | storeDirMapBitSet(fn); | |
2229 | return fn; | |
429fdbec | 2230 | } |
2231 | ||
2232 | static void | |
2233 | storePutUnusedFileno(int fileno) | |
2234 | { | |
bf33177b | 2235 | assert(storeDirMapBitTest(fileno)); |
9e4ad609 | 2236 | storeDirMapBitReset(fileno); |
429fdbec | 2237 | if (fileno_stack_count < FILENO_STACK_SIZE) |
2238 | fileno_stack[fileno_stack_count++] = fileno; | |
2239 | else | |
2240 | unlinkdUnlink(storeSwapFullPath(fileno, NULL)); | |
2241 | } | |
bfcaf585 | 2242 | |
2243 | void | |
2244 | storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) | |
2245 | { | |
2246 | MemObject *mem = e->mem_obj; | |
2247 | assert(mem); | |
2248 | assert(mem->abort.callback == NULL); | |
2249 | mem->abort.callback = cb; | |
2250 | mem->abort.data = data; | |
2251 | } | |
2252 | ||
2253 | void | |
2254 | storeUnregisterAbort(StoreEntry * e) | |
2255 | { | |
2256 | MemObject *mem = e->mem_obj; | |
2257 | assert(mem); | |
2258 | mem->abort.callback = NULL; | |
2259 | } | |
88738790 | 2260 | |
2261 | void | |
2262 | storeMemObjectDump(MemObject * mem) | |
2263 | { | |
2264 | debug(20, 1) ("MemObject->data: %p\n", | |
2265 | mem->data); | |
88738790 | 2266 | debug(20, 1) ("MemObject->start_ping: %d.%06d\n", |
2267 | mem->start_ping.tv_sec, | |
2268 | mem->start_ping.tv_usec); | |
8350fe9b | 2269 | debug(20, 1) ("MemObject->inmem_hi: %d\n", |
2270 | mem->inmem_hi); | |
2271 | debug(20, 1) ("MemObject->inmem_lo: %d\n", | |
2272 | mem->inmem_lo); | |
88738790 | 2273 | debug(20, 1) ("MemObject->clients: %p\n", |
2274 | mem->clients); | |
2275 | debug(20, 1) ("MemObject->nclients: %d\n", | |
2276 | mem->nclients); | |
8350fe9b | 2277 | debug(20, 1) ("MemObject->swapout.fd: %d\n", |
2278 | mem->swapout.fd); | |
88738790 | 2279 | debug(20, 1) ("MemObject->reply: %p\n", |
2280 | mem->reply); | |
2281 | debug(20, 1) ("MemObject->request: %p\n", | |
2282 | mem->request); | |
2283 | debug(20, 1) ("MemObject->log_url: %p %s\n", | |
2284 | mem->log_url, | |
2285 | checkNullString(mem->log_url)); | |
2286 | } | |
8350fe9b | 2287 | |
2288 | static void | |
b93bcace | 2289 | storeEntryListAdd(StoreEntry * e, dlink_node * m, dlink_list * list) |
8350fe9b | 2290 | { |
9fb13bb6 | 2291 | debug(20, 3) ("storeEntryListAdd: %s\n", storeKeyText(e->key)); |
b93bcace | 2292 | m->data = e; |
8350fe9b | 2293 | m->prev = NULL; |
b93bcace | 2294 | m->next = list->head; |
2295 | if (list->head) | |
2296 | list->head->prev = m; | |
2297 | list->head = m; | |
2298 | if (list->tail == NULL) | |
2299 | list->tail = m; | |
8350fe9b | 2300 | } |
2301 | ||
2302 | static void | |
b93bcace | 2303 | storeEntryListDelete(dlink_node * m, dlink_list * list) |
8350fe9b | 2304 | { |
8350fe9b | 2305 | if (m->next) |
2306 | m->next->prev = m->prev; | |
2307 | if (m->prev) | |
2308 | m->prev->next = m->next; | |
b93bcace | 2309 | if (m == list->head) |
2310 | list->head = m->next; | |
2311 | if (m == list->tail) | |
2312 | list->tail = m->prev; | |
8350fe9b | 2313 | } |
2314 | ||
2315 | /* NOTE, this function assumes only two mem states */ | |
2316 | static void | |
2317 | storeSetMemStatus(StoreEntry * e, int new_status) | |
2318 | { | |
b93bcace | 2319 | MemObject *mem = e->mem_obj; |
8350fe9b | 2320 | if (new_status == e->mem_status) |
2321 | return; | |
b93bcace | 2322 | assert(mem != NULL); |
2323 | if (new_status == IN_MEMORY) { | |
2324 | assert(mem->inmem_lo == 0); | |
2325 | storeEntryListAdd(e, &mem->lru, &inmem_list); | |
2326 | meta_data.hot_vm++; | |
2327 | } else { | |
2328 | storeEntryListDelete(&mem->lru, &inmem_list); | |
2329 | meta_data.hot_vm--; | |
2330 | } | |
8350fe9b | 2331 | e->mem_status = new_status; |
2332 | } | |
6e86c3e8 | 2333 | |
2334 | static void | |
2335 | storeSwapOutFileClose(StoreEntry * e) | |
2336 | { | |
bb5f11c4 | 2337 | MemObject *mem = e->mem_obj; |
2338 | if (mem->swapout.fd > -1) | |
20cba4b4 | 2339 | file_close(mem->swapout.fd); |
bb5f11c4 | 2340 | mem->swapout.fd = -1; |
6e86c3e8 | 2341 | storeUnlockObject(e); |
2342 | } | |
9fb13bb6 | 2343 | |
2344 | const char * | |
2345 | storeUrl(const StoreEntry * e) | |
2346 | { | |
2347 | if (e == NULL) | |
24ffafb4 | 2348 | return "[null_entry]"; |
9fb13bb6 | 2349 | else if (e->mem_obj == NULL) |
24ffafb4 | 2350 | return "[null_mem_obj]"; |
9fb13bb6 | 2351 | else |
2352 | return e->mem_obj->url; | |
2353 | } | |
24ffafb4 | 2354 | |
2355 | void | |
2356 | storeCreateMemObject(StoreEntry * e, const char *url, const char *log_url) | |
2357 | { | |
2358 | if (e->mem_obj) | |
2359 | return; | |
2360 | e->mem_obj = new_MemObject(url, log_url); | |
2361 | } |