]>
Commit | Line | Data |
---|---|---|
ff5731b1 | 1 | |
30a4f2a8 | 2 | /* |
76a5501f | 3 | * $Id: store.cc,v 1.322 1997/10/28 20:42:54 wessels Exp $ |
30a4f2a8 | 4 | * |
5 | * DEBUG: section 20 Storeage Manager | |
6 | * AUTHOR: Harvest Derived | |
7 | * | |
42c04c16 | 8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ |
30a4f2a8 | 9 | * -------------------------------------------------------- |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
13 | * National Laboratory for Applied Network Research and funded by | |
14 | * the National Science Foundation. | |
15 | * | |
16 | * This program is free software; you can redistribute it and/or modify | |
17 | * it under the terms of the GNU General Public License as published by | |
18 | * the Free Software Foundation; either version 2 of the License, or | |
19 | * (at your option) any later version. | |
20 | * | |
21 | * This program is distributed in the hope that it will be useful, | |
22 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
24 | * GNU General Public License for more details. | |
25 | * | |
26 | * You should have received a copy of the GNU General Public License | |
27 | * along with this program; if not, write to the Free Software | |
28 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | * | |
30 | */ | |
c943f331 | 31 | |
32 | /* | |
30a4f2a8 | 33 | * Copyright (c) 1994, 1995. All rights reserved. |
34 | * | |
35 | * The Harvest software was developed by the Internet Research Task | |
36 | * Force Research Group on Resource Discovery (IRTF-RD): | |
37 | * | |
38 | * Mic Bowman of Transarc Corporation. | |
39 | * Peter Danzig of the University of Southern California. | |
40 | * Darren R. Hardy of the University of Colorado at Boulder. | |
41 | * Udi Manber of the University of Arizona. | |
42 | * Michael F. Schwartz of the University of Colorado at Boulder. | |
43 | * Duane Wessels of the University of Colorado at Boulder. | |
44 | * | |
45 | * This copyright notice applies to software in the Harvest | |
46 | * ``src/'' directory only. Users should consult the individual | |
47 | * copyright notices in the ``components/'' subdirectories for | |
48 | * copyright information about other software bundled with the | |
49 | * Harvest source code distribution. | |
50 | * | |
51 | * TERMS OF USE | |
52 | * | |
53 | * The Harvest software may be used and re-distributed without | |
54 | * charge, provided that the software origin and research team are | |
55 | * cited in any use of the system. Most commonly this is | |
56 | * accomplished by including a link to the Harvest Home Page | |
57 | * (http://harvest.cs.colorado.edu/) from the query page of any | |
58 | * Broker you deploy, as well as in the query result pages. These | |
59 | * links are generated automatically by the standard Broker | |
60 | * software distribution. | |
61 | * | |
62 | * The Harvest software is provided ``as is'', without express or | |
63 | * implied warranty, and with no support nor obligation to assist | |
64 | * in its use, correction, modification or enhancement. We assume | |
65 | * no liability with respect to the infringement of copyrights, | |
66 | * trade secrets, or any patents, and are not responsible for | |
67 | * consequential damages. Proper use of the Harvest software is | |
68 | * entirely the responsibility of the user. | |
69 | * | |
70 | * DERIVATIVE WORKS | |
71 | * | |
72 | * Users may make derivative works from the Harvest software, subject | |
73 | * to the following constraints: | |
74 | * | |
75 | * - You must include the above copyright notice and these | |
76 | * accompanying paragraphs in all forms of derivative works, | |
77 | * and any documentation and other materials related to such | |
78 | * distribution and use acknowledge that the software was | |
79 | * developed at the above institutions. | |
80 | * | |
81 | * - You must notify IRTF-RD regarding your distribution of | |
82 | * the derivative work. | |
83 | * | |
84 | * - You must clearly notify users that your are distributing | |
85 | * a modified version and not the original Harvest software. | |
86 | * | |
87 | * - Any derivative product is also subject to these copyright | |
88 | * and use restrictions. | |
89 | * | |
90 | * Note that the Harvest software is NOT in the public domain. We | |
91 | * retain copyright, as specified above. | |
92 | * | |
93 | * HISTORY OF FREE SOFTWARE STATUS | |
94 | * | |
95 | * Originally we required sites to license the software in cases | |
96 | * where they were going to build commercial products/services | |
97 | * around Harvest. In June 1995 we changed this policy. We now | |
98 | * allow people to use the core Harvest software (the code found in | |
99 | * the Harvest ``src/'' directory) for free. We made this change | |
100 | * in the interest of encouraging the widest possible deployment of | |
101 | * the technology. The Harvest software is really a reference | |
102 | * implementation of a set of protocols and formats, some of which | |
103 | * we intend to standardize. We encourage commercial | |
104 | * re-implementations of code complying to this set of standards. | |
c943f331 | 105 | */ |
090089c4 | 106 | |
44a47c6e | 107 | #include "squid.h" /* goes first */ |
090089c4 | 108 | |
109 | #define REBUILD_TIMESTAMP_DELTA_MAX 2 | |
090089c4 | 110 | #define SWAP_BUF DISK_PAGE_SIZE |
8350fe9b | 111 | #define VM_WINDOW_SZ DISK_PAGE_SIZE |
090089c4 | 112 | |
227fbb74 | 113 | #define WITH_MEMOBJ 1 |
114 | #define WITHOUT_MEMOBJ 0 | |
115 | ||
fcf5283d | 116 | #define STORE_IN_MEM_BUCKETS (229) |
090089c4 | 117 | |
2285407f | 118 | #define STORE_LOG_CREATE 0 |
119 | #define STORE_LOG_SWAPIN 1 | |
120 | #define STORE_LOG_SWAPOUT 2 | |
121 | #define STORE_LOG_RELEASE 3 | |
122 | ||
147d3115 | 123 | static char *storeLogTags[] = |
124 | { | |
125 | "CREATE", | |
126 | "SWAPIN", | |
127 | "SWAPOUT", | |
128 | "RELEASE" | |
2285407f | 129 | }; |
130 | ||
0ee4272b | 131 | const char *memStatusStr[] = |
e62d2dea | 132 | { |
9dfb6c1c | 133 | "NOT_IN_MEMORY", |
9dfb6c1c | 134 | "IN_MEMORY" |
135 | }; | |
136 | ||
0ee4272b | 137 | const char *pingStatusStr[] = |
e62d2dea | 138 | { |
f17936ab | 139 | "PING_NONE", |
9dfb6c1c | 140 | "PING_WAITING", |
141 | "PING_TIMEOUT", | |
f17936ab | 142 | "PING_DONE" |
9dfb6c1c | 143 | }; |
144 | ||
0ee4272b | 145 | const char *storeStatusStr[] = |
e62d2dea | 146 | { |
9dfb6c1c | 147 | "STORE_OK", |
148 | "STORE_PENDING", | |
149 | "STORE_ABORTED" | |
150 | }; | |
151 | ||
0ee4272b | 152 | const char *swapStatusStr[] = |
e62d2dea | 153 | { |
8350fe9b | 154 | "SWAPOUT_NONE", |
155 | "SWAPOUT_OPENING", | |
156 | "SWAPOUT_WRITING", | |
157 | "SWAPOUT_DONE" | |
9dfb6c1c | 158 | }; |
159 | ||
5608850b | 160 | struct storeRebuildState { |
161 | struct _rebuild_dir { | |
162 | int dirn; | |
3e347513 | 163 | FILE *log; |
5608850b | 164 | int speed; |
165 | int clean; | |
166 | struct _rebuild_dir *next; | |
3e347513 | 167 | } *rebuild_dir; |
3facc31e | 168 | int objcount; /* # objects successfully reloaded */ |
169 | int expcount; /* # objects expired */ | |
170 | int linecount; /* # lines parsed from cache logfile */ | |
171 | int clashcount; /* # swapfile clashes avoided */ | |
172 | int dupcount; /* # duplicates purged */ | |
994bbf93 | 173 | int invalid; /* # bad lines */ |
a47b9029 | 174 | int badflags; /* # bad e->flags */ |
f182d1c5 | 175 | int need_to_validate; |
5608850b | 176 | time_t start; |
177 | time_t stop; | |
178 | char *line_in; | |
179 | size_t line_in_sz; | |
3facc31e | 180 | }; |
181 | ||
d3c12084 | 182 | struct _bucketOrder { |
0c77d853 | 183 | unsigned int bucket; |
184 | int index; | |
d3c12084 | 185 | }; |
186 | ||
0a0bf5db | 187 | typedef struct storeCleanList { |
188 | char *key; | |
189 | struct storeCleanList *next; | |
190 | } storeCleanList; | |
191 | ||
ea3a2a69 | 192 | typedef void (VCB) (void *); |
0a0bf5db | 193 | |
194 | typedef struct valid_ctrl_t { | |
195 | struct stat *sb; | |
196 | StoreEntry *e; | |
d7b78fbb | 197 | VCB *callback; |
0a0bf5db | 198 | void *callback_data; |
199 | } valid_ctrl_t; | |
200 | ||
201 | typedef struct swapin_ctrl_t { | |
202 | StoreEntry *e; | |
203 | char *path; | |
582b6456 | 204 | SIH *callback; |
0a0bf5db | 205 | void *callback_data; |
43329771 | 206 | store_client *sc; |
0a0bf5db | 207 | } swapin_ctrl_t; |
208 | ||
209 | typedef struct lock_ctrl_t { | |
582b6456 | 210 | SIH *callback; |
0a0bf5db | 211 | void *callback_data; |
212 | StoreEntry *e; | |
213 | } lock_ctrl_t; | |
214 | ||
215 | typedef struct swapout_ctrl_t { | |
216 | char *swapfilename; | |
217 | int oldswapstatus; | |
218 | StoreEntry *e; | |
0a0bf5db | 219 | } swapout_ctrl_t; |
220 | ||
30a4f2a8 | 221 | /* Static Functions */ |
f5b8bbc4 | 222 | static int compareLastRef(StoreEntry **, StoreEntry **); |
223 | static int compareBucketOrder(struct _bucketOrder *, struct _bucketOrder *); | |
224 | static int storeCheckExpired(const StoreEntry *, int flag); | |
225 | static store_client *storeClientListSearch(const MemObject *, void *); | |
f5b8bbc4 | 226 | static int storeEntryLocked(const StoreEntry *); |
227 | static int storeEntryValidLength(const StoreEntry *); | |
228 | static void storeGetMemSpace(int); | |
b93bcace | 229 | static void storeHashDelete(StoreEntry *); |
d7b78fbb | 230 | static VCB storeSwapInValidateComplete; |
f5b8bbc4 | 231 | static mem_hdr *new_MemObjectData(void); |
232 | static MemObject *new_MemObject(const char *); | |
233 | static StoreEntry *new_StoreEntry(int, const char *); | |
234 | static StoreEntry *storeAddDiskRestore(const char *, | |
ea3a2a69 | 235 | int, |
236 | int, | |
237 | time_t, | |
238 | time_t, | |
239 | time_t, | |
240 | time_t, | |
241 | u_num32, | |
242 | u_num32, | |
243 | int); | |
f5b8bbc4 | 244 | static unsigned int storeGetBucketNum(void); |
245 | static void destroy_MemObject(MemObject *); | |
246 | static void destroy_MemObjectData(MemObject *); | |
247 | static void destroy_StoreEntry(StoreEntry *); | |
248 | static void storePurgeMem(StoreEntry *); | |
249 | static void storeStartRebuildFromDisk(void); | |
250 | static void storeSwapOutStart(StoreEntry * e); | |
d89d1fb6 | 251 | static DWCB storeSwapOutHandle; |
f5b8bbc4 | 252 | static void storeSetPrivateKey(StoreEntry *); |
d7b78fbb | 253 | static EVH storeDoRebuildFromDisk; |
254 | static EVH storeCleanup; | |
255 | static VCB storeCleanupComplete; | |
f5b8bbc4 | 256 | static void storeValidate(StoreEntry *, VCB *, void *); |
d7b78fbb | 257 | static AIOCB storeValidateComplete; |
ea3a2a69 | 258 | static void storeRebuiltFromDisk(struct storeRebuildState *data); |
f5b8bbc4 | 259 | static unsigned int getKeyCounter(void); |
260 | static void storePutUnusedFileno(int fileno); | |
261 | static int storeGetUnusedFileno(void); | |
8350fe9b | 262 | static void storeCheckSwapOut(StoreEntry * e); |
263 | static void storeSwapoutFileOpened(void *data, int fd); | |
264 | static int storeCheckCachable(StoreEntry * e); | |
265 | static int storeKeepInMemory(const StoreEntry *); | |
266 | static SIH storeClientCopyFileOpened; | |
267 | static DRCB storeClientCopyHandleRead; | |
268 | static FOCB storeSwapInFileOpened; | |
5d86029a | 269 | static void storeClientCopyFileRead(store_client * sc); |
b93bcace | 270 | static void storeEntryListAdd(StoreEntry * e, dlink_node *, dlink_list *); |
271 | static void storeEntryListDelete(dlink_node *, dlink_list *); | |
8350fe9b | 272 | static void storeSetMemStatus(StoreEntry * e, int); |
dbfda5cb | 273 | static void storeClientCopy2(StoreEntry *, store_client *); |
b93bcace | 274 | static void storeHashInsert(StoreEntry * e); |
6e86c3e8 | 275 | static void storeSwapOutFileClose(StoreEntry * e); |
8350fe9b | 276 | |
090089c4 | 277 | /* Now, this table is inaccessible to outsider. They have to use a method |
278 | * to access a value in internal storage data structure. */ | |
35ac11a3 | 279 | static hash_table *store_table = NULL; |
b93bcace | 280 | static dlink_list inmem_list; |
281 | static dlink_list all_list; | |
090089c4 | 282 | |
13c72789 | 283 | static int store_pages_max = 0; |
e954773d | 284 | static int store_pages_high = 0; |
285 | static int store_pages_low = 0; | |
090089c4 | 286 | |
090089c4 | 287 | /* current file name, swap file, use number as a filename */ |
58104eab | 288 | static int store_swap_high = 0; |
289 | static int store_swap_low = 0; | |
2285407f | 290 | static int storelog_fd = -1; |
090089c4 | 291 | |
090089c4 | 292 | /* key temp buffer */ |
ec74cf5f | 293 | static char key_temp_buffer[MAX_URL + 100]; |
090089c4 | 294 | |
66cedb85 | 295 | /* expiration parameters and stats */ |
296 | static int store_buckets; | |
e924600d | 297 | static int store_maintain_rate; |
66cedb85 | 298 | static int store_maintain_buckets; |
e924600d | 299 | static int scan_revolutions; |
d3c12084 | 300 | static struct _bucketOrder *MaintBucketsOrder = NULL; |
66cedb85 | 301 | |
b8d8561b | 302 | static MemObject * |
88738790 | 303 | new_MemObject(const char *log_url) |
227fbb74 | 304 | { |
30a4f2a8 | 305 | MemObject *mem = get_free_mem_obj(); |
306 | mem->reply = xcalloc(1, sizeof(struct _http_reply)); | |
33b589ff | 307 | mem->reply->date = -2; |
308 | mem->reply->expires = -2; | |
309 | mem->reply->last_modified = -2; | |
88738790 | 310 | mem->log_url = xstrdup(log_url); |
8350fe9b | 311 | mem->swapout.fd = -1; |
579a45cd | 312 | meta_data.mem_obj_count++; |
30a4f2a8 | 313 | meta_data.misc += sizeof(struct _http_reply); |
88738790 | 314 | meta_data.url_strings += strlen(log_url); |
a3d5953d | 315 | debug(20, 3) ("new_MemObject: returning %p\n", mem); |
30a4f2a8 | 316 | return mem; |
227fbb74 | 317 | } |
318 | ||
b8d8561b | 319 | static StoreEntry * |
88738790 | 320 | new_StoreEntry(int mem_obj_flag, const char *log_url) |
090089c4 | 321 | { |
322 | StoreEntry *e = NULL; | |
323 | ||
30a4f2a8 | 324 | e = xcalloc(1, sizeof(StoreEntry)); |
227fbb74 | 325 | meta_data.store_entries++; |
326 | if (mem_obj_flag) | |
88738790 | 327 | e->mem_obj = new_MemObject(log_url); |
a3d5953d | 328 | debug(20, 3) ("new_StoreEntry: returning %p\n", e); |
227fbb74 | 329 | return e; |
090089c4 | 330 | } |
331 | ||
b8d8561b | 332 | static void |
333 | destroy_MemObject(MemObject * mem) | |
090089c4 | 334 | { |
a3d5953d | 335 | debug(20, 3) ("destroy_MemObject: destroying %p\n", mem); |
645afd3f | 336 | assert(mem->swapout.fd == -1); |
30a4f2a8 | 337 | destroy_MemObjectData(mem); |
88738790 | 338 | meta_data.url_strings -= strlen(mem->log_url); |
52d4522b | 339 | safe_free(mem->clients); |
30a4f2a8 | 340 | safe_free(mem->reply); |
88738790 | 341 | safe_free(mem->log_url); |
30a4f2a8 | 342 | requestUnlink(mem->request); |
343 | mem->request = NULL; | |
344 | put_free_mem_obj(mem); | |
579a45cd | 345 | meta_data.mem_obj_count--; |
30a4f2a8 | 346 | meta_data.misc -= sizeof(struct _http_reply); |
090089c4 | 347 | } |
348 | ||
b8d8561b | 349 | static void |
350 | destroy_StoreEntry(StoreEntry * e) | |
090089c4 | 351 | { |
a3d5953d | 352 | debug(20, 3) ("destroy_StoreEntry: destroying %p\n", e); |
9e975e4e | 353 | assert(e != NULL); |
227fbb74 | 354 | if (e->mem_obj) |
355 | destroy_MemObject(e->mem_obj); | |
30a4f2a8 | 356 | if (e->url) { |
357 | meta_data.url_strings -= strlen(e->url); | |
358 | safe_free(e->url); | |
359 | } else { | |
a3d5953d | 360 | debug(20, 3) ("destroy_StoreEntry: WARNING: Entry without URL string!\n"); |
30a4f2a8 | 361 | } |
362 | if (BIT_TEST(e->flag, KEY_URL)) | |
363 | e->key = NULL; | |
364 | else | |
365 | safe_free(e->key); | |
227fbb74 | 366 | xfree(e); |
367 | meta_data.store_entries--; | |
090089c4 | 368 | } |
369 | ||
f1dc9b30 | 370 | static mem_hdr * |
b8d8561b | 371 | new_MemObjectData(void) |
227fbb74 | 372 | { |
a3d5953d | 373 | debug(20, 3) ("new_MemObjectData: calling memInit()\n"); |
579a45cd | 374 | meta_data.mem_data_count++; |
227fbb74 | 375 | return memInit(); |
376 | } | |
090089c4 | 377 | |
b8d8561b | 378 | static void |
379 | destroy_MemObjectData(MemObject * mem) | |
090089c4 | 380 | { |
a3d5953d | 381 | debug(20, 3) ("destroy_MemObjectData: destroying %p, %d bytes\n", |
8350fe9b | 382 | mem->data, mem->inmem_hi); |
30a4f2a8 | 383 | if (mem->data) { |
d7b78fbb | 384 | memFree(mem->data); |
30a4f2a8 | 385 | mem->data = NULL; |
579a45cd | 386 | meta_data.mem_data_count--; |
090089c4 | 387 | } |
8350fe9b | 388 | mem->inmem_hi = 0; |
090089c4 | 389 | } |
390 | ||
090089c4 | 391 | /* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */ |
392 | ||
76dd1215 | 393 | static void |
b8d8561b | 394 | storeHashInsert(StoreEntry * e) |
090089c4 | 395 | { |
a3d5953d | 396 | debug(20, 3) ("storeHashInsert: Inserting Entry %p key '%s'\n", |
227fbb74 | 397 | e, e->key); |
b93bcace | 398 | hash_join(store_table, (hash_link *) e); |
399 | storeEntryListAdd(e, &e->lru, &all_list); | |
090089c4 | 400 | } |
401 | ||
b93bcace | 402 | static void |
b8d8561b | 403 | storeHashDelete(StoreEntry * e) |
090089c4 | 404 | { |
b93bcace | 405 | hash_remove_link(store_table, (hash_link *) e); |
406 | storeEntryListDelete(&e->lru, &all_list); | |
090089c4 | 407 | } |
408 | ||
090089c4 | 409 | /* -------------------------------------------------------------------------- */ |
410 | ||
b8d8561b | 411 | static void |
fe4e214f | 412 | storeLog(int tag, const StoreEntry * e) |
2285407f | 413 | { |
0be3ec60 | 414 | LOCAL_ARRAY(char, logmsg, MAX_URL << 1); |
ca98227c | 415 | MemObject *mem = e->mem_obj; |
416 | struct _http_reply *reply; | |
2285407f | 417 | if (storelog_fd < 0) |
418 | return; | |
ca98227c | 419 | if (mem == NULL) |
420 | return; | |
88738790 | 421 | if (mem->log_url == NULL) { |
422 | debug(20, 1) ("storeLog: NULL log_url for %s\n", e->url); | |
423 | storeMemObjectDump(mem); | |
424 | mem->log_url = xstrdup(e->url); | |
425 | } | |
ca98227c | 426 | reply = mem->reply; |
8350fe9b | 427 | snprintf(logmsg, MAX_URL << 1, "%9d.%03d %-7s %08X %4d %9d %9d %9d %s %d/%d %s %s\n", |
2285407f | 428 | (int) current_time.tv_sec, |
429 | (int) current_time.tv_usec / 1000, | |
430 | storeLogTags[tag], | |
2abe4822 | 431 | e->swap_file_number, |
ca98227c | 432 | reply->code, |
433 | (int) reply->date, | |
434 | (int) reply->last_modified, | |
435 | (int) reply->expires, | |
436 | reply->content_type[0] ? reply->content_type : "unknown", | |
437 | reply->content_length, | |
711982d8 | 438 | (int) (mem->inmem_hi - mem->reply->hdr_sz), |
ca98227c | 439 | RequestMethodStr[e->method], |
88738790 | 440 | mem->log_url); |
2285407f | 441 | file_write(storelog_fd, |
442 | xstrdup(logmsg), | |
443 | strlen(logmsg), | |
2285407f | 444 | NULL, |
e47afcdb | 445 | NULL, |
446 | xfree); | |
2285407f | 447 | } |
448 | ||
090089c4 | 449 | |
450 | /* get rid of memory copy of the object */ | |
620da955 | 451 | /* Only call this if storeCheckPurgeMem(e) returns 1 */ |
24382924 | 452 | static void |
b8d8561b | 453 | storePurgeMem(StoreEntry * e) |
090089c4 | 454 | { |
2aca8433 | 455 | if (e->mem_obj == NULL) |
090089c4 | 456 | return; |
8350fe9b | 457 | debug(20, 3) ("storePurgeMem: Freeing memory-copy of %s\n", e->key); |
090089c4 | 458 | storeSetMemStatus(e, NOT_IN_MEMORY); |
227fbb74 | 459 | destroy_MemObject(e->mem_obj); |
460 | e->mem_obj = NULL; | |
06e8899b | 461 | if (e->swap_status != SWAPOUT_DONE) |
8272aded | 462 | storeRelease(e); |
090089c4 | 463 | } |
464 | ||
0a0bf5db | 465 | void |
95c4b18f | 466 | storeLockObject(StoreEntry * e) |
090089c4 | 467 | { |
b93bcace | 468 | if (e->lock_count++ == 0) { |
469 | storeEntryListDelete(&e->lru, &all_list); | |
470 | storeEntryListAdd(e, &e->lru, &all_list); | |
471 | } | |
a3d5953d | 472 | debug(20, 3) ("storeLockObject: key '%s' count=%d\n", |
148a2921 | 473 | e->key, (int) e->lock_count); |
b8de7ebe | 474 | e->lastref = squid_curtime; |
090089c4 | 475 | } |
476 | ||
b8d8561b | 477 | void |
478 | storeReleaseRequest(StoreEntry * e) | |
2285407f | 479 | { |
aadcfbad | 480 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
58bcd31b | 481 | return; |
9e4ad609 | 482 | if (!storeEntryLocked(e)) |
483 | fatal_dump("storeReleaseRequest: unlocked entry"); | |
86101e40 | 484 | debug(20, 3) ("storeReleaseRequest: '%s'\n", |
485 | e->key ? e->key : e->url); | |
9e4ad609 | 486 | BIT_SET(e->flag, RELEASE_REQUEST); |
fe54d06d | 487 | storeSetPrivateKey(e); |
2285407f | 488 | } |
489 | ||
090089c4 | 490 | /* unlock object, return -1 if object get released after unlock |
491 | * otherwise lock_count */ | |
b8d8561b | 492 | int |
493 | storeUnlockObject(StoreEntry * e) | |
090089c4 | 494 | { |
6c895381 | 495 | e->lock_count--; |
a3d5953d | 496 | debug(20, 3) ("storeUnlockObject: key '%s' count=%d\n", |
6c895381 | 497 | e->key, e->lock_count); |
30a4f2a8 | 498 | if (e->lock_count) |
a1e47288 | 499 | return (int) e->lock_count; |
5b00be7a | 500 | if (e->store_status == STORE_PENDING) { |
f182d1c5 | 501 | assert(!BIT_TEST(e->flag, ENTRY_DISPATCHED)); |
502 | BIT_SET(e->flag, RELEASE_REQUEST); | |
5b00be7a | 503 | } |
9e975e4e | 504 | assert(storePendingNClients(e) == 0); |
8350fe9b | 505 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
30a4f2a8 | 506 | storeRelease(e); |
8350fe9b | 507 | else if (storeKeepInMemory(e)) { |
508 | storeSetMemStatus(e, IN_MEMORY); | |
509 | requestUnlink(e->mem_obj->request); | |
510 | e->mem_obj->request = NULL; | |
511 | } else | |
30a4f2a8 | 512 | storePurgeMem(e); |
6c895381 | 513 | return 0; |
090089c4 | 514 | } |
515 | ||
516 | /* Lookup an object in the cache. | |
517 | * return just a reference to object, don't start swapping in yet. */ | |
b8d8561b | 518 | StoreEntry * |
0ee4272b | 519 | storeGet(const char *url) |
090089c4 | 520 | { |
a3d5953d | 521 | debug(20, 3) ("storeGet: looking up %s\n", url); |
b15fe823 | 522 | return (StoreEntry *) hash_lookup(store_table, url); |
090089c4 | 523 | } |
524 | ||
88738790 | 525 | static unsigned int |
b8d8561b | 526 | getKeyCounter(void) |
04e8dbaa | 527 | { |
2d25dc1a | 528 | static unsigned int key_counter = 0; |
7cb386d0 | 529 | if (++key_counter == (1 << 24)) |
2cc3f720 | 530 | key_counter = 1; |
2d25dc1a | 531 | return key_counter; |
04e8dbaa | 532 | } |
533 | ||
2cc3f720 | 534 | unsigned int |
535 | storeReqnum(StoreEntry * entry, method_t method) | |
536 | { | |
69951dbc | 537 | unsigned int k; |
2cc3f720 | 538 | if (BIT_TEST(entry->flag, KEY_PRIVATE)) |
69951dbc | 539 | k = atoi(entry->key); |
540 | else | |
541 | k = getKeyCounter(); | |
2cc3f720 | 542 | if (method == METHOD_GET) |
69951dbc | 543 | return k; |
544 | return (method << 24) | k; | |
2cc3f720 | 545 | } |
546 | ||
0ee4272b | 547 | const char * |
548 | storeGeneratePrivateKey(const char *url, method_t method, int num) | |
6eb42cae | 549 | { |
6eb42cae | 550 | if (num == 0) |
04e8dbaa | 551 | num = getKeyCounter(); |
2cc3f720 | 552 | else if (num & 0xFF000000) { |
553 | method = (method_t) (num >> 24); | |
554 | num &= 0x00FFFFFF; | |
555 | } | |
a3d5953d | 556 | debug(20, 3) ("storeGeneratePrivateKey: '%s'\n", url); |
6eb42cae | 557 | key_temp_buffer[0] = '\0'; |
8350fe9b | 558 | snprintf(key_temp_buffer, MAX_URL + 100, "%d/%s/%s", |
6eb42cae | 559 | num, |
73ddce89 | 560 | RequestMethodStr[method], |
6eb42cae | 561 | url); |
562 | return key_temp_buffer; | |
563 | } | |
564 | ||
0ee4272b | 565 | const char * |
566 | storeGeneratePublicKey(const char *url, method_t method) | |
090089c4 | 567 | { |
a3d5953d | 568 | debug(20, 3) ("storeGeneratePublicKey: type=%d %s\n", method, url); |
73ddce89 | 569 | switch (method) { |
6eb42cae | 570 | case METHOD_GET: |
227fbb74 | 571 | return url; |
983061ed | 572 | /* NOTREACHED */ |
227fbb74 | 573 | break; |
6eb42cae | 574 | case METHOD_POST: |
30a4f2a8 | 575 | case METHOD_PUT: |
6eb42cae | 576 | case METHOD_HEAD: |
3facc31e | 577 | case METHOD_CONNECT: |
b3b64e58 | 578 | case METHOD_TRACE: |
8350fe9b | 579 | snprintf(key_temp_buffer, MAX_URL + 100, "/%s/%s", RequestMethodStr[method], url); |
b3b64e58 | 580 | return key_temp_buffer; |
581 | /* NOTREACHED */ | |
582 | break; | |
227fbb74 | 583 | default: |
9e975e4e | 584 | fatal_dump("storeGeneratePublicKey: Unsupported request method"); |
227fbb74 | 585 | break; |
586 | } | |
587 | return NULL; | |
588 | } | |
589 | ||
fe54d06d | 590 | static void |
b8d8561b | 591 | storeSetPrivateKey(StoreEntry * e) |
227fbb74 | 592 | { |
0ee4272b | 593 | const char *newkey = NULL; |
6eb42cae | 594 | if (e->key && BIT_TEST(e->flag, KEY_PRIVATE)) |
595 | return; /* is already private */ | |
73ddce89 | 596 | newkey = storeGeneratePrivateKey(e->url, e->method, 0); |
9e975e4e | 597 | assert(hash_lookup(store_table, newkey) == NULL); |
07622ccd | 598 | if (e->key) |
edae7bc0 | 599 | storeHashDelete(e); |
07622ccd | 600 | if (e->key && !BIT_TEST(e->flag, KEY_URL)) |
601 | safe_free(e->key); | |
602 | e->key = xstrdup(newkey); | |
6eb42cae | 603 | storeHashInsert(e); |
227fbb74 | 604 | BIT_RESET(e->flag, KEY_URL); |
6eb42cae | 605 | BIT_SET(e->flag, KEY_CHANGE); |
606 | BIT_SET(e->flag, KEY_PRIVATE); | |
227fbb74 | 607 | } |
608 | ||
b8d8561b | 609 | void |
610 | storeSetPublicKey(StoreEntry * e) | |
227fbb74 | 611 | { |
6eb42cae | 612 | StoreEntry *e2 = NULL; |
613 | hash_link *table_entry = NULL; | |
0ee4272b | 614 | const char *newkey = NULL; |
234967c9 | 615 | int loop_detect = 0; |
6eb42cae | 616 | if (e->key && !BIT_TEST(e->flag, KEY_PRIVATE)) |
617 | return; /* is already public */ | |
73ddce89 | 618 | newkey = storeGeneratePublicKey(e->url, e->method); |
ec74cf5f | 619 | while ((table_entry = hash_lookup(store_table, newkey))) { |
89e079ed | 620 | debug(20, 3) ("storeSetPublicKey: Making old '%s' private.\n", newkey); |
07622ccd | 621 | e2 = (StoreEntry *) table_entry; |
622 | storeSetPrivateKey(e2); | |
30a4f2a8 | 623 | storeRelease(e2); |
9e975e4e | 624 | assert(++loop_detect < 10); |
234967c9 | 625 | newkey = storeGeneratePublicKey(e->url, e->method); |
6eb42cae | 626 | } |
07622ccd | 627 | if (e->key) |
edae7bc0 | 628 | storeHashDelete(e); |
07622ccd | 629 | if (e->key && !BIT_TEST(e->flag, KEY_URL)) |
630 | safe_free(e->key); | |
73ddce89 | 631 | if (e->method == METHOD_GET) { |
6eb42cae | 632 | e->key = e->url; |
633 | BIT_SET(e->flag, KEY_URL); | |
634 | BIT_RESET(e->flag, KEY_CHANGE); | |
635 | } else { | |
07622ccd | 636 | e->key = xstrdup(newkey); |
6eb42cae | 637 | BIT_RESET(e->flag, KEY_URL); |
638 | BIT_SET(e->flag, KEY_CHANGE); | |
227fbb74 | 639 | } |
07622ccd | 640 | BIT_RESET(e->flag, KEY_PRIVATE); |
227fbb74 | 641 | storeHashInsert(e); |
642 | } | |
643 | ||
b8d8561b | 644 | StoreEntry * |
88738790 | 645 | storeCreateEntry(const char *url, const char *log_url, int flags, method_t method) |
090089c4 | 646 | { |
090089c4 | 647 | StoreEntry *e = NULL; |
30a4f2a8 | 648 | MemObject *mem = NULL; |
a3d5953d | 649 | debug(20, 3) ("storeCreateEntry: '%s' icp flags=%x\n", url, flags); |
090089c4 | 650 | |
88738790 | 651 | e = new_StoreEntry(WITH_MEMOBJ, log_url); |
30a4f2a8 | 652 | e->lock_count = 1; /* Note lock here w/o calling storeLock() */ |
653 | mem = e->mem_obj; | |
090089c4 | 654 | e->url = xstrdup(url); |
6eb42cae | 655 | meta_data.url_strings += strlen(url); |
73ddce89 | 656 | e->method = method; |
86101e40 | 657 | if (neighbors_do_private_keys || !BIT_TEST(flags, REQ_HIERARCHICAL)) |
658 | storeSetPrivateKey(e); | |
659 | else | |
660 | storeSetPublicKey(e); | |
234967c9 | 661 | if (BIT_TEST(flags, REQ_CACHABLE)) { |
1c481e00 | 662 | BIT_SET(e->flag, ENTRY_CACHABLE); |
090089c4 | 663 | BIT_RESET(e->flag, RELEASE_REQUEST); |
664 | } else { | |
1c481e00 | 665 | BIT_RESET(e->flag, ENTRY_CACHABLE); |
2daae136 | 666 | storeReleaseRequest(e); |
090089c4 | 667 | } |
234967c9 | 668 | if (BIT_TEST(flags, REQ_HIERARCHICAL)) |
669 | BIT_SET(e->flag, HIERARCHICAL); | |
670 | else | |
671 | BIT_RESET(e->flag, HIERARCHICAL); | |
234967c9 | 672 | e->store_status = STORE_PENDING; |
090089c4 | 673 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 674 | e->swap_status = SWAPOUT_NONE; |
090089c4 | 675 | e->swap_file_number = -1; |
30a4f2a8 | 676 | mem->data = new_MemObjectData(); |
090089c4 | 677 | e->refcount = 0; |
b8de7ebe | 678 | e->lastref = squid_curtime; |
ca98227c | 679 | e->timestamp = 0; /* set in storeTimestampsSet() */ |
30a4f2a8 | 680 | e->ping_status = PING_NONE; |
0a0bf5db | 681 | BIT_SET(e->flag, ENTRY_VALIDATED); |
090089c4 | 682 | return e; |
683 | } | |
684 | ||
685 | /* Add a new object to the cache with empty memory copy and pointer to disk | |
686 | * use to rebuild store from disk. */ | |
24382924 | 687 | static StoreEntry * |
919877b6 | 688 | storeAddDiskRestore(const char *url, |
23ff6968 | 689 | int file_number, |
690 | int size, | |
691 | time_t expires, | |
692 | time_t timestamp, | |
693 | time_t lastref, | |
694 | time_t lastmod, | |
695 | u_num32 refcount, | |
696 | u_num32 flags, | |
697 | int clean) | |
090089c4 | 698 | { |
699 | StoreEntry *e = NULL; | |
a47b9029 | 700 | debug(20, 5) ("StoreAddDiskRestore: %s, fileno=%08X\n", url, file_number); |
3facc31e | 701 | /* if you call this you'd better be sure file_number is not |
702 | * already in use! */ | |
090089c4 | 703 | meta_data.url_strings += strlen(url); |
88738790 | 704 | e = new_StoreEntry(WITHOUT_MEMOBJ, NULL); |
090089c4 | 705 | e->url = xstrdup(url); |
73ddce89 | 706 | e->method = METHOD_GET; |
6eb42cae | 707 | storeSetPublicKey(e); |
89e079ed | 708 | assert(e->key); |
234967c9 | 709 | e->store_status = STORE_OK; |
090089c4 | 710 | storeSetMemStatus(e, NOT_IN_MEMORY); |
8350fe9b | 711 | e->swap_status = SWAPOUT_DONE; |
090089c4 | 712 | e->swap_file_number = file_number; |
090089c4 | 713 | e->object_len = size; |
714 | e->lock_count = 0; | |
090089c4 | 715 | e->refcount = 0; |
919877b6 | 716 | e->lastref = lastref; |
f5469bce | 717 | e->timestamp = timestamp; |
718 | e->expires = expires; | |
719 | e->lastmod = lastmod; | |
c54e9052 | 720 | e->refcount = refcount; |
721 | e->flag = flags; | |
89e079ed | 722 | BIT_SET(e->flag, ENTRY_CACHABLE); |
723 | BIT_RESET(e->flag, RELEASE_REQUEST); | |
724 | BIT_RESET(e->flag, KEY_PRIVATE); | |
30a4f2a8 | 725 | e->ping_status = PING_NONE; |
311ea387 | 726 | if (clean) { |
3e347513 | 727 | BIT_SET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 728 | /* Only set the file bit if we know its a valid entry */ |
729 | /* otherwise, set it in the validation procedure */ | |
730 | storeDirMapBitSet(file_number); | |
731 | } else { | |
3e347513 | 732 | BIT_RESET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 733 | } |
090089c4 | 734 | return e; |
735 | } | |
736 | ||
b8d8561b | 737 | int |
382d851a | 738 | storeUnregister(StoreEntry * e, void *data) |
090089c4 | 739 | { |
6e40f263 | 740 | MemObject *mem = e->mem_obj; |
43329771 | 741 | store_client *sc; |
742 | store_client **S; | |
38792624 | 743 | if (mem == NULL) |
744 | return 0; | |
a3d5953d | 745 | debug(20, 3) ("storeUnregister: called for '%s'\n", e->key); |
43329771 | 746 | for (S = &mem->clients; (sc = *S); S = &(*S)->next) { |
747 | if (sc->callback_data == data) | |
748 | break; | |
749 | } | |
8350fe9b | 750 | if (sc == NULL) |
52d4522b | 751 | return 0; |
43329771 | 752 | *S = sc->next; |
753 | mem->nclients--; | |
645afd3f | 754 | if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) |
6a54c60e | 755 | storeCheckSwapOut(e); |
43329771 | 756 | if (sc->swapin_fd > -1) { |
757 | commSetSelect(sc->swapin_fd, COMM_SELECT_READ, NULL, NULL, 0); | |
8350fe9b | 758 | file_close(sc->swapin_fd); |
43329771 | 759 | } |
76a5501f | 760 | if (sc->callback) |
761 | debug(20,1)("WARNING: store_client for %s has a callback\n", e->url); | |
43329771 | 762 | cbdataFree(sc); |
52d4522b | 763 | return 1; |
090089c4 | 764 | } |
765 | ||
8350fe9b | 766 | off_t |
767 | storeLowestMemReaderOffset(const StoreEntry * entry) | |
234967c9 | 768 | { |
0ee4272b | 769 | const MemObject *mem = entry->mem_obj; |
8350fe9b | 770 | off_t lowest = mem->inmem_hi; |
43329771 | 771 | store_client *sc; |
772 | store_client *nx = NULL; | |
5d86029a | 773 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 774 | nx = sc->next; |
8350fe9b | 775 | if (sc->callback_data == NULL) /* open slot */ |
234967c9 | 776 | continue; |
43329771 | 777 | if (sc->type != STORE_MEM_CLIENT) |
8350fe9b | 778 | continue; |
779 | if (sc->copy_offset < lowest) | |
780 | lowest = sc->copy_offset; | |
234967c9 | 781 | } |
782 | return lowest; | |
783 | } | |
784 | ||
090089c4 | 785 | /* Call handlers waiting for data to be appended to E. */ |
498d87ae | 786 | void |
dbfda5cb | 787 | InvokeHandlers(StoreEntry * e) |
090089c4 | 788 | { |
43329771 | 789 | int i = 0; |
9dfb6c1c | 790 | MemObject *mem = e->mem_obj; |
d89d1fb6 | 791 | STCB *callback = NULL; |
43329771 | 792 | store_client *sc; |
793 | store_client *nx = NULL; | |
9e975e4e | 794 | assert(mem->clients != NULL || mem->nclients == 0); |
a47b9029 | 795 | debug(20, 3) ("InvokeHandlers: %s\n", e->key); |
d89d1fb6 | 796 | /* walk the entire list looking for valid callbacks */ |
5d86029a | 797 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 798 | nx = sc->next; |
799 | debug(20, 3) ("InvokeHandlers: checking client #%d\n", i++); | |
382d851a | 800 | if (sc->callback_data == NULL) |
63cc9284 | 801 | continue; |
d89d1fb6 | 802 | if ((callback = sc->callback) == NULL) |
63cc9284 | 803 | continue; |
dbfda5cb | 804 | storeClientCopy2(e, sc); |
090089c4 | 805 | } |
090089c4 | 806 | } |
807 | ||
6eb42cae | 808 | /* Mark object as expired */ |
b8d8561b | 809 | void |
810 | storeExpireNow(StoreEntry * e) | |
9174e204 | 811 | { |
a3d5953d | 812 | debug(20, 3) ("storeExpireNow: '%s'\n", e->key); |
b8de7ebe | 813 | e->expires = squid_curtime; |
9174e204 | 814 | } |
815 | ||
8350fe9b | 816 | static void |
817 | storeSwapoutFileOpened(void *data, int fd) | |
090089c4 | 818 | { |
8350fe9b | 819 | swapout_ctrl_t *ctrlp = data; |
820 | int oldswapstatus = ctrlp->oldswapstatus; | |
821 | char *swapfilename = ctrlp->swapfilename; | |
822 | StoreEntry *e = ctrlp->e; | |
823 | MemObject *mem; | |
824 | xfree(ctrlp); | |
825 | assert(e->swap_status == SWAPOUT_OPENING); | |
826 | if (fd < 0) { | |
827 | debug(20, 0) ("storeSwapoutFileOpened: Unable to open swapfile: %s\n", | |
828 | swapfilename); | |
829 | storeDirMapBitReset(e->swap_file_number); | |
830 | e->swap_file_number = -1; | |
831 | e->swap_status = oldswapstatus; | |
832 | xfree(swapfilename); | |
090089c4 | 833 | return; |
8350fe9b | 834 | } |
835 | mem = e->mem_obj; | |
836 | mem->swapout.fd = (short) fd; | |
837 | e->swap_status = SWAPOUT_WRITING; | |
838 | debug(20, 5) ("storeSwapoutFileOpened: Begin SwapOut '%s' to FD %d FILE %s.\n", | |
839 | e->url, fd, swapfilename); | |
840 | xfree(swapfilename); | |
841 | debug(20, 5) ("swap_file_number=%08X\n", e->swap_file_number); | |
8350fe9b | 842 | storeCheckSwapOut(e); |
843 | } | |
844 | ||
845 | /* start swapping object to disk */ | |
846 | static void | |
847 | storeSwapOutStart(StoreEntry * e) | |
848 | { | |
849 | swapout_ctrl_t *ctrlp; | |
850 | LOCAL_ARRAY(char, swapfilename, SQUID_MAXPATHLEN); | |
851 | storeLockObject(e); | |
852 | if ((e->swap_file_number = storeGetUnusedFileno()) < 0) | |
853 | e->swap_file_number = storeDirMapAllocate(); | |
854 | storeSwapFullPath(e->swap_file_number, swapfilename); | |
855 | ctrlp = xmalloc(sizeof(swapout_ctrl_t)); | |
856 | ctrlp->swapfilename = xstrdup(swapfilename); | |
857 | ctrlp->e = e; | |
858 | ctrlp->oldswapstatus = e->swap_status; | |
859 | e->swap_status = SWAPOUT_OPENING; | |
860 | file_open(swapfilename, | |
861 | O_WRONLY | O_CREAT | O_TRUNC, | |
862 | storeSwapoutFileOpened, | |
863 | ctrlp); | |
864 | } | |
865 | ||
866 | static void | |
867 | storeSwapOutHandle(int fd, int flag, size_t len, void *data) | |
868 | { | |
869 | StoreEntry *e = data; | |
870 | MemObject *mem = e->mem_obj; | |
871 | debug(20, 3) ("storeSwapOutHandle: '%s', len=%d\n", e->key, (int) len); | |
872 | assert(mem != NULL); | |
873 | if (flag < 0) { | |
874 | debug(20, 1) ("storeSwapOutHandle: SwapOut failure (err code = %d).\n", | |
875 | flag); | |
876 | e->swap_status = SWAPOUT_NONE; | |
8350fe9b | 877 | if (e->swap_file_number != -1) { |
878 | storePutUnusedFileno(e->swap_file_number); | |
879 | e->swap_file_number = -1; | |
880 | } | |
8350fe9b | 881 | if (flag == DISK_NO_SPACE_LEFT) { |
882 | /* reduce the swap_size limit to the current size. */ | |
883 | Config.Swap.maxSize = store_swap_size; | |
884 | storeConfigure(); | |
885 | } | |
6e86c3e8 | 886 | storeReleaseRequest(e); |
887 | storeSwapOutFileClose(e); | |
8350fe9b | 888 | return; |
889 | } | |
8c123b71 | 890 | mem->swapout.done_offset += len; |
891 | if (e->store_status == STORE_PENDING || mem->swapout.done_offset < e->object_len) { | |
8350fe9b | 892 | storeCheckSwapOut(e); |
893 | return; | |
894 | } | |
895 | /* swapping complete */ | |
8350fe9b | 896 | debug(20, 5) ("storeSwapOutHandle: SwapOut complete: '%s' to %s.\n", |
897 | e->url, storeSwapFullPath(e->swap_file_number, NULL)); | |
e6e887e9 | 898 | if (!storeCheckCachable(e)) { |
899 | storeReleaseRequest(e); | |
900 | } else { | |
6a54c60e | 901 | e->swap_status = SWAPOUT_DONE; |
43fee0b8 | 902 | storeDirUpdateSwapSize(e->swap_file_number, mem->swapout.done_offset, 1); |
6a54c60e | 903 | storeLog(STORE_LOG_SWAPOUT, e); |
904 | storeDirSwapLog(e); | |
e6e887e9 | 905 | } |
8350fe9b | 906 | HTTPCacheInfo->proto_newobject(HTTPCacheInfo, |
907 | mem->request->protocol, | |
908 | e->object_len, | |
909 | FALSE); | |
6e86c3e8 | 910 | storeSwapOutFileClose(e); |
8350fe9b | 911 | } |
912 | ||
913 | static void | |
914 | storeCheckSwapOut(StoreEntry * e) | |
915 | { | |
916 | MemObject *mem = e->mem_obj; | |
8350fe9b | 917 | off_t lowest_offset; |
8c123b71 | 918 | off_t new_mem_lo; |
919 | size_t swapout_size; | |
c6ac1aae | 920 | char *swap_buf; |
921 | size_t swap_buf_len; | |
dbfda5cb | 922 | int x; |
8350fe9b | 923 | assert(mem != NULL); |
924 | /* should we swap something out to disk? */ | |
925 | debug(20, 3) ("storeCheckSwapOut: %s\n", e->url); | |
926 | debug(20, 3) ("storeCheckSwapOut: store_status = %s\n", | |
927 | storeStatusStr[e->store_status]); | |
6e86c3e8 | 928 | if (e->store_status == STORE_ABORTED) { |
929 | assert(BIT_TEST(e->flag, RELEASE_REQUEST)); | |
930 | storeSwapOutFileClose(e); | |
8350fe9b | 931 | return; |
6e86c3e8 | 932 | } |
8c123b71 | 933 | debug(20, 3) ("storeCheckSwapOut: mem->inmem_lo = %d\n", |
934 | (int) mem->inmem_lo); | |
935 | debug(20, 3) ("storeCheckSwapOut: mem->inmem_hi = %d\n", | |
936 | (int) mem->inmem_hi); | |
c6ac1aae | 937 | debug(20, 3) ("storeCheckSwapOut: swapout.queue_offset = %d\n", |
938 | (int) mem->swapout.queue_offset); | |
939 | debug(20, 3) ("storeCheckSwapOut: swapout.done_offset = %d\n", | |
940 | (int) mem->swapout.done_offset); | |
8c123b71 | 941 | assert(mem->inmem_hi >= mem->swapout.queue_offset); |
942 | swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); | |
8350fe9b | 943 | lowest_offset = storeLowestMemReaderOffset(e); |
ea0dd42b | 944 | debug(20, 3) ("storeCheckSwapOut: lowest_offset = %d\n", |
945 | (int) lowest_offset); | |
8c123b71 | 946 | assert(lowest_offset >= mem->inmem_lo); |
947 | ||
948 | new_mem_lo = lowest_offset; | |
949 | if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) { | |
950 | memFreeDataUpto(mem->data, new_mem_lo); | |
951 | mem->inmem_lo = new_mem_lo; | |
8350fe9b | 952 | return; |
8c123b71 | 953 | } |
954 | if (mem->swapout.queue_offset < new_mem_lo) | |
955 | new_mem_lo = mem->swapout.queue_offset; | |
956 | memFreeDataUpto(mem->data, new_mem_lo); | |
957 | mem->inmem_lo = new_mem_lo; | |
958 | ||
959 | swapout_size = (size_t) (mem->inmem_hi - mem->swapout.queue_offset); | |
960 | debug(20, 3) ("storeCheckSwapOut: swapout_size = %d\n", | |
961 | (int) swapout_size); | |
962 | if (swapout_size == 0) | |
963 | return; | |
964 | if (e->store_status == STORE_PENDING && swapout_size < VM_WINDOW_SZ) | |
8350fe9b | 965 | return; /* wait for a full block */ |
966 | /* Ok, we have stuff to swap out. Is there a swapout.fd open? */ | |
967 | if (e->swap_status == SWAPOUT_NONE) { | |
968 | assert(mem->swapout.fd == -1); | |
8c123b71 | 969 | if (storeCheckCachable(e)) |
6a54c60e | 970 | storeSwapOutStart(e); |
8c123b71 | 971 | /* else ENTRY_CACHABLE will be cleared and we'll never get |
6a54c60e | 972 | * here again */ |
8350fe9b | 973 | return; |
974 | } | |
975 | if (e->swap_status == SWAPOUT_OPENING) | |
976 | return; | |
977 | assert(mem->swapout.fd > -1); | |
8c123b71 | 978 | if (swapout_size > SWAP_BUF) |
979 | swapout_size = SWAP_BUF; | |
c6ac1aae | 980 | swap_buf = get_free_8k_page(); |
dbfda5cb | 981 | swap_buf_len = memCopy(mem->data, |
8c123b71 | 982 | mem->swapout.queue_offset, |
c6ac1aae | 983 | swap_buf, |
dbfda5cb | 984 | swapout_size); |
985 | if (swap_buf_len < 0) { | |
986 | debug(20, 1) ("memCopy returned %d for '%s'\n", swap_buf_len, e->key); | |
987 | /* XXX This is probably wrong--we should storeRelease()? */ | |
8350fe9b | 988 | storeDirMapBitReset(e->swap_file_number); |
989 | safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1); | |
990 | e->swap_file_number = -1; | |
991 | e->swap_status = SWAPOUT_NONE; | |
c6ac1aae | 992 | put_free_8k_page(swap_buf); |
bb5f11c4 | 993 | storeSwapOutFileClose(e); |
8350fe9b | 994 | return; |
995 | } | |
c6ac1aae | 996 | debug(20, 3) ("storeCheckSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); |
997 | assert(swap_buf_len > 0); | |
8350fe9b | 998 | debug(20, 3) ("storeCheckSwapOut: swapping out %d bytes from %d\n", |
c6ac1aae | 999 | swap_buf_len, mem->swapout.queue_offset); |
520452b6 | 1000 | mem->swapout.queue_offset += swap_buf_len; |
8350fe9b | 1001 | x = file_write(mem->swapout.fd, |
c6ac1aae | 1002 | swap_buf, |
1003 | swap_buf_len, | |
8350fe9b | 1004 | storeSwapOutHandle, |
1005 | e, | |
c6ac1aae | 1006 | put_free_8k_page); |
8350fe9b | 1007 | assert(x == DISK_OK); |
090089c4 | 1008 | } |
1009 | ||
1010 | /* Append incoming data from a primary server to an entry. */ | |
b8d8561b | 1011 | void |
b8014333 | 1012 | storeAppend(StoreEntry * e, const char *buf, int len) |
090089c4 | 1013 | { |
3a1c3e2f | 1014 | MemObject *mem = e->mem_obj; |
1015 | assert(mem != NULL); | |
1016 | assert(len >= 0); | |
090089c4 | 1017 | if (len) { |
a3d5953d | 1018 | debug(20, 5) ("storeAppend: appending %d bytes for '%s'\n", len, e->key); |
38792624 | 1019 | storeGetMemSpace(len); |
3a1c3e2f | 1020 | memAppend(mem->data, buf, len); |
8350fe9b | 1021 | mem->inmem_hi += len; |
090089c4 | 1022 | } |
aadcfbad | 1023 | if (e->store_status != STORE_ABORTED && !BIT_TEST(e->flag, DELAY_SENDING)) |
090089c4 | 1024 | InvokeHandlers(e); |
8350fe9b | 1025 | storeCheckSwapOut(e); |
090089c4 | 1026 | } |
1027 | ||
24382924 | 1028 | #ifdef __STDC__ |
b8d8561b | 1029 | void |
fe4e214f | 1030 | storeAppendPrintf(StoreEntry * e, const char *fmt,...) |
c30c5a73 | 1031 | { |
15c05bb0 | 1032 | va_list args; |
95d659f0 | 1033 | LOCAL_ARRAY(char, buf, 4096); |
15c05bb0 | 1034 | va_start(args, fmt); |
c30c5a73 | 1035 | #else |
b8d8561b | 1036 | void |
1037 | storeAppendPrintf(va_alist) | |
15c05bb0 | 1038 | va_dcl |
1039 | { | |
1040 | va_list args; | |
1041 | StoreEntry *e = NULL; | |
0ee4272b | 1042 | const char *fmt = NULL; |
95d659f0 | 1043 | LOCAL_ARRAY(char, buf, 4096); |
15c05bb0 | 1044 | va_start(args); |
1045 | e = va_arg(args, StoreEntry *); | |
1046 | fmt = va_arg(args, char *); | |
c30c5a73 | 1047 | #endif |
15c05bb0 | 1048 | buf[0] = '\0'; |
8350fe9b | 1049 | vsnprintf(buf, 4096, fmt, args); |
15c05bb0 | 1050 | storeAppend(e, buf, strlen(buf)); |
1051 | va_end(args); | |
c30c5a73 | 1052 | } |
1053 | ||
090089c4 | 1054 | /* start swapping in */ |
95c4b18f | 1055 | void |
582b6456 | 1056 | storeSwapInStart(StoreEntry * e, SIH * callback, void *callback_data) |
090089c4 | 1057 | { |
0a0bf5db | 1058 | swapin_ctrl_t *ctrlp; |
8350fe9b | 1059 | assert(e->mem_status == NOT_IN_MEMORY); |
311ea387 | 1060 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
97a88399 | 1061 | if (storeDirMapBitTest(e->swap_file_number)) { |
311ea387 | 1062 | /* someone took our file while we weren't looking */ |
8350fe9b | 1063 | callback(-1, callback_data); |
311ea387 | 1064 | return; |
1065 | } | |
1066 | } | |
bbed1a99 | 1067 | assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); |
95c4b18f | 1068 | assert(e->swap_file_number >= 0); |
8350fe9b | 1069 | assert(e->mem_obj != NULL); |
0a0bf5db | 1070 | ctrlp = xmalloc(sizeof(swapin_ctrl_t)); |
1071 | ctrlp->e = e; | |
1072 | ctrlp->callback = callback; | |
1073 | ctrlp->callback_data = callback_data; | |
95c4b18f | 1074 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
35ac11a3 | 1075 | storeSwapInValidateComplete(ctrlp); |
95c4b18f | 1076 | else |
1077 | storeValidate(e, storeSwapInValidateComplete, ctrlp); | |
0a0bf5db | 1078 | } |
090089c4 | 1079 | |
0a0bf5db | 1080 | |
1081 | static void | |
35ac11a3 | 1082 | storeSwapInValidateComplete(void *data) |
0a0bf5db | 1083 | { |
1084 | swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; | |
0a0bf5db | 1085 | StoreEntry *e; |
1086 | e = ctrlp->e; | |
95c4b18f | 1087 | assert(e->mem_status == NOT_IN_MEMORY); |
0a0bf5db | 1088 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
fc07a0e4 | 1089 | /* Invoke a store abort that should free the memory object */ |
8350fe9b | 1090 | (ctrlp->callback) (-1, ctrlp->callback_data); |
0a0bf5db | 1091 | xfree(ctrlp); |
1092 | return; | |
090089c4 | 1093 | } |
95c4b18f | 1094 | ctrlp->path = xstrdup(storeSwapFullPath(e->swap_file_number, NULL)); |
8350fe9b | 1095 | file_open(ctrlp->path, O_RDONLY, storeSwapInFileOpened, ctrlp); |
0a0bf5db | 1096 | } |
1097 | ||
1098 | static void | |
8350fe9b | 1099 | storeSwapInFileOpened(void *data, int fd) |
0a0bf5db | 1100 | { |
0a0bf5db | 1101 | swapin_ctrl_t *ctrlp = (swapin_ctrl_t *) data; |
1102 | StoreEntry *e = ctrlp->e; | |
0379374a | 1103 | assert(e->mem_obj != NULL); |
95c4b18f | 1104 | assert(e->mem_status == NOT_IN_MEMORY); |
be9ce462 | 1105 | assert(e->swap_status == SWAPOUT_WRITING || e->swap_status == SWAPOUT_DONE); |
0a0bf5db | 1106 | if (fd < 0) { |
a3d5953d | 1107 | debug(20, 0) ("storeSwapInStartComplete: Failed for '%s'\n", e->url); |
0a0bf5db | 1108 | /* Invoke a store abort that should free the memory object */ |
8350fe9b | 1109 | (ctrlp->callback) (-1, ctrlp->callback_data); |
0a0bf5db | 1110 | xfree(ctrlp->path); |
1111 | xfree(ctrlp); | |
1112 | return; | |
1113 | } | |
a3d5953d | 1114 | debug(20, 5) ("storeSwapInStart: initialized swap file '%s' for '%s'\n", |
0a0bf5db | 1115 | ctrlp->path, e->url); |
8350fe9b | 1116 | (ctrlp->callback) (fd, ctrlp->callback_data); |
0a0bf5db | 1117 | xfree(ctrlp->path); |
1118 | xfree(ctrlp); | |
090089c4 | 1119 | } |
1120 | ||
090089c4 | 1121 | /* recreate meta data from disk image in swap directory */ |
3facc31e | 1122 | /* Add one swap file at a time from disk storage */ |
48f44632 | 1123 | static void |
1124 | storeDoRebuildFromDisk(void *data) | |
3facc31e | 1125 | { |
5608850b | 1126 | struct storeRebuildState *RB = data; |
429fdbec | 1127 | LOCAL_ARRAY(char, swapfile, MAXPATHLEN); |
f2052513 | 1128 | LOCAL_ARRAY(char, url, MAX_URL); |
090089c4 | 1129 | StoreEntry *e = NULL; |
090089c4 | 1130 | time_t expires; |
1131 | time_t timestamp; | |
919877b6 | 1132 | time_t lastref; |
f5469bce | 1133 | time_t lastmod; |
1134 | int scan1; | |
1135 | int scan2; | |
1136 | int scan3; | |
1137 | int scan4; | |
c54e9052 | 1138 | int scan5; |
1139 | int scan6; | |
919877b6 | 1140 | int scan7; |
3facc31e | 1141 | off_t size; |
090089c4 | 1142 | int sfileno = 0; |
30a4f2a8 | 1143 | int count; |
f5469bce | 1144 | int x; |
5608850b | 1145 | struct _rebuild_dir *d; |
1146 | struct _rebuild_dir **D; | |
429fdbec | 1147 | int used; /* is swapfile already in use? */ |
1148 | int newer; /* is the log entry newer than current entry? */ | |
1149 | ||
30a4f2a8 | 1150 | /* load a number of objects per invocation */ |
5608850b | 1151 | if ((d = RB->rebuild_dir) == NULL) { |
3e347513 | 1152 | storeRebuiltFromDisk(RB); |
1153 | return; | |
5608850b | 1154 | } |
1155 | for (count = 0; count < d->speed; count++) { | |
1156 | if (fgets(RB->line_in, RB->line_in_sz, d->log) == NULL) { | |
a3d5953d | 1157 | debug(20, 1) ("Done reading Cache Dir #%d swap log\n", d->dirn); |
3e347513 | 1158 | fclose(d->log); |
1159 | d->log = NULL; | |
1160 | storeDirCloseTmpSwapLog(d->dirn); | |
1161 | RB->rebuild_dir = d->next; | |
1162 | safe_free(d); | |
1163 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); | |
1164 | return; | |
48f44632 | 1165 | } |
5608850b | 1166 | if ((++RB->linecount & 0x3FFF) == 0) |
a3d5953d | 1167 | debug(20, 1) (" %7d Lines read so far.\n", RB->linecount); |
1168 | debug(20, 9) ("line_in: %s", RB->line_in); | |
5608850b | 1169 | if (RB->line_in[0] == '\0') |
1170 | continue; | |
1171 | if (RB->line_in[0] == '\n') | |
1172 | continue; | |
1173 | if (RB->line_in[0] == '#') | |
1174 | continue; | |
f5469bce | 1175 | url[0] = '\0'; |
f5469bce | 1176 | sfileno = 0; |
1177 | scan1 = 0; | |
1178 | scan2 = 0; | |
30a4f2a8 | 1179 | scan3 = 0; |
f5469bce | 1180 | scan4 = 0; |
919877b6 | 1181 | scan5 = 0; |
1182 | scan6 = 0; | |
1183 | scan7 = 0; | |
1184 | x = sscanf(RB->line_in, "%x %x %x %x %x %d %d %x %s", | |
0be3ec60 | 1185 | &sfileno, /* swap_file_number */ |
1186 | &scan1, /* timestamp */ | |
919877b6 | 1187 | &scan2, /* lastref */ |
1188 | &scan3, /* expires */ | |
1189 | &scan4, /* last modified */ | |
1190 | &scan5, /* size */ | |
1191 | &scan6, /* refcount */ | |
1192 | &scan7, /* flags */ | |
0be3ec60 | 1193 | url); /* url */ |
994bbf93 | 1194 | if (x < 1) { |
1195 | RB->invalid++; | |
30a4f2a8 | 1196 | continue; |
994bbf93 | 1197 | } |
b3264694 | 1198 | if (strncmp(url, "http://internal.squid", 21) == 0) |
1199 | continue; | |
429fdbec | 1200 | storeSwapFullPath(sfileno, swapfile); |
919877b6 | 1201 | if (x != 9) { |
994bbf93 | 1202 | RB->invalid++; |
429fdbec | 1203 | continue; |
994bbf93 | 1204 | } |
1205 | if (sfileno < 0) { | |
1206 | RB->invalid++; | |
0a0bf5db | 1207 | continue; |
994bbf93 | 1208 | } |
919877b6 | 1209 | if (BIT_TEST(scan7, KEY_PRIVATE)) { |
a47b9029 | 1210 | RB->badflags++; |
1211 | continue; | |
1212 | } | |
c932b107 | 1213 | sfileno = storeDirProperFileno(d->dirn, sfileno); |
f5469bce | 1214 | timestamp = (time_t) scan1; |
919877b6 | 1215 | lastref = (time_t) scan2; |
1216 | expires = (time_t) scan3; | |
1217 | lastmod = (time_t) scan4; | |
1218 | size = (off_t) scan5; | |
30a4f2a8 | 1219 | |
429fdbec | 1220 | e = storeGet(url); |
1221 | used = storeDirMapBitTest(sfileno); | |
1222 | /* If this URL already exists in the cache, does the swap log | |
1223 | * appear to have a newer entry? Compare 'timestamp' from the | |
1224 | * swap log to e->lastref. Note, we can't compare e->timestamp | |
1225 | * because it is the Date: header from the HTTP reply and | |
1226 | * doesn't really tell us when the object was added to the | |
1227 | * cache. */ | |
1228 | newer = e ? (timestamp > e->lastref ? 1 : 0) : 0; | |
1229 | if (used && !newer) { | |
1230 | /* log entry is old, ignore it */ | |
1231 | RB->clashcount++; | |
1232 | continue; | |
1233 | } else if (used && e && e->swap_file_number == sfileno) { | |
1234 | /* swapfile taken, same URL, newer, update meta */ | |
1235 | e->lastref = timestamp; | |
1236 | e->timestamp = timestamp; | |
1237 | e->expires = expires; | |
1238 | e->lastmod = lastmod; | |
919877b6 | 1239 | e->flag |= (u_num32) scan6; |
1240 | e->refcount += (u_num32) scan7; | |
429fdbec | 1241 | continue; |
1242 | } else if (used) { | |
1243 | /* swapfile in use, not by this URL, log entry is newer */ | |
1244 | /* This is sorta bad: the log entry should NOT be newer at this | |
1245 | * point. If the log is dirty, the filesize check should have | |
1246 | * caught this. If the log is clean, there should never be a | |
1247 | * newer entry. */ | |
a3d5953d | 1248 | debug(20, 1) ("WARNING: newer swaplog entry for fileno %08X\n", |
30a4f2a8 | 1249 | sfileno); |
429fdbec | 1250 | /* I'm tempted to remove the swapfile here just to be safe, |
1251 | * but there is a bad race condition in the NOVM version if | |
1252 | * the swapfile has recently been opened for writing, but | |
1253 | * not yet opened for reading. Because we can't map | |
1254 | * swapfiles back to StoreEntrys, we don't know the state | |
1255 | * of the entry using that file. */ | |
ccbc48f5 | 1256 | /* We'll assume the existing entry is valid, probably because |
365e5b34 | 1257 | * were in a slow rebuild and the the swap file number got taken |
1258 | * and the validation procedure hasn't run. */ | |
ccbc48f5 | 1259 | assert(RB->need_to_validate); |
5608850b | 1260 | RB->clashcount++; |
30a4f2a8 | 1261 | continue; |
429fdbec | 1262 | } else if (e) { |
1263 | /* URL already exists, this swapfile not being used */ | |
1264 | /* junk old, load new */ | |
1265 | storeRelease(e); /* release old entry */ | |
1266 | RB->dupcount++; | |
1267 | } else { | |
1268 | /* URL doesnt exist, swapfile not in use */ | |
1269 | /* load new */ | |
1270 | (void) 0; | |
30a4f2a8 | 1271 | } |
1272 | /* update store_swap_size */ | |
c932b107 | 1273 | storeDirUpdateSwapSize(sfileno, size, 1); |
5608850b | 1274 | RB->objcount++; |
f5469bce | 1275 | e = storeAddDiskRestore(url, |
30a4f2a8 | 1276 | sfileno, |
1277 | (int) size, | |
1278 | expires, | |
f5469bce | 1279 | timestamp, |
919877b6 | 1280 | lastref, |
311ea387 | 1281 | lastmod, |
919877b6 | 1282 | (u_num32) scan6, /* refcount */ |
1283 | (u_num32) scan7, /* flags */ | |
311ea387 | 1284 | d->clean); |
5608850b | 1285 | storeDirSwapLog(e); |
a528eb87 | 1286 | HTTPCacheInfo->proto_newobject(HTTPCacheInfo, |
1ac9bc38 | 1287 | urlParseProtocol(url), |
30a4f2a8 | 1288 | (int) size, |
1289 | TRUE); | |
c943f331 | 1290 | } |
5608850b | 1291 | RB->rebuild_dir = d->next; |
76fefb77 | 1292 | for (D = &RB->rebuild_dir; *D; D = &(*D)->next); |
5608850b | 1293 | *D = d; |
1294 | d->next = NULL; | |
1295 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); | |
3facc31e | 1296 | } |
1297 | ||
0a0bf5db | 1298 | static void |
1299 | storeCleanup(void *data) | |
1300 | { | |
1301 | static storeCleanList *list = NULL; | |
1302 | storeCleanList *curr; | |
1303 | static int bucketnum = -1; | |
1304 | static int validnum = 0; | |
1305 | StoreEntry *e; | |
1306 | hash_link *link_ptr = NULL; | |
1307 | if (list == NULL) { | |
1308 | if (++bucketnum >= store_buckets) { | |
a3d5953d | 1309 | debug(20, 1) (" Completed Validation Procedure\n"); |
1310 | debug(20, 1) (" Validated %d Entries\n", validnum); | |
f182d1c5 | 1311 | store_rebuilding = 0; |
0a0bf5db | 1312 | return; |
1313 | } | |
1314 | link_ptr = hash_get_bucket(store_table, bucketnum); | |
1315 | for (; link_ptr; link_ptr = link_ptr->next) { | |
1316 | e = (StoreEntry *) link_ptr; | |
35ac11a3 | 1317 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1318 | continue; | |
1319 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) | |
1320 | continue; | |
311ea387 | 1321 | curr = xcalloc(1, sizeof(storeCleanList)); |
0a0bf5db | 1322 | curr->key = xstrdup(e->key); |
1323 | curr->next = list; | |
1324 | list = curr; | |
1325 | } | |
1326 | } | |
1327 | if (list == NULL) { | |
1328 | eventAdd("storeCleanup", storeCleanup, NULL, 0); | |
1329 | return; | |
1330 | } | |
1331 | curr = list; | |
1332 | list = list->next; | |
1333 | e = (StoreEntry *) hash_lookup(store_table, curr->key); | |
c340827e | 1334 | if (e && !BIT_TEST(e->flag, ENTRY_VALIDATED)) { |
365e5b34 | 1335 | storeLockObject(e); |
3e347513 | 1336 | storeValidate(e, storeCleanupComplete, e); |
a3d5953d | 1337 | if ((++validnum & 0xFFF) == 0) |
1338 | debug(20, 1) (" %7d Entries Validated so far.\n", validnum); | |
35ac11a3 | 1339 | assert(validnum <= meta_data.store_entries); |
95c4b18f | 1340 | } |
0a0bf5db | 1341 | xfree(curr->key); |
1342 | xfree(curr); | |
1343 | eventAdd("storeCleanup", storeCleanup, NULL, 0); | |
1344 | } | |
1345 | ||
0a0bf5db | 1346 | static void |
35ac11a3 | 1347 | storeCleanupComplete(void *data) |
0a0bf5db | 1348 | { |
1349 | StoreEntry *e = data; | |
35ac11a3 | 1350 | storeUnlockObject(e); |
0a0bf5db | 1351 | if (!BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1352 | storeRelease(e); | |
1353 | } | |
1354 | ||
1355 | static void | |
1356 | storeValidate(StoreEntry * e, VCB callback, void *callback_data) | |
1357 | { | |
1358 | valid_ctrl_t *ctrlp; | |
1359 | char *path; | |
1360 | struct stat *sb; | |
606bd3b5 | 1361 | #if !USE_ASYNC_IO |
95c4b18f | 1362 | int x; |
606bd3b5 | 1363 | #endif |
d7b78fbb | 1364 | assert(!BIT_TEST(e->flag, ENTRY_VALIDATED)); |
0a0bf5db | 1365 | if (e->swap_file_number < 0) { |
1366 | BIT_RESET(e->flag, ENTRY_VALIDATED); | |
35ac11a3 | 1367 | callback(callback_data); |
0a0bf5db | 1368 | return; |
1369 | } | |
1370 | path = storeSwapFullPath(e->swap_file_number, NULL); | |
1371 | sb = xmalloc(sizeof(struct stat)); | |
1372 | ctrlp = xmalloc(sizeof(valid_ctrl_t)); | |
1373 | ctrlp->sb = sb; | |
1374 | ctrlp->e = e; | |
1375 | ctrlp->callback = callback; | |
1376 | ctrlp->callback_data = callback_data; | |
1377 | #if USE_ASYNC_IO | |
1378 | aioStat(path, sb, storeValidateComplete, ctrlp); | |
1379 | #else | |
95c4b18f | 1380 | /* When evaluating the actual arguments in a function call, the order |
1381 | * in which the arguments and the function expression are evaluated is | |
1382 | * not specified; */ | |
1383 | x = stat(path, sb); | |
1384 | storeValidateComplete(ctrlp, x, errno); | |
0a0bf5db | 1385 | #endif |
1386 | return; | |
1387 | } | |
1388 | ||
1389 | static void | |
1390 | storeValidateComplete(void *data, int retcode, int errcode) | |
1391 | { | |
1392 | valid_ctrl_t *ctrlp = data; | |
1393 | struct stat *sb = ctrlp->sb; | |
1394 | StoreEntry *e = ctrlp->e; | |
1395 | char *path; | |
1396 | if (retcode < 0 && errcode == EWOULDBLOCK) { | |
1397 | path = storeSwapFullPath(e->swap_file_number, NULL); | |
1398 | retcode = stat(path, sb); | |
1399 | } | |
9e4ad609 | 1400 | if (retcode < 0 || sb->st_size == 0 || sb->st_size != e->object_len) { |
0a0bf5db | 1401 | BIT_RESET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 1402 | } else { |
0a0bf5db | 1403 | BIT_SET(e->flag, ENTRY_VALIDATED); |
9e4ad609 | 1404 | storeDirMapBitSet(e->swap_file_number); |
1405 | } | |
0a0bf5db | 1406 | errno = errcode; |
35ac11a3 | 1407 | ctrlp->callback(ctrlp->callback_data); |
0a0bf5db | 1408 | xfree(sb); |
1409 | xfree(ctrlp); | |
1410 | } | |
1411 | ||
3facc31e | 1412 | /* meta data recreated from disk image in swap directory */ |
b8d8561b | 1413 | static void |
5608850b | 1414 | storeRebuiltFromDisk(struct storeRebuildState *data) |
3facc31e | 1415 | { |
1416 | time_t r; | |
1417 | time_t stop; | |
97c03d3c | 1418 | stop = squid_curtime; |
3facc31e | 1419 | r = stop - data->start; |
a3d5953d | 1420 | debug(20, 1) ("Finished rebuilding storage from disk image.\n"); |
1421 | debug(20, 1) (" %7d Lines read from previous logfile.\n", data->linecount); | |
994bbf93 | 1422 | debug(20, 1) (" %7d Invalid lines.\n", data->invalid); |
a47b9029 | 1423 | debug(20, 1) (" %7d With invalid flags.\n", data->badflags); |
a3d5953d | 1424 | debug(20, 1) (" %7d Objects loaded.\n", data->objcount); |
1425 | debug(20, 1) (" %7d Objects expired.\n", data->expcount); | |
1426 | debug(20, 1) (" %7d Duplicate URLs purged.\n", data->dupcount); | |
1427 | debug(20, 1) (" %7d Swapfile clashes avoided.\n", data->clashcount); | |
1428 | debug(20, 1) (" Took %d seconds (%6.1lf objects/sec).\n", | |
3facc31e | 1429 | r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1)); |
a3d5953d | 1430 | debug(20, 1) (" store_swap_size = %dk\n", store_swap_size); |
76f87348 | 1431 | if (data->need_to_validate && data->linecount) { |
a3d5953d | 1432 | debug(20, 1) ("Beginning Validation Procedure\n"); |
0a0bf5db | 1433 | eventAdd("storeCleanup", storeCleanup, NULL, 0); |
f182d1c5 | 1434 | } else { |
a3d5953d | 1435 | store_rebuilding = 0; |
0a0bf5db | 1436 | } |
f182d1c5 | 1437 | safe_free(data->line_in); |
1438 | safe_free(data); | |
090089c4 | 1439 | } |
1440 | ||
24382924 | 1441 | static void |
b8d8561b | 1442 | storeStartRebuildFromDisk(void) |
3facc31e | 1443 | { |
3facc31e | 1444 | int i; |
5608850b | 1445 | struct storeRebuildState *RB; |
1446 | struct _rebuild_dir *d; | |
1447 | FILE *fp; | |
1448 | int clean; | |
1449 | RB = xcalloc(1, sizeof(struct storeRebuildState)); | |
1450 | RB->start = squid_curtime; | |
f1dc9b30 | 1451 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
5608850b | 1452 | fp = storeDirOpenTmpSwapLog(i, &clean); |
1453 | if (fp == NULL) | |
1454 | continue; | |
1455 | d = xcalloc(1, sizeof(struct _rebuild_dir)); | |
1456 | d->dirn = i; | |
1457 | d->log = fp; | |
1458 | d->clean = clean; | |
1459 | d->speed = opt_foreground_rebuild ? 1 << 30 : 50; | |
1460 | d->next = RB->rebuild_dir; | |
1461 | RB->rebuild_dir = d; | |
1462 | if (!clean) | |
f182d1c5 | 1463 | RB->need_to_validate = 1; |
a3d5953d | 1464 | debug(20, 1) ("Rebuilding storage in Cache Dir #%d (%s)\n", |
5608850b | 1465 | i, clean ? "CLEAN" : "DIRTY"); |
1466 | } | |
1467 | RB->line_in_sz = 4096; | |
1468 | RB->line_in = xcalloc(1, RB->line_in_sz); | |
30a4f2a8 | 1469 | if (opt_foreground_rebuild) { |
5608850b | 1470 | storeDoRebuildFromDisk(RB); |
30a4f2a8 | 1471 | } else { |
5608850b | 1472 | eventAdd("storeRebuild", storeDoRebuildFromDisk, RB, 0); |
30a4f2a8 | 1473 | } |
3facc31e | 1474 | } |
090089c4 | 1475 | |
b8d8561b | 1476 | static int |
8350fe9b | 1477 | storeCheckCachable(StoreEntry * e) |
6602e70e | 1478 | { |
a7e59001 | 1479 | if (e->method != METHOD_GET) { |
8350fe9b | 1480 | debug(20, 2) ("storeCheckCachable: NO: non-GET method\n"); |
1c481e00 | 1481 | } else if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) { |
8350fe9b | 1482 | debug(20, 2) ("storeCheckCachable: NO: not cachable\n"); |
6602e70e | 1483 | } else if (BIT_TEST(e->flag, RELEASE_REQUEST)) { |
8350fe9b | 1484 | debug(20, 2) ("storeCheckCachable: NO: release requested\n"); |
83d453f3 | 1485 | } else if (e->store_status == STORE_OK && !storeEntryValidLength(e)) { |
8350fe9b | 1486 | debug(20, 2) ("storeCheckCachable: NO: wrong content-length\n"); |
79b5cc5f | 1487 | } else if (BIT_TEST(e->flag, ENTRY_NEGCACHED)) { |
8350fe9b | 1488 | debug(20, 2) ("storeCheckCachable: NO: negative cached\n"); |
3e98df20 | 1489 | return 0; /* avoid release call below */ |
8350fe9b | 1490 | } else if (e->mem_obj->inmem_hi > Config.Store.maxObjectSize) { |
1491 | debug(20, 2) ("storeCheckCachable: NO: too big\n"); | |
89e079ed | 1492 | } else if (BIT_TEST(e->flag, KEY_PRIVATE)) { |
8350fe9b | 1493 | debug(20, 3) ("storeCheckCachable: NO: private key\n"); |
d4432957 | 1494 | } else { |
6602e70e | 1495 | return 1; |
d4432957 | 1496 | } |
2daae136 | 1497 | storeReleaseRequest(e); |
1c481e00 | 1498 | BIT_RESET(e->flag, ENTRY_CACHABLE); |
6602e70e | 1499 | return 0; |
1500 | } | |
1501 | ||
090089c4 | 1502 | /* Complete transfer into the local cache. */ |
b8d8561b | 1503 | void |
1504 | storeComplete(StoreEntry * e) | |
090089c4 | 1505 | { |
a3d5953d | 1506 | debug(20, 3) ("storeComplete: '%s'\n", e->key); |
8350fe9b | 1507 | e->object_len = e->mem_obj->inmem_hi; |
234967c9 | 1508 | e->store_status = STORE_OK; |
8350fe9b | 1509 | assert(e->mem_status == NOT_IN_MEMORY); |
3a1c3e2f | 1510 | InvokeHandlers(e); |
8c123b71 | 1511 | storeCheckSwapOut(e); |
090089c4 | 1512 | } |
1513 | ||
1514 | /* | |
474cac1b | 1515 | * Someone wants to abort this transfer. Set the reason in the |
1516 | * request structure, call the server-side callback and mark the | |
1517 | * entry for releasing | |
090089c4 | 1518 | */ |
b8d8561b | 1519 | void |
9b312a19 | 1520 | storeAbort(StoreEntry * e, int cbflag) |
090089c4 | 1521 | { |
3e98df20 | 1522 | MemObject *mem = e->mem_obj; |
8f39b81d | 1523 | assert(e->store_status == STORE_PENDING); |
1524 | assert(mem != NULL); | |
9b312a19 | 1525 | debug(20, 6) ("storeAbort: %s\n", e->key); |
79b5cc5f | 1526 | storeNegativeCache(e); |
474cac1b | 1527 | storeReleaseRequest(e); |
234967c9 | 1528 | e->store_status = STORE_ABORTED; |
8350fe9b | 1529 | storeSetMemStatus(e, NOT_IN_MEMORY); |
090089c4 | 1530 | /* No DISK swap for negative cached object */ |
8350fe9b | 1531 | e->swap_status = SWAPOUT_NONE; |
090089c4 | 1532 | /* Count bytes faulted through cache but not moved to disk */ |
a528eb87 | 1533 | HTTPCacheInfo->proto_touchobject(HTTPCacheInfo, |
3e98df20 | 1534 | mem->request ? mem->request->protocol : PROTO_NONE, |
8350fe9b | 1535 | mem->inmem_hi); |
090089c4 | 1536 | /* We assign an object length here--The only other place we assign the |
1537 | * object length is in storeComplete() */ | |
8350fe9b | 1538 | e->object_len = mem->inmem_hi; |
474cac1b | 1539 | /* Notify the server side */ |
1540 | if (cbflag && mem->abort.callback) { | |
bfcaf585 | 1541 | mem->abort.callback(mem->abort.data); |
1542 | mem->abort.callback = NULL; | |
1543 | } | |
474cac1b | 1544 | /* Notify the client side */ |
090089c4 | 1545 | InvokeHandlers(e); |
6e86c3e8 | 1546 | /* Do we need to close the swapout file? */ |
1547 | /* Not if we never started swapping out */ | |
1548 | if (e->swap_file_number == -1) | |
1549 | return; | |
1550 | /* not if a disk write is queued, the handler will close up */ | |
1551 | if (mem->swapout.queue_offset > mem->swapout.done_offset) | |
1552 | return; | |
1553 | /* we do */ | |
1554 | storeSwapOutFileClose(e); | |
090089c4 | 1555 | } |
1556 | ||
090089c4 | 1557 | /* get the first entry in the storage */ |
b8d8561b | 1558 | StoreEntry * |
1559 | storeGetFirst(void) | |
090089c4 | 1560 | { |
c3c5b744 | 1561 | return ((StoreEntry *) hash_first(store_table)); |
090089c4 | 1562 | } |
1563 | ||
1564 | ||
1565 | /* get the next entry in the storage for a given search pointer */ | |
b8d8561b | 1566 | StoreEntry * |
1567 | storeGetNext(void) | |
090089c4 | 1568 | { |
c3c5b744 | 1569 | return ((StoreEntry *) hash_next(store_table)); |
090089c4 | 1570 | } |
1571 | ||
090089c4 | 1572 | |
090089c4 | 1573 | /* Clear Memory storage to accommodate the given object len */ |
38792624 | 1574 | static void |
d4432957 | 1575 | storeGetMemSpace(int size) |
090089c4 | 1576 | { |
b32508fb | 1577 | StoreEntry *e = NULL; |
090089c4 | 1578 | int n_expired = 0; |
66c2848f | 1579 | int n_purged = 0; |
579a45cd | 1580 | int n_released = 0; |
b32508fb | 1581 | static time_t last_check = 0; |
e954773d | 1582 | int pages_needed; |
b93bcace | 1583 | dlink_node *m; |
1584 | dlink_node *prev; | |
b32508fb | 1585 | if (squid_curtime == last_check) |
38792624 | 1586 | return; |
b32508fb | 1587 | last_check = squid_curtime; |
e954773d | 1588 | pages_needed = (size / SM_PAGE_SIZE) + 1; |
61cd203b | 1589 | if (sm_stats.n_pages_in_use + pages_needed < store_pages_high) |
38792624 | 1590 | return; |
311ea387 | 1591 | if (store_rebuilding) |
38792624 | 1592 | return; |
a3d5953d | 1593 | debug(20, 2) ("storeGetMemSpace: Starting, need %d pages\n", pages_needed); |
b93bcace | 1594 | for (m = inmem_list.tail; m; m = prev) { |
8350fe9b | 1595 | prev = m->prev; |
b93bcace | 1596 | e = m->data; |
8350fe9b | 1597 | if (storeEntryLocked(e)) |
1598 | continue; | |
1599 | if (storeCheckExpired(e, 0)) { | |
1600 | debug(20, 2) ("storeGetMemSpace: Expired: %s\n", e->url); | |
1601 | n_expired++; | |
1602 | storeRelease(e); | |
1603 | } else { | |
1604 | storePurgeMem(e); | |
1605 | n_purged++; | |
1606 | } | |
1607 | if (sm_stats.n_pages_in_use + pages_needed < store_pages_low) | |
1608 | break; | |
1609 | } | |
5e3d4fef | 1610 | debug(20, 3) ("storeGetMemSpace stats:\n"); |
1611 | debug(20, 3) (" %6d HOT objects\n", meta_data.hot_vm); | |
1612 | debug(20, 3) (" %6d were purged\n", n_purged); | |
1613 | debug(20, 3) (" %6d were released\n", n_released); | |
090089c4 | 1614 | } |
1615 | ||
b8d8561b | 1616 | static int |
1617 | compareLastRef(StoreEntry ** e1, StoreEntry ** e2) | |
090089c4 | 1618 | { |
1619 | if (!e1 || !e2) | |
1620 | fatal_dump(NULL); | |
090089c4 | 1621 | if ((*e1)->lastref > (*e2)->lastref) |
1622 | return (1); | |
090089c4 | 1623 | if ((*e1)->lastref < (*e2)->lastref) |
1624 | return (-1); | |
090089c4 | 1625 | return (0); |
1626 | } | |
1627 | ||
cb6ae8c3 | 1628 | static int |
d3c12084 | 1629 | compareBucketOrder(struct _bucketOrder *a, struct _bucketOrder *b) |
cb6ae8c3 | 1630 | { |
d3c12084 | 1631 | return a->index - b->index; |
cb6ae8c3 | 1632 | } |
1633 | ||
090089c4 | 1634 | /* returns the bucket number to work on, |
1635 | * pointer to next bucket after each calling | |
1636 | */ | |
24382924 | 1637 | static unsigned int |
b8d8561b | 1638 | storeGetBucketNum(void) |
090089c4 | 1639 | { |
1640 | static unsigned int bucket = 0; | |
66cedb85 | 1641 | if (bucket >= store_buckets) |
090089c4 | 1642 | bucket = 0; |
1643 | return (bucket++); | |
1644 | } | |
1645 | ||
66cedb85 | 1646 | #define SWAP_MAX_HELP (store_buckets/2) |
090089c4 | 1647 | |
1648 | /* The maximum objects to scan for maintain storage space */ | |
0a0bf5db | 1649 | #define SWAP_LRUSCAN_COUNT 1024 |
1650 | #define SWAP_LRU_REMOVE_COUNT 64 | |
090089c4 | 1651 | |
1652 | /* Clear Swap storage to accommodate the given object len */ | |
679ac4f0 | 1653 | void |
b8d8561b | 1654 | storeGetSwapSpace(int size) |
090089c4 | 1655 | { |
b93bcace | 1656 | dlink_node *m; |
1657 | dlink_node *prev; | |
a1b0d7cf | 1658 | StoreEntry *e = NULL; |
090089c4 | 1659 | int scanned = 0; |
090089c4 | 1660 | int locked = 0; |
679ac4f0 | 1661 | int expired = 0; |
b93bcace | 1662 | int purged = 0; |
ec14f197 | 1663 | static time_t last_warn_time = 0; |
63017fae | 1664 | int kb_size = ((size + 1023) >> 10); |
b93bcace | 1665 | if (store_swap_size + kb_size <= store_swap_high) |
679ac4f0 | 1666 | return; |
b93bcace | 1667 | debug(20, 1) ("storeGetSwapSpace: Need %d bytes...\n", size); |
1668 | for (m = all_list.tail; m; m = prev) { | |
1669 | prev = m->prev; | |
1670 | e = m->data; | |
1671 | if (storeEntryLocked(e)) { | |
1672 | locked++; | |
1673 | } else if (storeCheckExpired(e, 0)) { | |
1674 | expired++; | |
1675 | storeRelease(e); | |
1676 | } else { | |
1677 | storeRelease(e); | |
1678 | purged++; | |
090089c4 | 1679 | } |
b93bcace | 1680 | if (store_swap_size + kb_size <= store_swap_high) |
1681 | break; | |
1682 | if (++scanned > SWAP_LRUSCAN_COUNT) | |
1683 | break; | |
090089c4 | 1684 | } |
b93bcace | 1685 | debug(20, 1) ("storeGetSwapSpace stats:\n"); |
1686 | debug(20, 1) (" %6d objects\n", meta_data.store_entries); | |
1687 | debug(20, 1) (" %6d were scanned\n", scanned); | |
1688 | debug(20, 1) (" %6d were locked\n", locked); | |
1689 | debug(20, 1) (" %6d were expired\n", expired); | |
1690 | debug(20, 1) (" %6d were purged\n", purged); | |
1691 | if (store_swap_size + kb_size < Config.Swap.maxSize) | |
1692 | return; | |
1693 | if (squid_curtime - last_warn_time < 10) | |
1694 | return; | |
1695 | debug(20, 0) ("WARNING: Disk space over limit: %d KB > %d KB\n", | |
1696 | store_swap_size, Config.Swap.maxSize); | |
1697 | last_warn_time = squid_curtime; | |
090089c4 | 1698 | } |
1699 | ||
1700 | ||
1701 | /* release an object from a cache */ | |
cc61958c | 1702 | /* return number of objects released. */ |
b8d8561b | 1703 | int |
1704 | storeRelease(StoreEntry * e) | |
090089c4 | 1705 | { |
cc61958c | 1706 | StoreEntry *hentry = NULL; |
0ee4272b | 1707 | const char *hkey; |
a3d5953d | 1708 | debug(20, 3) ("storeRelease: Releasing: '%s'\n", e->key); |
090089c4 | 1709 | /* If, for any reason we can't discard this object because of an |
1710 | * outstanding request, mark it for pending release */ | |
1711 | if (storeEntryLocked(e)) { | |
9174e204 | 1712 | storeExpireNow(e); |
a3d5953d | 1713 | debug(20, 3) ("storeRelease: Only setting RELEASE_REQUEST bit\n"); |
2daae136 | 1714 | storeReleaseRequest(e); |
cc61958c | 1715 | return 0; |
090089c4 | 1716 | } |
cc61958c | 1717 | /* check if coresponding HEAD object exists. */ |
73ddce89 | 1718 | if (e->method == METHOD_GET) { |
cc61958c | 1719 | hkey = storeGeneratePublicKey(e->url, METHOD_HEAD); |
1720 | if ((hentry = (StoreEntry *) hash_lookup(store_table, hkey))) | |
38792624 | 1721 | storeExpireNow(hentry); |
090089c4 | 1722 | } |
311ea387 | 1723 | if (store_rebuilding) { |
a3d5953d | 1724 | debug(20, 2) ("storeRelease: Delaying release until store is rebuilt: '%s'\n", |
30a4f2a8 | 1725 | e->key ? e->key : e->url ? e->url : "NO URL"); |
1726 | storeExpireNow(e); | |
1727 | storeSetPrivateKey(e); | |
35ac11a3 | 1728 | BIT_SET(e->flag, RELEASE_REQUEST); |
cc61958c | 1729 | return 0; |
30a4f2a8 | 1730 | } |
43fee0b8 | 1731 | if (e->swap_file_number > -1) { |
9e4ad609 | 1732 | if (BIT_TEST(e->flag, ENTRY_VALIDATED)) |
1733 | storePutUnusedFileno(e->swap_file_number); | |
c932b107 | 1734 | storeDirUpdateSwapSize(e->swap_file_number, e->object_len, -1); |
090089c4 | 1735 | e->swap_file_number = -1; |
a528eb87 | 1736 | HTTPCacheInfo->proto_purgeobject(HTTPCacheInfo, |
1ac9bc38 | 1737 | urlParseProtocol(e->url), |
30a4f2a8 | 1738 | e->object_len); |
090089c4 | 1739 | } |
8350fe9b | 1740 | storeSetMemStatus(e, NOT_IN_MEMORY); |
f7a5493b | 1741 | storeHashDelete(e); |
147d3115 | 1742 | storeLog(STORE_LOG_RELEASE, e); |
30a4f2a8 | 1743 | destroy_StoreEntry(e); |
cc61958c | 1744 | return 1; |
090089c4 | 1745 | } |
1746 | ||
090089c4 | 1747 | /* return 1 if a store entry is locked */ |
24382924 | 1748 | static int |
fe4e214f | 1749 | storeEntryLocked(const StoreEntry * e) |
090089c4 | 1750 | { |
30a4f2a8 | 1751 | if (e->lock_count) |
1752 | return 1; | |
8350fe9b | 1753 | if (e->swap_status == SWAPOUT_WRITING) |
30a4f2a8 | 1754 | return 1; |
a1e47288 | 1755 | if (e->store_status == STORE_PENDING) |
1756 | return 1; | |
365cb147 | 1757 | if (BIT_TEST(e->flag, ENTRY_SPECIAL)) |
1758 | return 1; | |
30a4f2a8 | 1759 | return 0; |
090089c4 | 1760 | } |
1761 | ||
090089c4 | 1762 | /* check if there is any client waiting for this object at all */ |
1763 | /* return 1 if there is at least one client */ | |
b8d8561b | 1764 | int |
fe4e214f | 1765 | storeClientWaiting(const StoreEntry * e) |
090089c4 | 1766 | { |
6e40f263 | 1767 | MemObject *mem = e->mem_obj; |
43329771 | 1768 | store_client *sc; |
1769 | for (sc = mem->clients; sc; sc = sc->next) { | |
1770 | if (sc->callback_data != NULL) | |
1771 | return 1; | |
090089c4 | 1772 | } |
1773 | return 0; | |
1774 | } | |
1775 | ||
43329771 | 1776 | static store_client * |
382d851a | 1777 | storeClientListSearch(const MemObject * mem, void *data) |
090089c4 | 1778 | { |
43329771 | 1779 | store_client *sc; |
1780 | for (sc = mem->clients; sc; sc = sc->next) { | |
1781 | if (sc->callback_data == data) | |
1782 | break; | |
090089c4 | 1783 | } |
43329771 | 1784 | return sc; |
090089c4 | 1785 | } |
1786 | ||
1787 | /* add client with fd to client list */ | |
43329771 | 1788 | void |
fe96bbe6 | 1789 | storeClientListAdd(StoreEntry * e, void *data) |
090089c4 | 1790 | { |
6e40f263 | 1791 | MemObject *mem = e->mem_obj; |
43329771 | 1792 | store_client **T; |
1793 | store_client *sc; | |
8350fe9b | 1794 | if (mem == NULL) |
1795 | mem = e->mem_obj = new_MemObject(urlClean(e->url)); | |
43329771 | 1796 | if (storeClientListSearch(mem, data) != NULL) |
1797 | return; | |
1798 | mem->nclients++; | |
1799 | sc = xcalloc(1, sizeof(store_client)); | |
5d86029a | 1800 | cbdataAdd(sc); /* sc is callback_data for file_read */ |
5f0e9628 | 1801 | sc->callback_data = data; |
1802 | sc->seen_offset = 0; | |
1803 | sc->copy_offset = 0; | |
1804 | sc->swapin_fd = -1; | |
1805 | sc->mem = mem; | |
43329771 | 1806 | if (e->store_status == STORE_PENDING && mem->swapout.fd == -1) |
1807 | sc->type = STORE_MEM_CLIENT; | |
1808 | else | |
1809 | sc->type = STORE_DISK_CLIENT; | |
1810 | for (T = &mem->clients; *T; T = &(*T)->next); | |
1811 | *T = sc; | |
090089c4 | 1812 | } |
1813 | ||
dbfda5cb | 1814 | /* copy bytes requested by the client */ |
d89d1fb6 | 1815 | void |
38792624 | 1816 | storeClientCopy(StoreEntry * e, |
fe96bbe6 | 1817 | off_t seen_offset, |
1818 | off_t copy_offset, | |
d89d1fb6 | 1819 | size_t size, |
38792624 | 1820 | char *buf, |
d89d1fb6 | 1821 | STCB * callback, |
382d851a | 1822 | void *data) |
38792624 | 1823 | { |
43329771 | 1824 | store_client *sc; |
d4b1b930 | 1825 | static int recurse_detect = 0; |
b974bc9d | 1826 | assert(e->store_status != STORE_ABORTED); |
f182d1c5 | 1827 | assert(recurse_detect < 3); /* could == 1 for IMS not modified's */ |
8350fe9b | 1828 | debug(20, 3) ("storeClientCopy: %s, seen %d, want %d, size %d, cb %p, cbdata %p\n", |
86101e40 | 1829 | e->key, |
1830 | (int) seen_offset, | |
1831 | (int) copy_offset, | |
1832 | (int) size, | |
1833 | callback, | |
1834 | data); | |
dbfda5cb | 1835 | sc = storeClientListSearch(e->mem_obj, data); |
8350fe9b | 1836 | assert(sc != NULL); |
edbaeece | 1837 | assert(sc->callback == NULL); |
fe96bbe6 | 1838 | sc->copy_offset = copy_offset; |
1839 | sc->seen_offset = seen_offset; | |
8350fe9b | 1840 | sc->callback = callback; |
1841 | sc->copy_buf = buf; | |
1842 | sc->copy_size = size; | |
1843 | sc->copy_offset = copy_offset; | |
dbfda5cb | 1844 | storeClientCopy2(e, sc); |
1845 | recurse_detect--; | |
1846 | } | |
1847 | ||
1848 | static void | |
1849 | storeClientCopy2(StoreEntry * e, store_client * sc) | |
1850 | { | |
1851 | STCB *callback = sc->callback; | |
1852 | MemObject *mem = e->mem_obj; | |
1853 | size_t sz; | |
6e86c3e8 | 1854 | debug(20, 3) ("storeClientCopy2: %s\n", e->key); |
b93bcace | 1855 | assert(callback != NULL); |
1856 | if (e->store_status == STORE_ABORTED) { | |
1857 | sc->callback = NULL; | |
1858 | callback(sc->callback_data, sc->copy_buf, 0); | |
1859 | return; | |
1860 | } | |
1861 | assert(sc->seen_offset <= mem->inmem_hi || e->store_status == STORE_OK); | |
dbfda5cb | 1862 | if (e->store_status == STORE_PENDING && sc->seen_offset == mem->inmem_hi) { |
fe96bbe6 | 1863 | /* client has already seen this, wait for more */ |
dbfda5cb | 1864 | debug(20, 3) ("storeClientCopy2: Waiting for more\n"); |
d89d1fb6 | 1865 | return; |
090089c4 | 1866 | } |
dbfda5cb | 1867 | if (sc->copy_offset >= mem->inmem_lo && mem->inmem_lo < mem->inmem_hi) { |
8350fe9b | 1868 | /* What the client wants is in memory */ |
dbfda5cb | 1869 | debug(20, 3) ("storeClientCopy2: Copying from memory\n"); |
1870 | sz = memCopy(mem->data, sc->copy_offset, sc->copy_buf, sc->copy_size); | |
8350fe9b | 1871 | sc->callback = NULL; |
dbfda5cb | 1872 | callback(sc->callback_data, sc->copy_buf, sz); |
8350fe9b | 1873 | } else if (sc->swapin_fd < 0) { |
dbfda5cb | 1874 | debug(20, 3) ("storeClientCopy2: Need to open swap in file\n"); |
43329771 | 1875 | assert(sc->type == STORE_DISK_CLIENT); |
8350fe9b | 1876 | /* gotta open the swapin file */ |
dbfda5cb | 1877 | assert(sc->copy_offset == 0); |
8350fe9b | 1878 | storeSwapInStart(e, storeClientCopyFileOpened, sc); |
1879 | } else { | |
1880 | debug(20, 3) ("storeClientCopy: reading from disk FD %d\n", | |
1881 | sc->swapin_fd); | |
43329771 | 1882 | assert(sc->type == STORE_DISK_CLIENT); |
8350fe9b | 1883 | storeClientCopyFileRead(sc); |
1884 | } | |
090089c4 | 1885 | } |
1886 | ||
8350fe9b | 1887 | static void |
1888 | storeClientCopyFileOpened(int fd, void *data) | |
1889 | { | |
43329771 | 1890 | store_client *sc = data; |
1891 | STCB *callback = sc->callback; | |
8350fe9b | 1892 | if (fd < 0) { |
edbaeece | 1893 | debug(20, 3) ("storeClientCopyFileOpened: failed\n"); |
43329771 | 1894 | sc->callback = NULL; |
1895 | callback(sc->callback_data, sc->copy_buf, -1); | |
8350fe9b | 1896 | return; |
1897 | } | |
1898 | sc->swapin_fd = fd; | |
1899 | storeClientCopyFileRead(sc); | |
1900 | } | |
1901 | ||
1902 | static void | |
5d86029a | 1903 | storeClientCopyFileRead(store_client * sc) |
8350fe9b | 1904 | { |
5d86029a | 1905 | assert(sc->callback != NULL); |
8350fe9b | 1906 | file_read(sc->swapin_fd, |
1907 | sc->copy_buf, | |
1908 | sc->copy_size, | |
1909 | sc->copy_offset, | |
1910 | storeClientCopyHandleRead, | |
1911 | sc); | |
1912 | } | |
1913 | ||
1914 | static void | |
1915 | storeClientCopyHandleRead(int fd, const char *buf, int len, int flag, void *data) | |
1916 | { | |
43329771 | 1917 | store_client *sc = data; |
5f0e9628 | 1918 | MemObject *mem = sc->mem; |
43329771 | 1919 | STCB *callback = sc->callback; |
06e8899b | 1920 | assert(sc->callback != NULL); |
5f0e9628 | 1921 | debug(20, 3) ("storeClientCopyHandleRead: FD %d, len %d\n", fd, len); |
43329771 | 1922 | if (sc->copy_offset == 0 && len > 0 && mem != NULL) |
6a54c60e | 1923 | httpParseReplyHeaders(buf, mem->reply); |
43329771 | 1924 | sc->callback = NULL; |
1925 | callback(sc->callback_data, sc->copy_buf, len); | |
8350fe9b | 1926 | } |
1927 | ||
24382924 | 1928 | static int |
fe4e214f | 1929 | storeEntryValidLength(const StoreEntry * e) |
6602e70e | 1930 | { |
ffe4a367 | 1931 | int diff; |
2285407f | 1932 | int hdr_sz; |
1933 | int content_length; | |
ffe4a367 | 1934 | |
2285407f | 1935 | if (e->mem_obj == NULL) |
ffe4a367 | 1936 | fatal_dump("storeEntryValidLength: NULL mem_obj"); |
1937 | ||
2285407f | 1938 | hdr_sz = e->mem_obj->reply->hdr_sz; |
147d3115 | 1939 | content_length = e->mem_obj->reply->content_length; |
2285407f | 1940 | |
a3d5953d | 1941 | debug(20, 3) ("storeEntryValidLength: Checking '%s'\n", e->key); |
1942 | debug(20, 5) ("storeEntryValidLength: object_len = %d\n", e->object_len); | |
1943 | debug(20, 5) ("storeEntryValidLength: hdr_sz = %d\n", hdr_sz); | |
1944 | debug(20, 5) ("storeEntryValidLength: content_length = %d\n", content_length); | |
ffe4a367 | 1945 | |
2285407f | 1946 | if (content_length == 0) { |
a3d5953d | 1947 | debug(20, 5) ("storeEntryValidLength: Zero content length; assume valid; '%s'\n", |
ffe4a367 | 1948 | e->key); |
1949 | return 1; | |
1950 | } | |
2285407f | 1951 | if (hdr_sz == 0) { |
a3d5953d | 1952 | debug(20, 5) ("storeEntryValidLength: Zero header size; assume valid; '%s'\n", |
ffe4a367 | 1953 | e->key); |
1954 | return 1; | |
1955 | } | |
2285407f | 1956 | diff = hdr_sz + content_length - e->object_len; |
ffe4a367 | 1957 | if (diff != 0) { |
a3d5953d | 1958 | debug(20, 3) ("storeEntryValidLength: %d bytes too %s; '%s'\n", |
ffe4a367 | 1959 | diff < 0 ? -diff : diff, |
1960 | diff < 0 ? "small" : "big", | |
1961 | e->key); | |
1962 | return 0; | |
1963 | } | |
1964 | return 1; | |
1965 | } | |
6602e70e | 1966 | |
d3c12084 | 1967 | #if HAVE_RANDOM |
1968 | #define squid_random random | |
1969 | #elif HAVE_LRAND48 | |
1970 | #define squid_random lrand48 | |
1971 | #else | |
1972 | #define squid_random rand | |
1973 | #endif | |
1974 | ||
63017fae | 1975 | static void |
1976 | storeRandomizeBuckets(void) | |
1977 | { | |
1978 | int i; | |
d3c12084 | 1979 | struct _bucketOrder *b; |
63017fae | 1980 | if (MaintBucketsOrder == NULL) |
d3c12084 | 1981 | MaintBucketsOrder = xcalloc(store_buckets, sizeof(struct _bucketOrder)); |
1982 | for (i = 0; i < store_buckets; i++) { | |
1983 | b = MaintBucketsOrder + i; | |
1984 | b->bucket = (unsigned int) i; | |
1985 | b->index = (int) squid_random(); | |
1986 | } | |
63017fae | 1987 | qsort((char *) MaintBucketsOrder, |
1988 | store_buckets, | |
d3c12084 | 1989 | sizeof(struct _bucketOrder), |
582b6456 | 1990 | (QS *) compareBucketOrder); |
63017fae | 1991 | } |
1992 | ||
66cedb85 | 1993 | static void |
1994 | storeInitHashValues(void) | |
1995 | { | |
1996 | int i; | |
a7e59001 | 1997 | /* Calculate size of hash table (maximum currently 64k buckets). */ |
38792624 | 1998 | i = Config.Swap.maxSize / Config.Store.avgObjectSize; |
86101e40 | 1999 | debug(20, 1) ("Swap maxSize %d KB, estimated %d objects\n", |
66cedb85 | 2000 | Config.Swap.maxSize, i); |
38792624 | 2001 | i /= Config.Store.objectsPerBucket; |
a3d5953d | 2002 | debug(20, 1) ("Target number of buckets: %d\n", i); |
66cedb85 | 2003 | /* ideally the full scan period should be configurable, for the |
2004 | * moment it remains at approximately 24 hours. */ | |
2005 | if (i < 8192) | |
2006 | store_buckets = 7951, store_maintain_rate = 10; | |
2007 | else if (i < 12288) | |
2008 | store_buckets = 12149, store_maintain_rate = 7; | |
2009 | else if (i < 16384) | |
2010 | store_buckets = 16231, store_maintain_rate = 5; | |
2011 | else if (i < 32768) | |
2012 | store_buckets = 33493, store_maintain_rate = 2; | |
2013 | else | |
2014 | store_buckets = 65357, store_maintain_rate = 1; | |
2015 | store_maintain_buckets = 1; | |
63017fae | 2016 | storeRandomizeBuckets(); |
a3d5953d | 2017 | debug(20, 1) ("Using %d Store buckets, maintain %d bucket%s every %d second%s\n", |
66cedb85 | 2018 | store_buckets, |
2019 | store_maintain_buckets, | |
2020 | store_maintain_buckets == 1 ? null_string : "s", | |
2021 | store_maintain_rate, | |
2022 | store_maintain_rate == 1 ? null_string : "s"); | |
a47b9029 | 2023 | debug(20, 1) ("Max Mem size: %d KB\n", Config.Mem.maxSize >> 10); |
2024 | debug(20, 1) ("Max Swap size: %d KB\n", Config.Swap.maxSize); | |
66cedb85 | 2025 | } |
2026 | ||
b8d8561b | 2027 | void |
2028 | storeInit(void) | |
c943f331 | 2029 | { |
234967c9 | 2030 | char *fname = NULL; |
66cedb85 | 2031 | storeInitHashValues(); |
b93bcace | 2032 | store_table = hash_create(urlcmp, store_buckets, hash4); |
b6f794d6 | 2033 | if (strcmp((fname = Config.Log.store), "none") == 0) |
234967c9 | 2034 | storelog_fd = -1; |
2035 | else | |
f17936ab | 2036 | storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); |
234967c9 | 2037 | if (storelog_fd < 0) |
a3d5953d | 2038 | debug(20, 1) ("Store logging disabled\n"); |
e7a22b88 | 2039 | storeVerifySwapDirs(); |
2040 | storeDirOpenSwapLogs(); | |
b6f794d6 | 2041 | if (!opt_zap_disk_store) |
3facc31e | 2042 | storeStartRebuildFromDisk(); |
30a4f2a8 | 2043 | else |
311ea387 | 2044 | store_rebuilding = 0; |
b93bcace | 2045 | all_list.head = all_list.tail = NULL; |
2046 | inmem_list.head = inmem_list.tail = NULL; | |
b1c0cc67 | 2047 | } |
c943f331 | 2048 | |
b8d8561b | 2049 | void |
2050 | storeConfigure(void) | |
b1c0cc67 | 2051 | { |
e954773d | 2052 | int store_mem_high = 0; |
2053 | int store_mem_low = 0; | |
b6f794d6 | 2054 | store_mem_high = (long) (Config.Mem.maxSize / 100) * |
2055 | Config.Mem.highWaterMark; | |
2056 | store_mem_low = (long) (Config.Mem.maxSize / 100) * | |
2057 | Config.Mem.lowWaterMark; | |
090089c4 | 2058 | |
b1c0cc67 | 2059 | store_swap_high = (long) (((float) Config.Swap.maxSize * |
2060 | (float) Config.Swap.highWaterMark) / (float) 100); | |
2061 | store_swap_low = (long) (((float) Config.Swap.maxSize * | |
2062 | (float) Config.Swap.lowWaterMark) / (float) 100); | |
e954773d | 2063 | |
13c72789 | 2064 | store_pages_max = Config.Mem.maxSize / SM_PAGE_SIZE; |
e954773d | 2065 | store_pages_high = store_mem_high / SM_PAGE_SIZE; |
2066 | store_pages_low = store_mem_low / SM_PAGE_SIZE; | |
090089c4 | 2067 | } |
2068 | ||
b8d8561b | 2069 | int |
0ee4272b | 2070 | urlcmp(const char *url1, const char *url2) |
090089c4 | 2071 | { |
2072 | if (!url1 || !url2) | |
6eb42cae | 2073 | fatal_dump("urlcmp: Got a NULL url pointer."); |
090089c4 | 2074 | return (strcmp(url1, url2)); |
2075 | } | |
2076 | ||
090089c4 | 2077 | /* |
2078 | * This routine is to be called by main loop in main.c. | |
2079 | * It removes expired objects on only one bucket for each time called. | |
2080 | * returns the number of objects removed | |
2081 | * | |
2082 | * This should get called 1/s from main(). | |
2083 | */ | |
48f44632 | 2084 | void |
2085 | storeMaintainSwapSpace(void *unused) | |
090089c4 | 2086 | { |
30a4f2a8 | 2087 | static time_t last_time = 0; |
cb6ae8c3 | 2088 | static int bucket_index = 0; |
090089c4 | 2089 | hash_link *link_ptr = NULL, *next = NULL; |
2090 | StoreEntry *e = NULL; | |
2091 | int rm_obj = 0; | |
58104eab | 2092 | int scan_buckets = 0; |
66cedb85 | 2093 | int scan_obj = 0; |
d3c12084 | 2094 | static struct _bucketOrder *b; |
090089c4 | 2095 | |
48f44632 | 2096 | eventAdd("storeMaintain", storeMaintainSwapSpace, NULL, 1); |
30a4f2a8 | 2097 | /* We can't delete objects while rebuilding swap */ |
311ea387 | 2098 | if (store_rebuilding) |
48f44632 | 2099 | return; |
30a4f2a8 | 2100 | |
090089c4 | 2101 | /* Purges expired objects, check one bucket on each calling */ |
66cedb85 | 2102 | if (squid_curtime - last_time >= store_maintain_rate) { |
58104eab | 2103 | for (;;) { |
2104 | if (scan_obj && scan_buckets >= store_maintain_buckets) | |
2105 | break; | |
2106 | if (++scan_buckets > 100) | |
2107 | break; | |
66cedb85 | 2108 | last_time = squid_curtime; |
cb6ae8c3 | 2109 | if (bucket_index >= store_buckets) { |
2110 | bucket_index = 0; | |
66cedb85 | 2111 | scan_revolutions++; |
a3d5953d | 2112 | debug(51, 1) ("Completed %d full expiration scans of store table\n", |
66cedb85 | 2113 | scan_revolutions); |
d25ad3e1 | 2114 | storeRandomizeBuckets(); |
66cedb85 | 2115 | } |
d3c12084 | 2116 | b = MaintBucketsOrder + bucket_index++; |
2117 | next = hash_get_bucket(store_table, b->bucket); | |
cc61958c | 2118 | while ((link_ptr = next)) { |
66cedb85 | 2119 | scan_obj++; |
2120 | next = link_ptr->next; | |
2121 | e = (StoreEntry *) link_ptr; | |
429fdbec | 2122 | if (!storeCheckExpired(e, 1)) |
66cedb85 | 2123 | continue; |
cc61958c | 2124 | rm_obj += storeRelease(e); |
66cedb85 | 2125 | } |
090089c4 | 2126 | } |
2127 | } | |
a3d5953d | 2128 | debug(51, rm_obj ? 2 : 9) ("Removed %d of %d objects from bucket %d\n", |
d3c12084 | 2129 | rm_obj, scan_obj, (int) b->bucket); |
79b5cc5f | 2130 | /* Scan row of hash table each second and free storage if we're |
2131 | * over the high-water mark */ | |
2132 | storeGetSwapSpace(0); | |
090089c4 | 2133 | } |
2134 | ||
090089c4 | 2135 | |
2136 | /* | |
5608850b | 2137 | * storeWriteCleanLogs |
090089c4 | 2138 | * |
2139 | * Writes a "clean" swap log file from in-memory metadata. | |
2140 | */ | |
b8d8561b | 2141 | int |
e102ebda | 2142 | storeWriteCleanLogs(int reopen) |
090089c4 | 2143 | { |
2144 | StoreEntry *e = NULL; | |
9e6c462c | 2145 | int *fd; |
5608850b | 2146 | char *line; |
090089c4 | 2147 | int n = 0; |
2148 | time_t start, stop, r; | |
10389b74 | 2149 | struct stat sb; |
9e6c462c | 2150 | char **cur; |
2151 | char **new; | |
2152 | char **cln; | |
5608850b | 2153 | int dirn; |
30a4f2a8 | 2154 | if (store_rebuilding) { |
a3d5953d | 2155 | debug(20, 1) ("Not currently OK to rewrite swap log.\n"); |
2156 | debug(20, 1) ("storeWriteCleanLogs: Operation aborted.\n"); | |
bb5f11c4 | 2157 | storeDirCloseSwapLogs(); |
090089c4 | 2158 | return 0; |
2159 | } | |
a3d5953d | 2160 | debug(20, 1) ("storeWriteCleanLogs: Starting...\n"); |
97c03d3c | 2161 | start = squid_curtime; |
f1dc9b30 | 2162 | fd = xcalloc(Config.cacheSwap.n_configured, sizeof(int)); |
2163 | cur = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
2164 | new = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
2165 | cln = xcalloc(Config.cacheSwap.n_configured, sizeof(char *)); | |
2166 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { | |
3e347513 | 2167 | fd[dirn] = -1; |
2168 | cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL)); | |
2169 | new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean")); | |
2170 | cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean")); | |
2171 | safeunlink(new[dirn], 1); | |
2172 | safeunlink(cln[dirn], 1); | |
f17936ab | 2173 | fd[dirn] = file_open(new[dirn], |
9e4ad609 | 2174 | O_WRONLY | O_CREAT | O_TRUNC, |
2175 | NULL, | |
2176 | NULL); | |
3e347513 | 2177 | if (fd[dirn] < 0) { |
a3d5953d | 2178 | debug(50, 0) ("storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); |
5608850b | 2179 | continue; |
2180 | } | |
4ac29a5b | 2181 | #if HAVE_FCHMOD |
3e347513 | 2182 | if (stat(cur[dirn], &sb) == 0) |
5608850b | 2183 | fchmod(fd[dirn], sb.st_mode); |
4ac29a5b | 2184 | #endif |
5608850b | 2185 | } |
2186 | line = xcalloc(1, 16384); | |
090089c4 | 2187 | for (e = storeGetFirst(); e; e = storeGetNext()) { |
090089c4 | 2188 | if (e->swap_file_number < 0) |
2189 | continue; | |
8350fe9b | 2190 | if (e->swap_status != SWAPOUT_DONE) |
090089c4 | 2191 | continue; |
2192 | if (e->object_len <= 0) | |
2193 | continue; | |
3ba50d75 | 2194 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) |
0a21bd84 | 2195 | continue; |
2196 | if (BIT_TEST(e->flag, KEY_PRIVATE)) | |
2197 | continue; | |
9e975e4e | 2198 | dirn = storeDirNumber(e->swap_file_number); |
2199 | assert(dirn < Config.cacheSwap.n_configured); | |
5608850b | 2200 | if (fd[dirn] < 0) |
2201 | continue; | |
042461c3 | 2202 | snprintf(line, 16384, "%08x %08x %08x %08x %08x %9d %6d %08x %s\n", |
f5469bce | 2203 | (int) e->swap_file_number, |
2204 | (int) e->timestamp, | |
919877b6 | 2205 | (int) e->lastref, |
f5469bce | 2206 | (int) e->expires, |
2207 | (int) e->lastmod, | |
2208 | e->object_len, | |
994bbf93 | 2209 | e->refcount, |
2210 | e->flag, | |
f5469bce | 2211 | e->url); |
5608850b | 2212 | if (write(fd[dirn], line, strlen(line)) < 0) { |
a3d5953d | 2213 | debug(50, 0) ("storeWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); |
2214 | debug(20, 0) ("storeWriteCleanLogs: Current swap logfile not replaced.\n"); | |
5608850b | 2215 | file_close(fd[dirn]); |
2216 | fd[dirn] = -1; | |
2217 | safeunlink(cln[dirn], 0); | |
2218 | continue; | |
30a4f2a8 | 2219 | } |
5608850b | 2220 | if ((++n & 0x3FFF) == 0) { |
a3d5953d | 2221 | debug(20, 1) (" %7d lines written so far.\n", n); |
090089c4 | 2222 | } |
2223 | } | |
5608850b | 2224 | safe_free(line); |
f1dc9b30 | 2225 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { |
3e347513 | 2226 | file_close(fd[dirn]); |
2227 | fd[dirn] = -1; | |
2228 | if (rename(new[dirn], cur[dirn]) < 0) { | |
a3d5953d | 2229 | debug(50, 0) ("storeWriteCleanLogs: rename failed: %s\n", |
3e347513 | 2230 | xstrerror()); |
2231 | } | |
5608850b | 2232 | } |
2233 | storeDirCloseSwapLogs(); | |
e102ebda | 2234 | if (reopen) |
2235 | storeDirOpenSwapLogs(); | |
97c03d3c | 2236 | stop = squid_curtime; |
090089c4 | 2237 | r = stop - start; |
a3d5953d | 2238 | debug(20, 1) (" Finished. Wrote %d lines.\n", n); |
2239 | debug(20, 1) (" Took %d seconds (%6.1lf lines/sec).\n", | |
090089c4 | 2240 | r > 0 ? r : 0, (double) n / (r > 0 ? r : 1)); |
0a0bf5db | 2241 | /* touch a timestamp file if we're not still validating */ |
f1dc9b30 | 2242 | for (dirn = 0; dirn < Config.cacheSwap.n_configured; dirn++) { |
f182d1c5 | 2243 | if (!store_rebuilding) |
f17936ab | 2244 | file_close(file_open(cln[dirn], |
3e347513 | 2245 | O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL)); |
5608850b | 2246 | safe_free(cur[dirn]); |
2247 | safe_free(new[dirn]); | |
2248 | safe_free(cln[dirn]); | |
0a0bf5db | 2249 | } |
9e6c462c | 2250 | safe_free(cur); |
2251 | safe_free(new); | |
2252 | safe_free(cln); | |
2253 | safe_free(fd); | |
090089c4 | 2254 | return n; |
2255 | } | |
2256 | ||
b8d8561b | 2257 | int |
fe4e214f | 2258 | storePendingNClients(const StoreEntry * e) |
090089c4 | 2259 | { |
2260 | int npend = 0; | |
52d4522b | 2261 | MemObject *mem = e->mem_obj; |
43329771 | 2262 | store_client *sc; |
2263 | store_client *nx = NULL; | |
52d4522b | 2264 | if (mem == NULL) |
090089c4 | 2265 | return 0; |
5d86029a | 2266 | for (sc = mem->clients; sc; sc = nx) { |
43329771 | 2267 | nx = sc->next; |
2268 | if (sc->callback_data == NULL) | |
52d4522b | 2269 | continue; |
b012353a | 2270 | npend++; |
090089c4 | 2271 | } |
2272 | return npend; | |
2273 | } | |
d8b45066 | 2274 | |
b8d8561b | 2275 | void |
2276 | storeRotateLog(void) | |
d8b45066 | 2277 | { |
2278 | char *fname = NULL; | |
2279 | int i; | |
95d659f0 | 2280 | LOCAL_ARRAY(char, from, MAXPATHLEN); |
2281 | LOCAL_ARRAY(char, to, MAXPATHLEN); | |
88738790 | 2282 | #ifdef S_ISREG |
2edc2504 | 2283 | struct stat sb; |
88738790 | 2284 | #endif |
d8b45066 | 2285 | |
234967c9 | 2286 | if (storelog_fd > -1) { |
2287 | file_close(storelog_fd); | |
2288 | storelog_fd = -1; | |
2289 | } | |
b6f794d6 | 2290 | if ((fname = Config.Log.store) == NULL) |
2d25dc1a | 2291 | return; |
234967c9 | 2292 | if (strcmp(fname, "none") == 0) |
2293 | return; | |
71d95578 | 2294 | #ifdef S_ISREG |
2edc2504 | 2295 | if (stat(fname, &sb) == 0) |
2296 | if (S_ISREG(sb.st_mode) == 0) | |
2297 | return; | |
71d95578 | 2298 | #endif |
234967c9 | 2299 | |
a3d5953d | 2300 | debug(20, 1) ("storeRotateLog: Rotating.\n"); |
d8b45066 | 2301 | |
2302 | /* Rotate numbers 0 through N up one */ | |
b6f794d6 | 2303 | for (i = Config.Log.rotateNumber; i > 1;) { |
2d25dc1a | 2304 | i--; |
8350fe9b | 2305 | snprintf(from, MAXPATHLEN, "%s.%d", fname, i - 1); |
042461c3 | 2306 | snprintf(to, MAXPATHLEN, "%s.%d", fname, i); |
2d25dc1a | 2307 | rename(from, to); |
d8b45066 | 2308 | } |
2309 | /* Rotate the current log to .0 */ | |
b6f794d6 | 2310 | if (Config.Log.rotateNumber > 0) { |
8350fe9b | 2311 | snprintf(to, MAXPATHLEN, "%s.%d", fname, 0); |
2d25dc1a | 2312 | rename(fname, to); |
d8b45066 | 2313 | } |
f17936ab | 2314 | storelog_fd = file_open(fname, O_WRONLY | O_CREAT, NULL, NULL); |
234967c9 | 2315 | if (storelog_fd < 0) { |
a3d5953d | 2316 | debug(50, 0) ("storeRotateLog: %s: %s\n", fname, xstrerror()); |
2317 | debug(20, 1) ("Store logging disabled\n"); | |
234967c9 | 2318 | } |
d8b45066 | 2319 | } |
30a4f2a8 | 2320 | |
56f29785 | 2321 | static int |
8350fe9b | 2322 | storeKeepInMemory(const StoreEntry * e) |
56f29785 | 2323 | { |
8350fe9b | 2324 | MemObject *mem = e->mem_obj; |
2325 | if (mem == NULL) | |
56f29785 | 2326 | return 0; |
8350fe9b | 2327 | if (mem->data == NULL) |
2328 | return 0; | |
2329 | return mem->inmem_lo == 0; | |
56f29785 | 2330 | } |
2331 | ||
b8d8561b | 2332 | static int |
429fdbec | 2333 | storeCheckExpired(const StoreEntry * e, int check_lru_age) |
620da955 | 2334 | { |
429fdbec | 2335 | time_t max_age; |
620da955 | 2336 | if (storeEntryLocked(e)) |
2337 | return 0; | |
7cc1830f | 2338 | if (BIT_TEST(e->flag, ENTRY_NEGCACHED) && squid_curtime >= e->expires) |
2339 | return 1; | |
429fdbec | 2340 | if (!check_lru_age) |
2341 | return 0; | |
2342 | if ((max_age = storeExpiredReferenceAge()) <= 0) | |
db9ccd5a | 2343 | return 0; |
2344 | if (squid_curtime - e->lastref > max_age) | |
66cedb85 | 2345 | return 1; |
66cedb85 | 2346 | return 0; |
620da955 | 2347 | } |
2348 | ||
db9ccd5a | 2349 | /* |
2350 | * storeExpiredReferenceAge | |
2351 | * | |
429fdbec | 2352 | * The LRU age is scaled exponentially between 1 minute and |
2353 | * Config.referenceAge , when store_swap_low < store_swap_size < | |
2354 | * store_swap_high. This keeps store_swap_size within the low and high | |
2355 | * water marks. If the cache is very busy then store_swap_size stays | |
2356 | * closer to the low water mark, if it is not busy, then it will stay | |
2357 | * near the high water mark. The LRU age value can be examined on the | |
2358 | * cachemgr 'info' page. | |
fbdfb3cf | 2359 | */ |
35feb4aa | 2360 | time_t |
58104eab | 2361 | storeExpiredReferenceAge(void) |
2362 | { | |
58104eab | 2363 | double x; |
2364 | double z; | |
2365 | time_t age; | |
429fdbec | 2366 | if (Config.referenceAge == 0) |
2367 | return 0; | |
2368 | x = (double) (store_swap_high - store_swap_size) / (store_swap_high - store_swap_low); | |
2369 | x = x < 0.0 ? 0.0 : x > 1.0 ? 1.0 : x; | |
2370 | z = pow((double) Config.referenceAge, x); | |
58104eab | 2371 | age = (time_t) (z * 60.0); |
2372 | if (age < 60) | |
2373 | age = 60; | |
2374 | else if (age > 31536000) | |
2375 | age = 31536000; | |
2376 | return age; | |
0a362c34 | 2377 | } |
2edc2504 | 2378 | |
b8d8561b | 2379 | void |
2380 | storeCloseLog(void) | |
457bedbd | 2381 | { |
15be2259 | 2382 | if (storelog_fd >= 0) |
40fcf5d4 | 2383 | file_close(storelog_fd); |
457bedbd | 2384 | } |
79b5cc5f | 2385 | |
b8d8561b | 2386 | void |
2387 | storeNegativeCache(StoreEntry * e) | |
79b5cc5f | 2388 | { |
ff5731b1 | 2389 | e->expires = squid_curtime + Config.negativeTtl; |
2390 | BIT_SET(e->flag, ENTRY_NEGCACHED); | |
79b5cc5f | 2391 | } |
0a21bd84 | 2392 | |
2393 | void | |
2394 | storeFreeMemory(void) | |
2395 | { | |
2396 | StoreEntry *e; | |
df92bd2a | 2397 | StoreEntry **list; |
2398 | int i = 0; | |
ea6b05e3 | 2399 | int j; |
56e15c50 | 2400 | list = xcalloc(meta_data.store_entries, sizeof(StoreEntry *)); |
df92bd2a | 2401 | e = (StoreEntry *) hash_first(store_table); |
2402 | while (e && i < meta_data.store_entries) { | |
2403 | *(list + i) = e; | |
2404 | i++; | |
2405 | e = (StoreEntry *) hash_next(store_table); | |
2406 | } | |
ea6b05e3 | 2407 | for (j = 0; j < i; j++) |
2408 | destroy_StoreEntry(*(list + j)); | |
df92bd2a | 2409 | xfree(list); |
0a21bd84 | 2410 | hashFreeMemory(store_table); |
afe95a7e | 2411 | store_table = NULL; |
e161be2e | 2412 | safe_free(MaintBucketsOrder); |
0a21bd84 | 2413 | } |
a7e59001 | 2414 | |
2415 | int | |
2416 | expiresMoreThan(time_t expires, time_t when) | |
2417 | { | |
48f44632 | 2418 | if (expires < 0) /* No Expires given */ |
2419 | return 1; | |
2420 | return (expires > (squid_curtime + when)); | |
a7e59001 | 2421 | } |
fe54d06d | 2422 | |
2423 | int | |
2424 | storeEntryValidToSend(StoreEntry * e) | |
2425 | { | |
2426 | if (BIT_TEST(e->flag, RELEASE_REQUEST)) | |
2427 | return 0; | |
2428 | if (BIT_TEST(e->flag, ENTRY_NEGCACHED)) | |
44f78c24 | 2429 | if (e->expires <= squid_curtime) |
fe54d06d | 2430 | return 0; |
2431 | if (e->store_status == STORE_ABORTED) | |
2432 | return 0; | |
2433 | return 1; | |
2434 | } | |
62663274 | 2435 | |
ca98227c | 2436 | void |
2437 | storeTimestampsSet(StoreEntry * entry) | |
2438 | { | |
2439 | time_t served_date = -1; | |
2440 | struct _http_reply *reply = entry->mem_obj->reply; | |
2441 | served_date = reply->date > -1 ? reply->date : squid_curtime; | |
2442 | entry->expires = reply->expires; | |
2443 | if (reply->last_modified > -1) | |
2444 | entry->lastmod = reply->last_modified; | |
2445 | else | |
2446 | entry->lastmod = served_date; | |
2447 | entry->timestamp = served_date; | |
2448 | } | |
429fdbec | 2449 | |
2450 | #define FILENO_STACK_SIZE 128 | |
2451 | static int fileno_stack[FILENO_STACK_SIZE]; | |
429fdbec | 2452 | |
2453 | static int | |
2454 | storeGetUnusedFileno(void) | |
2455 | { | |
9e4ad609 | 2456 | int fn; |
429fdbec | 2457 | if (fileno_stack_count < 1) |
2458 | return -1; | |
9e4ad609 | 2459 | fn = fileno_stack[--fileno_stack_count]; |
2460 | storeDirMapBitSet(fn); | |
2461 | return fn; | |
429fdbec | 2462 | } |
2463 | ||
2464 | static void | |
2465 | storePutUnusedFileno(int fileno) | |
2466 | { | |
bf33177b | 2467 | assert(storeDirMapBitTest(fileno)); |
9e4ad609 | 2468 | storeDirMapBitReset(fileno); |
429fdbec | 2469 | if (fileno_stack_count < FILENO_STACK_SIZE) |
2470 | fileno_stack[fileno_stack_count++] = fileno; | |
2471 | else | |
2472 | unlinkdUnlink(storeSwapFullPath(fileno, NULL)); | |
2473 | } | |
bfcaf585 | 2474 | |
2475 | void | |
2476 | storeRegisterAbort(StoreEntry * e, STABH * cb, void *data) | |
2477 | { | |
2478 | MemObject *mem = e->mem_obj; | |
2479 | assert(mem); | |
2480 | assert(mem->abort.callback == NULL); | |
2481 | mem->abort.callback = cb; | |
2482 | mem->abort.data = data; | |
2483 | } | |
2484 | ||
2485 | void | |
2486 | storeUnregisterAbort(StoreEntry * e) | |
2487 | { | |
2488 | MemObject *mem = e->mem_obj; | |
2489 | assert(mem); | |
2490 | mem->abort.callback = NULL; | |
2491 | } | |
88738790 | 2492 | |
2493 | void | |
2494 | storeMemObjectDump(MemObject * mem) | |
2495 | { | |
2496 | debug(20, 1) ("MemObject->data: %p\n", | |
2497 | mem->data); | |
88738790 | 2498 | debug(20, 1) ("MemObject->start_ping: %d.%06d\n", |
2499 | mem->start_ping.tv_sec, | |
2500 | mem->start_ping.tv_usec); | |
8350fe9b | 2501 | debug(20, 1) ("MemObject->inmem_hi: %d\n", |
2502 | mem->inmem_hi); | |
2503 | debug(20, 1) ("MemObject->inmem_lo: %d\n", | |
2504 | mem->inmem_lo); | |
88738790 | 2505 | debug(20, 1) ("MemObject->clients: %p\n", |
2506 | mem->clients); | |
2507 | debug(20, 1) ("MemObject->nclients: %d\n", | |
2508 | mem->nclients); | |
8350fe9b | 2509 | debug(20, 1) ("MemObject->swapout.fd: %d\n", |
2510 | mem->swapout.fd); | |
88738790 | 2511 | debug(20, 1) ("MemObject->reply: %p\n", |
2512 | mem->reply); | |
2513 | debug(20, 1) ("MemObject->request: %p\n", | |
2514 | mem->request); | |
2515 | debug(20, 1) ("MemObject->log_url: %p %s\n", | |
2516 | mem->log_url, | |
2517 | checkNullString(mem->log_url)); | |
2518 | } | |
8350fe9b | 2519 | |
2520 | static void | |
b93bcace | 2521 | storeEntryListAdd(StoreEntry * e, dlink_node * m, dlink_list * list) |
8350fe9b | 2522 | { |
b93bcace | 2523 | debug(20, 3) ("storeEntryListAdd: %s\n", e->url); |
2524 | m->data = e; | |
8350fe9b | 2525 | m->prev = NULL; |
b93bcace | 2526 | m->next = list->head; |
2527 | if (list->head) | |
2528 | list->head->prev = m; | |
2529 | list->head = m; | |
2530 | if (list->tail == NULL) | |
2531 | list->tail = m; | |
8350fe9b | 2532 | } |
2533 | ||
2534 | static void | |
b93bcace | 2535 | storeEntryListDelete(dlink_node * m, dlink_list * list) |
8350fe9b | 2536 | { |
8350fe9b | 2537 | if (m->next) |
2538 | m->next->prev = m->prev; | |
2539 | if (m->prev) | |
2540 | m->prev->next = m->next; | |
b93bcace | 2541 | if (m == list->head) |
2542 | list->head = m->next; | |
2543 | if (m == list->tail) | |
2544 | list->tail = m->prev; | |
8350fe9b | 2545 | } |
2546 | ||
2547 | /* NOTE, this function assumes only two mem states */ | |
2548 | static void | |
2549 | storeSetMemStatus(StoreEntry * e, int new_status) | |
2550 | { | |
b93bcace | 2551 | MemObject *mem = e->mem_obj; |
8350fe9b | 2552 | if (new_status == e->mem_status) |
2553 | return; | |
b93bcace | 2554 | assert(mem != NULL); |
2555 | if (new_status == IN_MEMORY) { | |
2556 | assert(mem->inmem_lo == 0); | |
2557 | storeEntryListAdd(e, &mem->lru, &inmem_list); | |
2558 | meta_data.hot_vm++; | |
2559 | } else { | |
2560 | storeEntryListDelete(&mem->lru, &inmem_list); | |
2561 | meta_data.hot_vm--; | |
2562 | } | |
8350fe9b | 2563 | e->mem_status = new_status; |
2564 | } | |
6e86c3e8 | 2565 | |
2566 | static void | |
2567 | storeSwapOutFileClose(StoreEntry * e) | |
2568 | { | |
bb5f11c4 | 2569 | MemObject *mem = e->mem_obj; |
2570 | if (mem->swapout.fd > -1) | |
2571 | file_close(mem->swapout.fd); | |
2572 | mem->swapout.fd = -1; | |
6e86c3e8 | 2573 | storeUnlockObject(e); |
2574 | } |