]>
Commit | Line | Data |
---|---|---|
9cef6668 | 1 | |
2 | /* | |
adcceb47 | 3 | * $Id: store_swapout.cc,v 1.86 2002/04/13 14:16:04 hno Exp $ |
9cef6668 | 4 | * |
5 | * DEBUG: section 20 Storage Manager Swapout Functions | |
6 | * AUTHOR: Duane Wessels | |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
9cef6668 | 9 | * ---------------------------------------------------------- |
10 | * | |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
9cef6668 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
f09f5b26 | 36 | #include "squid.h" |
37 | ||
bbcd7374 | 38 | static off_t storeSwapOutObjectBytesOnDisk(const MemObject *); |
d54f0ab3 | 39 | static void storeSwapOutStart(StoreEntry * e); |
2391a162 | 40 | static STIOCB storeSwapOutFileClosed; |
cd748f27 | 41 | static STIOCB storeSwapOutFileNotify; |
f09f5b26 | 42 | |
43 | /* start swapping object to disk */ | |
d54f0ab3 | 44 | static void |
f09f5b26 | 45 | storeSwapOutStart(StoreEntry * e) |
46 | { | |
2391a162 | 47 | generic_cbdata *c; |
48 | MemObject *mem = e->mem_obj; | |
49 | int swap_hdr_sz = 0; | |
50 | tlv *tlv_list; | |
51 | char *buf; | |
52 | assert(mem); | |
cd748f27 | 53 | /* Build the swap metadata, so the filesystem will know how much |
54 | * metadata there is to store | |
55 | */ | |
56 | debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n", | |
57 | storeUrl(e), e->swap_dirn, e->swap_filen); | |
58 | e->swap_status = SWAPOUT_WRITING; | |
59 | tlv_list = storeSwapMetaBuild(e); | |
60 | buf = storeSwapMetaPack(tlv_list, &swap_hdr_sz); | |
61 | storeSwapTLVFree(tlv_list); | |
62 | mem->swap_hdr_sz = (size_t) swap_hdr_sz; | |
63 | /* Create the swap file */ | |
72711e31 | 64 | c = cbdataAlloc(generic_cbdata); |
2391a162 | 65 | c->data = e; |
cd748f27 | 66 | mem->swapout.sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c); |
3fc29499 | 67 | if (NULL == mem->swapout.sio) { |
68 | e->swap_status = SWAPOUT_NONE; | |
3fc29499 | 69 | cbdataFree(c); |
cd748f27 | 70 | xfree(buf); |
284ac611 | 71 | storeLog(STORE_LOG_SWAPOUTFAIL, e); |
3fc29499 | 72 | return; |
73 | } | |
a4b8110e | 74 | storeLockObject(e); /* Don't lock until after create, or the replacement |
75 | * code might get confused */ | |
cd748f27 | 76 | /* Pick up the file number if it was assigned immediately */ |
77 | e->swap_filen = mem->swapout.sio->swap_filen; | |
78 | e->swap_dirn = mem->swapout.sio->swap_dirn; | |
79 | /* write out the swap metadata */ | |
3fc29499 | 80 | cbdataLock(mem->swapout.sio); |
50f4d6ae | 81 | storeWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree); |
f09f5b26 | 82 | } |
83 | ||
cd748f27 | 84 | static void |
85 | storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio) | |
86 | { | |
87 | generic_cbdata *c = data; | |
88 | StoreEntry *e = c->data; | |
89 | MemObject *mem = e->mem_obj; | |
90 | assert(e->swap_status == SWAPOUT_WRITING); | |
91 | assert(mem); | |
92 | assert(mem->swapout.sio == sio); | |
93 | assert(errflag == 0); | |
94 | e->swap_filen = mem->swapout.sio->swap_filen; | |
95 | e->swap_dirn = mem->swapout.sio->swap_dirn; | |
96 | } | |
97 | ||
f09f5b26 | 98 | void |
2391a162 | 99 | storeSwapOut(StoreEntry * e) |
f09f5b26 | 100 | { |
101 | MemObject *mem = e->mem_obj; | |
102 | off_t lowest_offset; | |
103 | off_t new_mem_lo; | |
2391a162 | 104 | off_t on_disk = 0; |
86105c45 | 105 | ssize_t swapout_size; |
f09f5b26 | 106 | ssize_t swap_buf_len; |
77b32a34 | 107 | if (mem == NULL) |
108 | return; | |
f09f5b26 | 109 | /* should we swap something out to disk? */ |
2391a162 | 110 | debug(20, 7) ("storeSwapOut: %s\n", storeUrl(e)); |
111 | debug(20, 7) ("storeSwapOut: store_status = %s\n", | |
f09f5b26 | 112 | storeStatusStr[e->store_status]); |
b7fe0ab0 | 113 | if (EBIT_TEST(e->flags, ENTRY_ABORTED)) { |
d46a87a8 | 114 | assert(EBIT_TEST(e->flags, RELEASE_REQUEST)); |
f09f5b26 | 115 | storeSwapOutFileClose(e); |
116 | return; | |
117 | } | |
557b96c0 | 118 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) { |
119 | debug(20, 3) ("storeSwapOut: %s SPECIAL\n", storeUrl(e)); | |
120 | return; | |
121 | } | |
2391a162 | 122 | debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n", |
f09f5b26 | 123 | (int) mem->inmem_lo); |
2391a162 | 124 | debug(20, 7) ("storeSwapOut: mem->inmem_hi = %d\n", |
f09f5b26 | 125 | (int) mem->inmem_hi); |
2391a162 | 126 | debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n", |
f09f5b26 | 127 | (int) mem->swapout.queue_offset); |
2391a162 | 128 | if (mem->swapout.sio) |
60df005c | 129 | debug(20, 7) ("storeSwapOut: storeOffset() = %d\n", |
2391a162 | 130 | (int) storeOffset(mem->swapout.sio)); |
f09f5b26 | 131 | assert(mem->inmem_hi >= mem->swapout.queue_offset); |
f09f5b26 | 132 | lowest_offset = storeLowestMemReaderOffset(e); |
2391a162 | 133 | debug(20, 7) ("storeSwapOut: lowest_offset = %d\n", |
f09f5b26 | 134 | (int) lowest_offset); |
cd748f27 | 135 | /* |
136 | * Grab the swapout_size and check to see whether we're going to defer | |
137 | * the swapout based upon size | |
138 | */ | |
86105c45 | 139 | swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset); |
cd748f27 | 140 | if ((e->store_status != STORE_OK) && (swapout_size < store_maxobjsize)) { |
73ebf122 | 141 | /* |
142 | * NOTE: the store_maxobjsize here is the max of optional | |
143 | * max-size values from 'cache_dir' lines. It is not the | |
144 | * same as 'maximum_object_size'. By default, store_maxobjsize | |
145 | * will be set to -1. However, I am worried that this | |
146 | * deferance may consume a lot of memory in some cases. | |
147 | * It would be good to make this decision based on reply | |
148 | * content-length, rather than wait to accumulate huge | |
149 | * amounts of object data in memory. | |
150 | */ | |
a4b8110e | 151 | debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); |
152 | return; | |
cd748f27 | 153 | } |
50a49a6f | 154 | /* |
155 | * Careful. lowest_offset can be greater than inmem_hi, such | |
156 | * as in the case of a range request. | |
157 | */ | |
158 | if (mem->inmem_hi < lowest_offset) | |
159 | new_mem_lo = lowest_offset; | |
7981509a | 160 | else if (mem->inmem_hi - mem->inmem_lo > Config.Store.maxInMemObjSize) |
50a49a6f | 161 | new_mem_lo = lowest_offset; |
279447ff | 162 | else |
50a49a6f | 163 | new_mem_lo = mem->inmem_lo; |
c2725718 | 164 | assert(new_mem_lo >= mem->inmem_lo); |
eb824054 | 165 | if (storeSwapOutAble(e)) { |
1f38f50a | 166 | /* |
167 | * We should only free up to what we know has been written | |
168 | * to disk, not what has been queued for writing. Otherwise | |
169 | * there will be a chunk of the data which is not in memory | |
170 | * and is not yet on disk. | |
1a6c39bc | 171 | * The -1 makes sure the page isn't freed until storeSwapOut has |
172 | * walked to the next page. (mem->swapout.memnode) | |
1f38f50a | 173 | */ |
1a6c39bc | 174 | if ((on_disk = storeSwapOutObjectBytesOnDisk(mem)) - 1 < new_mem_lo) |
175 | new_mem_lo = on_disk - 1; | |
c1dd71ae | 176 | if (new_mem_lo == -1) |
177 | new_mem_lo = 0; /* the above might become -1 */ | |
279447ff | 178 | } else if (new_mem_lo > 0) { |
1f38f50a | 179 | /* |
279447ff | 180 | * Its not swap-able, and we're about to delete a chunk, |
181 | * so we must make it PRIVATE. This is tricky/ugly because | |
182 | * for the most part, we treat swapable == cachable here. | |
1f38f50a | 183 | */ |
eb824054 | 184 | storeReleaseRequest(e); |
1f38f50a | 185 | } |
18fe65d0 | 186 | stmemFreeDataUpto(&mem->data_hdr, new_mem_lo); |
f09f5b26 | 187 | mem->inmem_lo = new_mem_lo; |
adcceb47 | 188 | #if SIZEOF_OFF_T == 4 |
189 | if (mem->inmem_hi > 0x7FFF0000) { | |
190 | debug(20, 0) ("WARNING: preventing off_t overflow for %s\n", storeUrl(e)); | |
191 | storeAbort(e); | |
192 | return; | |
193 | } | |
194 | #endif | |
c2725718 | 195 | if (e->swap_status == SWAPOUT_WRITING) |
2391a162 | 196 | assert(mem->inmem_lo <= on_disk); |
c2725718 | 197 | if (!storeSwapOutAble(e)) |
1b72bda1 | 198 | return; |
2391a162 | 199 | debug(20, 7) ("storeSwapOut: swapout_size = %d\n", |
f09f5b26 | 200 | (int) swapout_size); |
1b72bda1 | 201 | if (swapout_size == 0) { |
614a44a6 | 202 | if (e->store_status == STORE_OK) |
1b72bda1 | 203 | storeSwapOutFileClose(e); |
614a44a6 | 204 | return; /* Nevermore! */ |
1b72bda1 | 205 | } |
c47511fd | 206 | if (e->store_status == STORE_PENDING) { |
207 | /* wait for a full block to write */ | |
cd748f27 | 208 | if (swapout_size < SM_PAGE_SIZE) |
c47511fd | 209 | return; |
210 | /* | |
211 | * Wait until we are below the disk FD limit, only if the | |
212 | * next server-side read won't be deferred. | |
213 | */ | |
214 | if (storeTooManyDiskFilesOpen() && !fwdCheckDeferRead(-1, e)) | |
215 | return; | |
216 | } | |
2391a162 | 217 | /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ |
f09f5b26 | 218 | if (e->swap_status == SWAPOUT_NONE) { |
2391a162 | 219 | assert(mem->swapout.sio == NULL); |
c2725718 | 220 | assert(mem->inmem_lo == 0); |
f09f5b26 | 221 | if (storeCheckCachable(e)) |
222 | storeSwapOutStart(e); | |
2391a162 | 223 | else |
224 | return; | |
614a44a6 | 225 | /* ENTRY_CACHABLE will be cleared and we'll never get here again */ |
f09f5b26 | 226 | } |
3fc29499 | 227 | if (NULL == mem->swapout.sio) |
228 | return; | |
614a44a6 | 229 | do { |
a4b8110e | 230 | /* |
cd748f27 | 231 | * Evil hack time. |
232 | * We are paging out to disk in page size chunks. however, later on when | |
233 | * we update the queue position, we might not have a page (I *think*), | |
234 | * so we do the actual page update here. | |
a4b8110e | 235 | */ |
cd748f27 | 236 | |
a4b8110e | 237 | if (mem->swapout.memnode == NULL) { |
238 | /* We need to swap out the first page */ | |
cd748f27 | 239 | mem->swapout.memnode = mem->data_hdr.head; |
240 | } else { | |
a4b8110e | 241 | /* We need to swap out the next page */ |
cd748f27 | 242 | mem->swapout.memnode = mem->swapout.memnode->next; |
614a44a6 | 243 | } |
cd748f27 | 244 | /* |
245 | * Get the length of this buffer. We are assuming(!) that the buffer | |
246 | * length won't change on this buffer, or things are going to be very | |
247 | * strange. I think that after the copy to a buffer is done, the buffer | |
248 | * size should stay fixed regardless so that this code isn't confused, | |
249 | * but we can look at this at a later date or whenever the code results | |
250 | * in bad swapouts, whichever happens first. :-) | |
251 | */ | |
252 | swap_buf_len = mem->swapout.memnode->len; | |
253 | ||
614a44a6 | 254 | debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); |
255 | assert(swap_buf_len > 0); | |
ed19251a | 256 | debug(20, 3) ("storeSwapOut: swapping out %ld bytes from %ld\n", |
257 | (long int) swap_buf_len, (long int) mem->swapout.queue_offset); | |
614a44a6 | 258 | mem->swapout.queue_offset += swap_buf_len; |
cd748f27 | 259 | storeWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL); |
614a44a6 | 260 | /* the storeWrite() call might generate an error */ |
261 | if (e->swap_status != SWAPOUT_WRITING) | |
262 | break; | |
86105c45 | 263 | swapout_size = (ssize_t) (mem->inmem_hi - mem->swapout.queue_offset); |
1792fbdb | 264 | if (e->store_status == STORE_PENDING) |
cd748f27 | 265 | if (swapout_size < SM_PAGE_SIZE) |
1792fbdb | 266 | break; |
267 | } while (swapout_size > 0); | |
3fc29499 | 268 | if (NULL == mem->swapout.sio) |
269 | /* oops, we're not swapping out any more */ | |
270 | return; | |
1792fbdb | 271 | if (e->store_status == STORE_OK) { |
272 | /* | |
273 | * If the state is STORE_OK, then all data must have been given | |
274 | * to the filesystem at this point because storeSwapOut() is | |
275 | * not going to be called again for this entry. | |
276 | */ | |
277 | assert(mem->inmem_hi == mem->swapout.queue_offset); | |
278 | storeSwapOutFileClose(e); | |
279 | } | |
f09f5b26 | 280 | } |
281 | ||
282 | void | |
283 | storeSwapOutFileClose(StoreEntry * e) | |
284 | { | |
285 | MemObject *mem = e->mem_obj; | |
4215b049 | 286 | assert(mem != NULL); |
186477c1 | 287 | debug(20, 3) ("storeSwapOutFileClose: %s\n", storeKeyText(e->hash.key)); |
5bd1abac | 288 | debug(20, 3) ("storeSwapOutFileClose: sio = %p\n", mem->swapout.sio); |
2391a162 | 289 | if (mem->swapout.sio == NULL) |
25354045 | 290 | return; |
2391a162 | 291 | storeClose(mem->swapout.sio); |
f09f5b26 | 292 | } |
293 | ||
294 | static void | |
2391a162 | 295 | storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio) |
f09f5b26 | 296 | { |
2391a162 | 297 | generic_cbdata *c = data; |
298 | StoreEntry *e = c->data; | |
25354045 | 299 | MemObject *mem = e->mem_obj; |
2391a162 | 300 | assert(e->swap_status == SWAPOUT_WRITING); |
301 | cbdataFree(c); | |
302 | if (errflag) { | |
cd748f27 | 303 | debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n", |
304 | e->swap_dirn, e->swap_filen, errflag, xstrerror()); | |
2391a162 | 305 | if (errflag == DISK_NO_SPACE_LEFT) { |
cd748f27 | 306 | storeDirDiskFull(e->swap_dirn); |
2391a162 | 307 | storeDirConfigure(); |
308 | storeConfigure(); | |
309 | } | |
cd748f27 | 310 | if (e->swap_filen > 0) |
311 | storeUnlink(e); | |
a4b8110e | 312 | e->swap_filen = -1; |
313 | e->swap_dirn = -1; | |
2391a162 | 314 | e->swap_status = SWAPOUT_NONE; |
cd748f27 | 315 | storeReleaseRequest(e); |
2391a162 | 316 | } else { |
317 | /* swapping complete */ | |
cd748f27 | 318 | debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n", |
319 | storeUrl(e), e->swap_dirn, e->swap_filen); | |
2391a162 | 320 | e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz; |
321 | e->swap_status = SWAPOUT_DONE; | |
cd748f27 | 322 | storeDirUpdateSwapSize(&Config.cacheSwap.swapDirs[e->swap_dirn], e->swap_file_sz, 1); |
2391a162 | 323 | if (storeCheckCachable(e)) { |
324 | storeLog(STORE_LOG_SWAPOUT, e); | |
325 | storeDirSwapLog(e, SWAP_LOG_ADD); | |
326 | } | |
83704487 | 327 | statCounter.swap.outs++; |
f09f5b26 | 328 | } |
5bd1abac | 329 | debug(20, 3) ("storeSwapOutFileClosed: %s:%d\n", __FILE__, __LINE__); |
330 | mem->swapout.sio = NULL; | |
2391a162 | 331 | cbdataUnlock(sio); |
5bd1abac | 332 | storeUnlockObject(e); |
f09f5b26 | 333 | } |
61038223 | 334 | |
bbcd7374 | 335 | /* |
336 | * How much of the object data is on the disk? | |
337 | */ | |
338 | static off_t | |
339 | storeSwapOutObjectBytesOnDisk(const MemObject * mem) | |
340 | { | |
341 | /* | |
2391a162 | 342 | * NOTE: storeOffset() represents the disk file size, |
bbcd7374 | 343 | * not the amount of object data on disk. |
344 | * | |
345 | * If we don't have at least 'swap_hdr_sz' bytes | |
346 | * then none of the object data is on disk. | |
347 | * | |
348 | * This should still be safe if swap_hdr_sz == 0, | |
349 | * meaning we haven't even opened the swapout file | |
350 | * yet. | |
351 | */ | |
2391a162 | 352 | off_t nwritten; |
60df005c | 353 | if (mem->swapout.sio == NULL) |
bbcd7374 | 354 | return 0; |
2391a162 | 355 | nwritten = storeOffset(mem->swapout.sio); |
356 | if (nwritten <= mem->swap_hdr_sz) | |
357 | return 0; | |
358 | return nwritten - mem->swap_hdr_sz; | |
bbcd7374 | 359 | } |
c2725718 | 360 | |
361 | /* | |
362 | * Is this entry a candidate for writing to disk? | |
363 | */ | |
5f25e839 | 364 | int |
c2725718 | 365 | storeSwapOutAble(const StoreEntry * e) |
366 | { | |
06d2839d | 367 | dlink_node *node; |
2391a162 | 368 | if (e->mem_obj->swapout.sio != NULL) |
c2725718 | 369 | return 1; |
370 | if (e->mem_obj->inmem_lo > 0) | |
371 | return 0; | |
5f25e839 | 372 | /* |
373 | * If there are DISK clients, we must write to disk | |
374 | * even if its not cachable | |
375 | */ | |
06d2839d | 376 | for (node = e->mem_obj->clients.head; node; node = node->next) { |
a4b8110e | 377 | if (((store_client *) node->data)->type == STORE_DISK_CLIENT) |
378 | return 1; | |
06d2839d | 379 | } |
a56b2212 | 380 | /* Don't pollute the disk with icons and other special entries */ |
381 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) | |
382 | return 0; | |
d46a87a8 | 383 | return EBIT_TEST(e->flags, ENTRY_CACHABLE); |
c2725718 | 384 | } |