]>
Commit | Line | Data |
---|---|---|
9cef6668 | 1 | |
2 | /* | |
ad8f212f | 3 | * $Id: store_swapout.cc,v 1.98 2004/02/12 02:31:55 adrian Exp $ |
9cef6668 | 4 | * |
5 | * DEBUG: section 20 Storage Manager Swapout Functions | |
6 | * AUTHOR: Duane Wessels | |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
9cef6668 | 9 | * ---------------------------------------------------------- |
10 | * | |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
9cef6668 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
f09f5b26 | 36 | #include "squid.h" |
c8be6d7b | 37 | #include "StoreClient.h" |
e6ccf245 | 38 | #include "Store.h" |
528b2c61 | 39 | /* FIXME: Abstract the use of this more */ |
40 | #include "mem_node.h" | |
41 | #include "MemObject.h" | |
d3b3ab85 | 42 | #include "SwapDir.h" |
f09f5b26 | 43 | |
d54f0ab3 | 44 | static void storeSwapOutStart(StoreEntry * e); |
2391a162 | 45 | static STIOCB storeSwapOutFileClosed; |
cd748f27 | 46 | static STIOCB storeSwapOutFileNotify; |
f09f5b26 | 47 | |
48 | /* start swapping object to disk */ | |
d54f0ab3 | 49 | static void |
f09f5b26 | 50 | storeSwapOutStart(StoreEntry * e) |
51 | { | |
2391a162 | 52 | MemObject *mem = e->mem_obj; |
d3b3ab85 | 53 | StoreIOState::Pointer sio; |
2391a162 | 54 | assert(mem); |
cd748f27 | 55 | /* Build the swap metadata, so the filesystem will know how much |
56 | * metadata there is to store | |
57 | */ | |
58 | debug(20, 5) ("storeSwapOutStart: Begin SwapOut '%s' to dirno %d, fileno %08X\n", | |
62e76326 | 59 | storeUrl(e), e->swap_dirn, e->swap_filen); |
cd748f27 | 60 | e->swap_status = SWAPOUT_WRITING; |
528b2c61 | 61 | /* If we start swapping out objects with OutOfBand Metadata, |
62 | * then this code needs changing | |
63 | */ | |
cd748f27 | 64 | /* Create the swap file */ |
528b2c61 | 65 | generic_cbdata *c = cbdataAlloc(generic_cbdata); |
2391a162 | 66 | c->data = e; |
fa80a8ef | 67 | sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c); |
62e76326 | 68 | |
528b2c61 | 69 | if (NULL == sio.getRaw()) { |
62e76326 | 70 | e->swap_status = SWAPOUT_NONE; |
ad8f212f | 71 | cbdataFree(c); |
62e76326 | 72 | storeLog(STORE_LOG_SWAPOUTFAIL, e); |
73 | return; | |
3fc29499 | 74 | } |
62e76326 | 75 | |
528b2c61 | 76 | mem->swapout.sio = sio; |
77 | /* Don't lock until after create, or the replacement | |
78 | * code might get confused */ | |
79 | storeLockObject(e); | |
cd748f27 | 80 | /* Pick up the file number if it was assigned immediately */ |
81 | e->swap_filen = mem->swapout.sio->swap_filen; | |
82 | e->swap_dirn = mem->swapout.sio->swap_dirn; | |
83 | /* write out the swap metadata */ | |
528b2c61 | 84 | /* TODO: make some sort of data,size refcounted immutable buffer |
85 | * for use by this sort of function. | |
86 | */ | |
87 | char const *buf = e->getSerialisedMetaData (); | |
88 | /* If we start swapping out with out of band metadata, this assert | |
89 | * will catch it - this code needs to be adjusted if that happens | |
90 | */ | |
91 | assert (buf); | |
92 | storeIOWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree); | |
f09f5b26 | 93 | } |
94 | ||
cd748f27 | 95 | static void |
96 | storeSwapOutFileNotify(void *data, int errflag, storeIOState * sio) | |
97 | { | |
e6ccf245 | 98 | generic_cbdata *c = (generic_cbdata *)data; |
99 | StoreEntry *e = (StoreEntry *)c->data; | |
cd748f27 | 100 | MemObject *mem = e->mem_obj; |
101 | assert(e->swap_status == SWAPOUT_WRITING); | |
102 | assert(mem); | |
103 | assert(mem->swapout.sio == sio); | |
104 | assert(errflag == 0); | |
105 | e->swap_filen = mem->swapout.sio->swap_filen; | |
106 | e->swap_dirn = mem->swapout.sio->swap_dirn; | |
107 | } | |
108 | ||
528b2c61 | 109 | static void |
110 | doPages(StoreEntry *anEntry) | |
111 | { | |
112 | MemObject *mem = anEntry->mem_obj; | |
62e76326 | 113 | |
528b2c61 | 114 | do { |
62e76326 | 115 | /* |
116 | * Evil hack time. | |
117 | * We are paging out to disk in page size chunks. however, later on when | |
118 | * we update the queue position, we might not have a page (I *think*), | |
119 | * so we do the actual page update here. | |
120 | */ | |
121 | ||
122 | if (mem->swapout.memnode == NULL) { | |
123 | /* We need to swap out the first page */ | |
42a503bd | 124 | mem->swapout.memnode = const_cast<mem_node *>(mem->data_hdr.start()); |
62e76326 | 125 | } else { |
126 | /* We need to swap out the next page */ | |
42a503bd | 127 | /* 20030636 RBC - we don't have ->next anymore. |
128 | * But we do have the next location */ | |
129 | mem->swapout.memnode = mem->data_hdr.getBlockContainingLocation (mem->swapout.memnode->end()); | |
62e76326 | 130 | } |
131 | ||
132 | /* | |
133 | * Get the length of this buffer. We are assuming(!) that the buffer | |
134 | * length won't change on this buffer, or things are going to be very | |
135 | * strange. I think that after the copy to a buffer is done, the buffer | |
136 | * size should stay fixed regardless so that this code isn't confused, | |
137 | * but we can look at this at a later date or whenever the code results | |
138 | * in bad swapouts, whichever happens first. :-) | |
139 | */ | |
140 | ssize_t swap_buf_len = mem->swapout.memnode->nodeBuffer.length; | |
141 | ||
142 | debug(20, 3) ("storeSwapOut: swap_buf_len = %d\n", (int) swap_buf_len); | |
143 | ||
144 | assert(swap_buf_len > 0); | |
145 | ||
146 | debug(20, 3) ("storeSwapOut: swapping out %ld bytes from %ld\n", | |
147 | (long int) swap_buf_len, (long int) mem->swapout.queue_offset); | |
148 | ||
149 | mem->swapout.queue_offset += swap_buf_len; | |
150 | ||
151 | storeIOWrite(mem->swapout.sio, mem->swapout.memnode->data, swap_buf_len, -1, NULL); | |
152 | ||
153 | /* the storeWrite() call might generate an error */ | |
154 | if (anEntry->swap_status != SWAPOUT_WRITING) | |
155 | break; | |
156 | ||
157 | ssize_t swapout_size = (ssize_t) (mem->endOffset() - mem->swapout.queue_offset); | |
158 | ||
159 | if (anEntry->store_status == STORE_PENDING) | |
160 | if (swapout_size < SM_PAGE_SIZE) | |
161 | break; | |
162 | ||
163 | if (swapout_size <= 0) | |
164 | return; | |
528b2c61 | 165 | } while (true); |
166 | } | |
167 | ||
168 | ||
169 | /* This routine is called every time data is sent to the client side. | |
170 | * It's overhead is therefor, significant. | |
171 | */ | |
f09f5b26 | 172 | void |
2391a162 | 173 | storeSwapOut(StoreEntry * e) |
f09f5b26 | 174 | { |
528b2c61 | 175 | if (!e->mem_obj) |
62e76326 | 176 | return; |
177 | ||
528b2c61 | 178 | if (!e->swapoutPossible()) |
62e76326 | 179 | return; |
528b2c61 | 180 | |
181 | MemObject *mem = e->mem_obj; | |
182 | ||
2391a162 | 183 | debug(20, 7) ("storeSwapOut: mem->inmem_lo = %d\n", |
62e76326 | 184 | (int) mem->inmem_lo); |
185 | ||
528b2c61 | 186 | debug(20, 7) ("storeSwapOut: mem->endOffset() = %d\n", |
62e76326 | 187 | (int) mem->endOffset()); |
188 | ||
2391a162 | 189 | debug(20, 7) ("storeSwapOut: swapout.queue_offset = %d\n", |
62e76326 | 190 | (int) mem->swapout.queue_offset); |
191 | ||
d3b3ab85 | 192 | if (mem->swapout.sio.getRaw()) |
62e76326 | 193 | debug(20, 7) ("storeSwapOut: storeOffset() = %d\n", |
194 | (int) mem->swapout.sio->offset()); | |
195 | ||
528b2c61 | 196 | ssize_t swapout_maxsize = (ssize_t) (mem->endOffset() - mem->swapout.queue_offset); |
62e76326 | 197 | |
528b2c61 | 198 | assert(swapout_maxsize >= 0); |
62e76326 | 199 | |
528b2c61 | 200 | off_t const lowest_offset = mem->lowestMemReaderOffset(); |
62e76326 | 201 | |
2391a162 | 202 | debug(20, 7) ("storeSwapOut: lowest_offset = %d\n", |
62e76326 | 203 | (int) lowest_offset); |
204 | ||
cd748f27 | 205 | /* |
206 | * Grab the swapout_size and check to see whether we're going to defer | |
207 | * the swapout based upon size | |
208 | */ | |
528b2c61 | 209 | if ((e->store_status != STORE_OK) && (swapout_maxsize < store_maxobjsize)) { |
62e76326 | 210 | /* |
211 | * NOTE: the store_maxobjsize here is the max of optional | |
212 | * max-size values from 'cache_dir' lines. It is not the | |
213 | * same as 'maximum_object_size'. By default, store_maxobjsize | |
214 | * will be set to -1. However, I am worried that this | |
215 | * deferance may consume a lot of memory in some cases. | |
216 | * It would be good to make this decision based on reply | |
217 | * content-length, rather than wait to accumulate huge | |
218 | * amounts of object data in memory. | |
219 | */ | |
220 | debug(20, 5) ("storeSwapOut: Deferring starting swapping out\n"); | |
221 | return; | |
cd748f27 | 222 | } |
62e76326 | 223 | |
528b2c61 | 224 | e->trimMemory(); |
adcceb47 | 225 | #if SIZEOF_OFF_T == 4 |
62e76326 | 226 | |
528b2c61 | 227 | if (mem->endOffset() > 0x7FFF0000) { |
62e76326 | 228 | debug(20, 0) ("WARNING: preventing off_t overflow for %s\n", storeUrl(e)); |
229 | storeAbort(e); | |
230 | return; | |
adcceb47 | 231 | } |
62e76326 | 232 | |
adcceb47 | 233 | #endif |
c2725718 | 234 | if (e->swap_status == SWAPOUT_WRITING) |
62e76326 | 235 | assert(mem->inmem_lo <= (off_t)mem->objectBytesOnDisk() ); |
236 | ||
c2725718 | 237 | if (!storeSwapOutAble(e)) |
62e76326 | 238 | return; |
239 | ||
2391a162 | 240 | debug(20, 7) ("storeSwapOut: swapout_size = %d\n", |
62e76326 | 241 | (int) swapout_maxsize); |
242 | ||
528b2c61 | 243 | if (swapout_maxsize == 0) { |
62e76326 | 244 | if (e->store_status == STORE_OK) |
245 | storeSwapOutFileClose(e); | |
246 | ||
247 | return; /* Nevermore! */ | |
1b72bda1 | 248 | } |
62e76326 | 249 | |
c47511fd | 250 | if (e->store_status == STORE_PENDING) { |
62e76326 | 251 | /* wait for a full block to write */ |
252 | ||
253 | if (swapout_maxsize < SM_PAGE_SIZE) | |
254 | return; | |
255 | ||
256 | /* | |
257 | * Wait until we are below the disk FD limit, only if the | |
258 | * next server-side read won't be deferred. | |
259 | */ | |
a46d2c0e | 260 | if (storeTooManyDiskFilesOpen() && !e->checkDeferRead(-1)) |
62e76326 | 261 | return; |
c47511fd | 262 | } |
62e76326 | 263 | |
2391a162 | 264 | /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ |
f09f5b26 | 265 | if (e->swap_status == SWAPOUT_NONE) { |
62e76326 | 266 | assert(mem->swapout.sio == NULL); |
267 | assert(mem->inmem_lo == 0); | |
268 | ||
269 | if (storeCheckCachable(e)) | |
270 | storeSwapOutStart(e); | |
271 | else | |
272 | return; | |
273 | ||
274 | /* ENTRY_CACHABLE will be cleared and we'll never get here again */ | |
f09f5b26 | 275 | } |
62e76326 | 276 | |
d3b3ab85 | 277 | if (mem->swapout.sio == NULL) |
62e76326 | 278 | return; |
279 | ||
528b2c61 | 280 | doPages(e); |
62e76326 | 281 | |
528b2c61 | 282 | if (NULL == mem->swapout.sio.getRaw()) |
62e76326 | 283 | /* oops, we're not swapping out any more */ |
284 | return; | |
285 | ||
1792fbdb | 286 | if (e->store_status == STORE_OK) { |
62e76326 | 287 | /* |
288 | * If the state is STORE_OK, then all data must have been given | |
289 | * to the filesystem at this point because storeSwapOut() is | |
290 | * not going to be called again for this entry. | |
291 | */ | |
292 | assert(mem->endOffset() == mem->swapout.queue_offset); | |
293 | storeSwapOutFileClose(e); | |
1792fbdb | 294 | } |
f09f5b26 | 295 | } |
296 | ||
297 | void | |
298 | storeSwapOutFileClose(StoreEntry * e) | |
299 | { | |
300 | MemObject *mem = e->mem_obj; | |
4215b049 | 301 | assert(mem != NULL); |
332dafa2 | 302 | debug(20, 3) ("storeSwapOutFileClose: %s\n", e->getMD5Text()); |
d3b3ab85 | 303 | debug(20, 3) ("storeSwapOutFileClose: sio = %p\n", mem->swapout.sio.getRaw()); |
62e76326 | 304 | |
2391a162 | 305 | if (mem->swapout.sio == NULL) |
62e76326 | 306 | return; |
307 | ||
2391a162 | 308 | storeClose(mem->swapout.sio); |
f09f5b26 | 309 | } |
310 | ||
311 | static void | |
2391a162 | 312 | storeSwapOutFileClosed(void *data, int errflag, storeIOState * sio) |
f09f5b26 | 313 | { |
e6ccf245 | 314 | generic_cbdata *c = (generic_cbdata *)data; |
315 | StoreEntry *e = (StoreEntry *)c->data; | |
25354045 | 316 | MemObject *mem = e->mem_obj; |
2391a162 | 317 | assert(e->swap_status == SWAPOUT_WRITING); |
318 | cbdataFree(c); | |
62e76326 | 319 | |
2391a162 | 320 | if (errflag) { |
62e76326 | 321 | debug(20, 1) ("storeSwapOutFileClosed: dirno %d, swapfile %08X, errflag=%d\n\t%s\n", |
322 | e->swap_dirn, e->swap_filen, errflag, xstrerror()); | |
323 | ||
324 | if (errflag == DISK_NO_SPACE_LEFT) { | |
325 | storeDirDiskFull(e->swap_dirn); | |
326 | storeDirConfigure(); | |
327 | storeConfigure(); | |
328 | } | |
329 | ||
330 | if (e->swap_filen > 0) | |
331 | storeUnlink(e); | |
332 | ||
333 | e->swap_filen = -1; | |
334 | ||
335 | e->swap_dirn = -1; | |
336 | ||
337 | e->swap_status = SWAPOUT_NONE; | |
338 | ||
339 | storeReleaseRequest(e); | |
2391a162 | 340 | } else { |
62e76326 | 341 | /* swapping complete */ |
342 | debug(20, 3) ("storeSwapOutFileClosed: SwapOut complete: '%s' to %d, %08X\n", | |
343 | storeUrl(e), e->swap_dirn, e->swap_filen); | |
344 | e->swap_file_sz = objectLen(e) + mem->swap_hdr_sz; | |
345 | e->swap_status = SWAPOUT_DONE; | |
346 | storeDirUpdateSwapSize(INDEXSD(e->swap_dirn), e->swap_file_sz, 1); | |
347 | ||
348 | if (storeCheckCachable(e)) { | |
349 | storeLog(STORE_LOG_SWAPOUT, e); | |
350 | storeDirSwapLog(e, SWAP_LOG_ADD); | |
351 | } | |
352 | ||
353 | statCounter.swap.outs++; | |
f09f5b26 | 354 | } |
62e76326 | 355 | |
5bd1abac | 356 | debug(20, 3) ("storeSwapOutFileClosed: %s:%d\n", __FILE__, __LINE__); |
d3b3ab85 | 357 | mem->swapout.sio = NULL; |
5bd1abac | 358 | storeUnlockObject(e); |
f09f5b26 | 359 | } |
61038223 | 360 | |
c2725718 | 361 | /* |
362 | * Is this entry a candidate for writing to disk? | |
363 | */ | |
5f25e839 | 364 | int |
c2725718 | 365 | storeSwapOutAble(const StoreEntry * e) |
366 | { | |
06d2839d | 367 | dlink_node *node; |
62e76326 | 368 | |
d3b3ab85 | 369 | if (e->mem_obj->swapout.sio.getRaw() != NULL) |
62e76326 | 370 | return 1; |
371 | ||
c2725718 | 372 | if (e->mem_obj->inmem_lo > 0) |
62e76326 | 373 | return 0; |
374 | ||
5f25e839 | 375 | /* |
376 | * If there are DISK clients, we must write to disk | |
377 | * even if its not cachable | |
528b2c61 | 378 | * RBC: Surely we should not create disk client on non cacheable objects? |
379 | * therefore this should be an assert? | |
4e70dae3 | 380 | * RBC 20030708: We can use disk to avoid mem races, so this shouldn't be |
381 | * an assert. | |
5f25e839 | 382 | */ |
06d2839d | 383 | for (node = e->mem_obj->clients.head; node; node = node->next) { |
62e76326 | 384 | if (((store_client *) node->data)->getType() == STORE_DISK_CLIENT) |
385 | return 1; | |
06d2839d | 386 | } |
62e76326 | 387 | |
a56b2212 | 388 | /* Don't pollute the disk with icons and other special entries */ |
389 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) | |
62e76326 | 390 | return 0; |
391 | ||
528b2c61 | 392 | if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) |
62e76326 | 393 | return 0; |
394 | ||
528b2c61 | 395 | if (!e->mem_obj->isContiguous()) |
62e76326 | 396 | return 0; |
397 | ||
528b2c61 | 398 | return 1; |
c2725718 | 399 | } |