]>
Commit | Line | Data |
---|---|---|
a4b8110e | 1 | |
cd748f27 | 2 | /* |
4989878c | 3 | * $Id: store_dir_coss.cc,v 1.49 2003/08/30 06:39:24 robertc Exp $ |
cd748f27 | 4 | * |
17bb3486 | 5 | * DEBUG: section 47 Store COSS Directory Routines |
cd748f27 | 6 | * AUTHOR: Eric Stern |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
cd748f27 | 9 | * ---------------------------------------------------------- |
10 | * | |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
cd748f27 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
36 | #include "squid.h" | |
e6ccf245 | 37 | #include "Store.h" |
cd748f27 | 38 | |
39 | #include "store_coss.h" | |
528b2c61 | 40 | #include "fde.h" |
d3b3ab85 | 41 | #include "SwapDir.h" |
51ee7c82 | 42 | #include "StoreSwapLogData.h" |
cd748f27 | 43 | |
44 | #define STORE_META_BUFSZ 4096 | |
45 | ||
cd748f27 | 46 | int n_coss_dirs = 0; |
47 | /* static int last_coss_pick_index = -1; */ | |
6a566b9c | 48 | MemPool *coss_index_pool = NULL; |
cd748f27 | 49 | |
50 | typedef struct _RebuildState RebuildState; | |
62e76326 | 51 | |
1a224843 | 52 | void storeCossDirParseBlkSize(SwapDir *, const char *, const char *, int); |
53 | void storeCossDirDumpBlkSize(StoreEntry *, const char *, const SwapDir *); | |
54 | ||
62e76326 | 55 | struct _RebuildState |
56 | { | |
d3b3ab85 | 57 | CossSwapDir *sd; |
cd748f27 | 58 | int n_read; |
59 | FILE *log; | |
60 | int speed; | |
62e76326 | 61 | |
62 | struct | |
63 | { | |
64 | ||
65 | unsigned int clean: | |
66 | 1; | |
67 | } | |
68 | ||
69 | flags; | |
70 | ||
cd748f27 | 71 | struct _store_rebuild_data counts; |
72 | }; | |
73 | ||
74 | static char *storeCossDirSwapLogFile(SwapDir *, const char *); | |
75 | static EVH storeCossRebuildFromSwapLog; | |
d3b3ab85 | 76 | static StoreEntry *storeCossAddDiskRestore(CossSwapDir * SD, const cache_key * key, |
62e76326 | 77 | int file_number, |
78 | size_t swap_file_sz, | |
79 | time_t expires, | |
80 | time_t timestamp, | |
81 | time_t lastref, | |
82 | time_t lastmod, | |
83 | u_int32_t refcount, | |
84 | u_int16_t flags, | |
85 | int clean); | |
d3b3ab85 | 86 | static void storeCossDirRebuild(CossSwapDir * sd); |
87 | static void storeCossDirCloseTmpSwapLog(CossSwapDir * sd); | |
88 | static FILE *storeCossDirOpenTmpSwapLog(CossSwapDir *, int *, int *); | |
cd748f27 | 89 | |
90 | static char * | |
91 | storeCossDirSwapLogFile(SwapDir * sd, const char *ext) | |
92 | { | |
93 | LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); | |
94 | LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); | |
95 | LOCAL_ARRAY(char, digit, 32); | |
96 | char *pathtmp2; | |
62e76326 | 97 | |
cd748f27 | 98 | if (Config.Log.swap) { |
62e76326 | 99 | xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); |
100 | pathtmp2 = pathtmp; | |
101 | ||
102 | while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) | |
103 | *pathtmp2 = '.'; | |
104 | ||
105 | while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') | |
106 | pathtmp[strlen(pathtmp) - 1] = '\0'; | |
107 | ||
108 | for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++) | |
109 | ||
110 | ; | |
111 | snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); | |
112 | ||
113 | if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { | |
114 | strcat(path, "."); | |
115 | snprintf(digit, 32, "%02d", sd->index); | |
116 | strncat(path, digit, 3); | |
117 | } | |
cd748f27 | 118 | } else { |
62e76326 | 119 | xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); |
120 | strcat(path, "/swap.state"); | |
cd748f27 | 121 | } |
62e76326 | 122 | |
cd748f27 | 123 | if (ext) |
62e76326 | 124 | strncat(path, ext, 16); |
125 | ||
cd748f27 | 126 | return path; |
127 | } | |
128 | ||
d3b3ab85 | 129 | void |
130 | CossSwapDir::openLog() | |
cd748f27 | 131 | { |
d3b3ab85 | 132 | char *logPath; |
133 | logPath = storeCossDirSwapLogFile(this, NULL); | |
134 | swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); | |
62e76326 | 135 | |
d3b3ab85 | 136 | if (swaplog_fd < 0) { |
62e76326 | 137 | debug(47, 1) ("%s: %s\n", logPath, xstrerror()); |
138 | fatal("storeCossDirOpenSwapLog: Failed to open swap log."); | |
cd748f27 | 139 | } |
62e76326 | 140 | |
d3b3ab85 | 141 | debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", index, swaplog_fd); |
cd748f27 | 142 | } |
143 | ||
d3b3ab85 | 144 | void |
145 | CossSwapDir::closeLog() | |
cd748f27 | 146 | { |
d3b3ab85 | 147 | if (swaplog_fd < 0) /* not open */ |
62e76326 | 148 | return; |
149 | ||
d3b3ab85 | 150 | file_close(swaplog_fd); |
62e76326 | 151 | |
99cce4cb | 152 | debug(47, 3) ("Cache COSS Dir #%d log closed on FD %d\n", |
62e76326 | 153 | index, swaplog_fd); |
154 | ||
d3b3ab85 | 155 | swaplog_fd = -1; |
cd748f27 | 156 | } |
157 | ||
d3b3ab85 | 158 | void |
159 | CossSwapDir::init() | |
160 | { | |
161 | a_file_setupqueue(&aq); | |
162 | openLog(); | |
163 | storeCossDirRebuild(this); | |
164 | fd = file_open(path, O_RDWR | O_CREAT); | |
62e76326 | 165 | |
d3b3ab85 | 166 | if (fd < 0) { |
62e76326 | 167 | debug(47, 1) ("%s: %s\n", path, xstrerror()); |
1a224843 | 168 | fatal("storeCossDirInit: Failed to open a COSS file."); |
697c29aa | 169 | } |
62e76326 | 170 | |
cd748f27 | 171 | n_coss_dirs++; |
1a224843 | 172 | /* |
173 | * fs.blksize is normally determined by calling statvfs() etc, | |
174 | * but we just set it here. It is used in accounting the | |
175 | * total store size, and is reported in cachemgr 'storedir' | |
176 | * page. | |
177 | */ | |
178 | fs.blksize = 1 << blksz_bits; | |
cd748f27 | 179 | } |
180 | ||
6a566b9c | 181 | void |
d3b3ab85 | 182 | storeCossRemove(CossSwapDir * sd, StoreEntry * e) |
6a566b9c | 183 | { |
e6ccf245 | 184 | CossIndexNode *coss_node = (CossIndexNode *)e->repl.data; |
6a566b9c | 185 | e->repl.data = NULL; |
d3b3ab85 | 186 | dlinkDelete(&coss_node->node, &sd->cossindex); |
6a566b9c | 187 | memPoolFree(coss_index_pool, coss_node); |
d3b3ab85 | 188 | sd->count -= 1; |
6a566b9c | 189 | } |
190 | ||
191 | void | |
d3b3ab85 | 192 | storeCossAdd(CossSwapDir * sd, StoreEntry * e) |
6a566b9c | 193 | { |
e6ccf245 | 194 | CossIndexNode *coss_node = (CossIndexNode *)memPoolAlloc(coss_index_pool); |
6a566b9c | 195 | assert(!e->repl.data); |
196 | e->repl.data = coss_node; | |
d3b3ab85 | 197 | dlinkAdd(e, &coss_node->node, &sd->cossindex); |
198 | sd->count += 1; | |
6a566b9c | 199 | } |
200 | ||
201 | static void | |
202 | storeCossRebuildComplete(void *data) | |
203 | { | |
e6ccf245 | 204 | RebuildState *rb = (RebuildState *)data; |
d3b3ab85 | 205 | CossSwapDir *sd = rb->sd; |
6a566b9c | 206 | storeCossStartMembuf(sd); |
207 | store_dirs_rebuilding--; | |
208 | storeCossDirCloseTmpSwapLog(rb->sd); | |
209 | storeRebuildComplete(&rb->counts); | |
210 | cbdataFree(rb); | |
211 | } | |
212 | ||
cd748f27 | 213 | static void |
214 | storeCossRebuildFromSwapLog(void *data) | |
215 | { | |
e6ccf245 | 216 | RebuildState *rb = (RebuildState *)data; |
cd748f27 | 217 | StoreEntry *e = NULL; |
51ee7c82 | 218 | StoreSwapLogData s; |
219 | size_t ss = sizeof(StoreSwapLogData); | |
cd748f27 | 220 | double x; |
221 | assert(rb != NULL); | |
222 | /* load a number of objects per invocation */ | |
62e76326 | 223 | |
d3b3ab85 | 224 | for (int aCount = 0; aCount < rb->speed; aCount++) { |
62e76326 | 225 | if (fread(&s, ss, 1, rb->log) != 1) { |
226 | debug(47, 1) ("Done reading %s swaplog (%d entries)\n", | |
227 | rb->sd->path, rb->n_read); | |
228 | fclose(rb->log); | |
229 | rb->log = NULL; | |
230 | storeCossRebuildComplete(rb); | |
231 | return; | |
232 | } | |
233 | ||
234 | rb->n_read++; | |
235 | ||
236 | if (s.op <= SWAP_LOG_NOP) | |
237 | continue; | |
238 | ||
239 | if (s.op >= SWAP_LOG_MAX) | |
240 | continue; | |
241 | ||
242 | debug(47, 3) ("storeCossRebuildFromSwapLog: %s %s %08X\n", | |
243 | swap_log_op_str[(int) s.op], | |
244 | storeKeyText(s.key), | |
245 | s.swap_filen); | |
246 | ||
247 | if (s.op == SWAP_LOG_ADD) { | |
248 | (void) 0; | |
249 | } else if (s.op == SWAP_LOG_DEL) { | |
250 | /* Delete unless we already have a newer copy */ | |
251 | ||
252 | if ((e = storeGet(s.key)) != NULL && s.lastref > e->lastref) { | |
253 | /* | |
254 | * Make sure we don't unlink the file, it might be | |
255 | * in use by a subsequent entry. Also note that | |
256 | * we don't have to subtract from store_swap_size | |
257 | * because adding to store_swap_size happens in | |
258 | * the cleanup procedure. | |
259 | */ | |
260 | storeExpireNow(e); | |
261 | storeReleaseRequest(e); | |
262 | ||
263 | if (e->swap_filen > -1) { | |
264 | e->swap_filen = -1; | |
265 | } | |
266 | ||
267 | storeRelease(e); | |
268 | /* Fake an unlink here, this is a bad hack :( */ | |
269 | storeCossRemove(rb->sd, e); | |
270 | rb->counts.objcount--; | |
271 | rb->counts.cancelcount++; | |
272 | } | |
273 | ||
274 | continue; | |
275 | } else { | |
276 | x = log(++rb->counts.bad_log_op) / log(10.0); | |
277 | ||
278 | if (0.0 == x - (double) (int) x) | |
279 | debug(47, 1) ("WARNING: %d invalid swap log entries found\n", | |
280 | rb->counts.bad_log_op); | |
281 | ||
282 | rb->counts.invalid++; | |
283 | ||
284 | continue; | |
285 | } | |
286 | ||
287 | if ((++rb->counts.scancount & 0xFFF) == 0) { | |
288 | ||
289 | struct stat sb; | |
290 | ||
291 | if (0 == fstat(fileno(rb->log), &sb)) | |
292 | storeRebuildProgress(rb->sd->index, | |
293 | (int) sb.st_size / ss, rb->n_read); | |
294 | } | |
295 | ||
296 | if (EBIT_TEST(s.flags, KEY_PRIVATE)) { | |
297 | rb->counts.badflags++; | |
298 | continue; | |
299 | } | |
300 | ||
301 | e = storeGet(s.key); | |
302 | ||
303 | if (e) { | |
304 | /* key already exists, current entry is newer */ | |
305 | /* keep old, ignore new */ | |
306 | rb->counts.dupcount++; | |
307 | continue; | |
308 | } | |
309 | ||
310 | /* update store_swap_size */ | |
311 | rb->counts.objcount++; | |
312 | ||
313 | e = storeCossAddDiskRestore(rb->sd, s.key, | |
314 | s.swap_filen, | |
315 | s.swap_file_sz, | |
316 | s.expires, | |
317 | s.timestamp, | |
318 | s.lastref, | |
319 | s.lastmod, | |
320 | s.refcount, | |
321 | s.flags, | |
322 | (int) rb->flags.clean); | |
323 | ||
324 | storeDirSwapLog(e, SWAP_LOG_ADD); | |
cd748f27 | 325 | } |
62e76326 | 326 | |
cd748f27 | 327 | eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); |
328 | } | |
329 | ||
330 | /* Add a new object to the cache with empty memory copy and pointer to disk | |
331 | * use to rebuild store from disk. */ | |
332 | static StoreEntry * | |
d3b3ab85 | 333 | storeCossAddDiskRestore(CossSwapDir * SD, const cache_key * key, |
62e76326 | 334 | int file_number, |
335 | size_t swap_file_sz, | |
336 | time_t expires, | |
337 | time_t timestamp, | |
338 | time_t lastref, | |
339 | time_t lastmod, | |
340 | u_int32_t refcount, | |
341 | u_int16_t flags, | |
342 | int clean) | |
cd748f27 | 343 | { |
344 | StoreEntry *e = NULL; | |
99cce4cb | 345 | debug(47, 5) ("storeCossAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); |
62e76326 | 346 | /* if you call this you'd better be sure file_number is not |
cd748f27 | 347 | * already in use! */ |
348 | e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); | |
349 | e->store_status = STORE_OK; | |
350 | e->swap_dirn = SD->index; | |
351 | storeSetMemStatus(e, NOT_IN_MEMORY); | |
352 | e->swap_status = SWAPOUT_DONE; | |
353 | e->swap_filen = file_number; | |
354 | e->swap_file_sz = swap_file_sz; | |
355 | e->lock_count = 0; | |
cd748f27 | 356 | e->lastref = lastref; |
357 | e->timestamp = timestamp; | |
358 | e->expires = expires; | |
359 | e->lastmod = lastmod; | |
360 | e->refcount = refcount; | |
361 | e->flags = flags; | |
362 | EBIT_SET(e->flags, ENTRY_CACHABLE); | |
363 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
364 | EBIT_CLR(e->flags, KEY_PRIVATE); | |
365 | e->ping_status = PING_NONE; | |
366 | EBIT_CLR(e->flags, ENTRY_VALIDATED); | |
367 | storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ | |
6a566b9c | 368 | storeCossAdd(SD, e); |
1a224843 | 369 | #if USE_COSS_ALLOC_NOTIFY |
370 | ||
cd748f27 | 371 | e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); |
1a224843 | 372 | #endif |
373 | ||
374 | assert(e->swap_filen >= 0); | |
cd748f27 | 375 | return e; |
376 | } | |
377 | ||
28c60158 | 378 | CBDATA_TYPE(RebuildState); |
cd748f27 | 379 | static void |
d3b3ab85 | 380 | storeCossDirRebuild(CossSwapDir * sd) |
cd748f27 | 381 | { |
28c60158 | 382 | RebuildState *rb; |
cd748f27 | 383 | int clean = 0; |
384 | int zero = 0; | |
385 | FILE *fp; | |
386 | EVH *func = NULL; | |
28c60158 | 387 | CBDATA_INIT_TYPE(RebuildState); |
72711e31 | 388 | rb = cbdataAlloc(RebuildState); |
cd748f27 | 389 | rb->sd = sd; |
390 | rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; | |
6a566b9c | 391 | func = storeCossRebuildFromSwapLog; |
392 | rb->flags.clean = (unsigned int) clean; | |
cd748f27 | 393 | /* |
394 | * If the swap.state file exists in the cache_dir, then | |
395 | * we'll use storeCossRebuildFromSwapLog(). | |
396 | */ | |
397 | fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero); | |
99cce4cb | 398 | debug(47, 1) ("Rebuilding COSS storage in %s (%s)\n", |
62e76326 | 399 | sd->path, clean ? "CLEAN" : "DIRTY"); |
6a566b9c | 400 | rb->log = fp; |
401 | store_dirs_rebuilding++; | |
62e76326 | 402 | |
cd748f27 | 403 | if (!clean || fp == NULL) { |
62e76326 | 404 | /* COSS cannot yet rebuild from a dirty state. If the log |
405 | * is dirty then the COSS contents is thrown away. | |
406 | * Why? I guess it is because some contents will be lost, | |
407 | * and COSS cannot verify this.. | |
408 | */ | |
409 | ||
410 | if (fp != NULL) | |
411 | fclose(fp); | |
412 | ||
413 | /* | |
414 | * XXX Make sure we don't trigger an assertion if this is the first | |
415 | * storedir, since if we are, this call will cause storeRebuildComplete | |
416 | * to prematurely complete the rebuild process, and then some other | |
417 | * storedir will try to rebuild and eventually die. | |
418 | */ | |
419 | eventAdd("storeCossRebuildComplete", storeCossRebuildComplete, rb, 0.0, 0); | |
420 | ||
421 | return; | |
cd748f27 | 422 | } |
62e76326 | 423 | |
cd748f27 | 424 | eventAdd("storeCossRebuild", func, rb, 0.0, 1); |
425 | } | |
426 | ||
427 | static void | |
d3b3ab85 | 428 | storeCossDirCloseTmpSwapLog(CossSwapDir * sd) |
cd748f27 | 429 | { |
cd748f27 | 430 | char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); |
431 | char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); | |
d3b3ab85 | 432 | int anfd; |
433 | file_close(sd->swaplog_fd); | |
3d37fe17 | 434 | #if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_) |
62e76326 | 435 | |
cd748f27 | 436 | if (unlink(swaplog_path) < 0) { |
62e76326 | 437 | debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); |
438 | fatal("storeCossDirCloseTmpSwapLog: unlink failed"); | |
cd748f27 | 439 | } |
62e76326 | 440 | |
cd748f27 | 441 | #endif |
442 | if (xrename(new_path, swaplog_path) < 0) { | |
62e76326 | 443 | fatal("storeCossDirCloseTmpSwapLog: rename failed"); |
cd748f27 | 444 | } |
62e76326 | 445 | |
d3b3ab85 | 446 | anfd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); |
62e76326 | 447 | |
d3b3ab85 | 448 | if (anfd < 0) { |
62e76326 | 449 | debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); |
450 | fatal("storeCossDirCloseTmpSwapLog: Failed to open swap log."); | |
cd748f27 | 451 | } |
62e76326 | 452 | |
cd748f27 | 453 | safe_free(swaplog_path); |
454 | safe_free(new_path); | |
d3b3ab85 | 455 | sd->swaplog_fd = anfd; |
456 | debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, anfd); | |
cd748f27 | 457 | } |
458 | ||
459 | static FILE * | |
d3b3ab85 | 460 | storeCossDirOpenTmpSwapLog(CossSwapDir * sd, int *clean_flag, int *zero_flag) |
cd748f27 | 461 | { |
cd748f27 | 462 | char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); |
463 | char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); | |
464 | char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); | |
62e76326 | 465 | |
cd748f27 | 466 | struct stat log_sb; |
62e76326 | 467 | |
cd748f27 | 468 | struct stat clean_sb; |
469 | FILE *fp; | |
d3b3ab85 | 470 | int anfd; |
62e76326 | 471 | |
cd748f27 | 472 | if (stat(swaplog_path, &log_sb) < 0) { |
62e76326 | 473 | debug(50, 1) ("Cache COSS Dir #%d: No log file\n", sd->index); |
474 | safe_free(swaplog_path); | |
475 | safe_free(clean_path); | |
476 | safe_free(new_path); | |
477 | return NULL; | |
cd748f27 | 478 | } |
62e76326 | 479 | |
cd748f27 | 480 | *zero_flag = log_sb.st_size == 0 ? 1 : 0; |
481 | /* close the existing write-only FD */ | |
62e76326 | 482 | |
d3b3ab85 | 483 | if (sd->swaplog_fd >= 0) |
62e76326 | 484 | file_close(sd->swaplog_fd); |
485 | ||
cd748f27 | 486 | /* open a write-only FD for the new log */ |
d3b3ab85 | 487 | anfd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); |
62e76326 | 488 | |
d3b3ab85 | 489 | if (anfd < 0) { |
62e76326 | 490 | debug(50, 1) ("%s: %s\n", new_path, xstrerror()); |
491 | fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); | |
cd748f27 | 492 | } |
62e76326 | 493 | |
d3b3ab85 | 494 | sd->swaplog_fd = anfd; |
cd748f27 | 495 | /* open a read-only stream of the old log */ |
3d37fe17 | 496 | fp = fopen(swaplog_path, "rb"); |
62e76326 | 497 | |
cd748f27 | 498 | if (fp == NULL) { |
62e76326 | 499 | debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); |
500 | fatal("Failed to open swap log for reading"); | |
cd748f27 | 501 | } |
62e76326 | 502 | |
cd748f27 | 503 | memset(&clean_sb, '\0', sizeof(struct stat)); |
62e76326 | 504 | |
cd748f27 | 505 | if (stat(clean_path, &clean_sb) < 0) |
62e76326 | 506 | *clean_flag = 0; |
cd748f27 | 507 | else if (clean_sb.st_mtime < log_sb.st_mtime) |
62e76326 | 508 | *clean_flag = 0; |
cd748f27 | 509 | else |
62e76326 | 510 | *clean_flag = 1; |
511 | ||
cd748f27 | 512 | safeunlink(clean_path, 1); |
62e76326 | 513 | |
cd748f27 | 514 | safe_free(swaplog_path); |
62e76326 | 515 | |
cd748f27 | 516 | safe_free(clean_path); |
62e76326 | 517 | |
cd748f27 | 518 | safe_free(new_path); |
62e76326 | 519 | |
cd748f27 | 520 | return fp; |
521 | } | |
522 | ||
62e76326 | 523 | class CossCleanLog : public SwapDir::CleanLog |
524 | { | |
525 | ||
526 | public: | |
d3b3ab85 | 527 | CossCleanLog(SwapDir *); |
528 | virtual const StoreEntry *nextEntry(); | |
529 | virtual void write(StoreEntry const &); | |
cd748f27 | 530 | char *cur; |
e6ccf245 | 531 | char *newLog; |
cd748f27 | 532 | char *cln; |
533 | char *outbuf; | |
534 | off_t outbuf_offset; | |
535 | int fd; | |
6a566b9c | 536 | dlink_node *current; |
d3b3ab85 | 537 | SwapDir *sd; |
cd748f27 | 538 | }; |
539 | ||
540 | #define CLEAN_BUF_SZ 16384 | |
d3b3ab85 | 541 | |
542 | CossCleanLog::CossCleanLog(SwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), | |
62e76326 | 543 | outbuf_offset(0), fd(-1),current(NULL), sd(aSwapDir) |
544 | {} | |
d3b3ab85 | 545 | |
cd748f27 | 546 | /* |
547 | * Begin the process to write clean cache state. For COSS this means | |
548 | * opening some log files and allocating write buffers. Return 0 if | |
549 | * we succeed, and assign the 'func' and 'data' return pointers. | |
550 | */ | |
d3b3ab85 | 551 | int |
552 | CossSwapDir::writeCleanStart() | |
cd748f27 | 553 | { |
d3b3ab85 | 554 | CossCleanLog *state = new CossCleanLog(this); |
1f56cfef | 555 | #if HAVE_FCHMOD |
62e76326 | 556 | |
cd748f27 | 557 | struct stat sb; |
1f56cfef | 558 | #endif |
62e76326 | 559 | |
d3b3ab85 | 560 | state->newLog = xstrdup(storeCossDirSwapLogFile(this, ".clean")); |
e6ccf245 | 561 | state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); |
d3b3ab85 | 562 | cleanLog = NULL; |
62e76326 | 563 | |
81928dfe | 564 | if (state->fd < 0) { |
62e76326 | 565 | xfree(state->newLog); |
566 | delete state; | |
567 | return -1; | |
81928dfe | 568 | } |
62e76326 | 569 | |
d3b3ab85 | 570 | state->cur = xstrdup(storeCossDirSwapLogFile(this, NULL)); |
571 | state->cln = xstrdup(storeCossDirSwapLogFile(this, ".last-clean")); | |
e6ccf245 | 572 | state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); |
cd748f27 | 573 | state->outbuf_offset = 0; |
d3b3ab85 | 574 | ::unlink(state->cln); |
575 | state->current = cossindex.tail; | |
99cce4cb | 576 | debug(50, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n", |
62e76326 | 577 | state->newLog, state->fd); |
cd748f27 | 578 | #if HAVE_FCHMOD |
62e76326 | 579 | |
cd748f27 | 580 | if (stat(state->cur, &sb) == 0) |
62e76326 | 581 | fchmod(state->fd, sb.st_mode); |
582 | ||
cd748f27 | 583 | #endif |
62e76326 | 584 | |
d3b3ab85 | 585 | cleanLog = state; |
6a566b9c | 586 | |
cd748f27 | 587 | return 0; |
588 | } | |
589 | ||
d3b3ab85 | 590 | const StoreEntry * |
591 | CossCleanLog::nextEntry() | |
6a566b9c | 592 | { |
6a566b9c | 593 | const StoreEntry *entry; |
62e76326 | 594 | |
d3b3ab85 | 595 | if (!current) |
62e76326 | 596 | return NULL; |
597 | ||
d3b3ab85 | 598 | entry = (const StoreEntry *) current->data; |
62e76326 | 599 | |
d3b3ab85 | 600 | current = current->prev; |
62e76326 | 601 | |
6a566b9c | 602 | return entry; |
603 | } | |
604 | ||
cd748f27 | 605 | /* |
606 | * "write" an entry to the clean log file. | |
607 | */ | |
d3b3ab85 | 608 | void |
609 | CossCleanLog::write(StoreEntry const &e) | |
cd748f27 | 610 | { |
d3b3ab85 | 611 | CossCleanLog *state = this; |
51ee7c82 | 612 | StoreSwapLogData s; |
613 | static size_t ss = sizeof(StoreSwapLogData); | |
cd748f27 | 614 | s.op = (char) SWAP_LOG_ADD; |
d3b3ab85 | 615 | s.swap_filen = e.swap_filen; |
616 | s.timestamp = e.timestamp; | |
617 | s.lastref = e.lastref; | |
618 | s.expires = e.expires; | |
619 | s.lastmod = e.lastmod; | |
620 | s.swap_file_sz = e.swap_file_sz; | |
621 | s.refcount = e.refcount; | |
622 | s.flags = e.flags; | |
623 | xmemcpy(&s.key, e.key, MD5_DIGEST_CHARS); | |
cd748f27 | 624 | xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); |
625 | state->outbuf_offset += ss; | |
626 | /* buffered write */ | |
62e76326 | 627 | |
cd748f27 | 628 | if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { |
62e76326 | 629 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
630 | debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", | |
631 | state->newLog, xstrerror()); | |
632 | debug(50, 0) ("storeCossDirWriteCleanLogs: Current swap logfile not replaced.\n"); | |
633 | file_close(state->fd); | |
634 | state->fd = -1; | |
635 | unlink(state->newLog); | |
636 | delete state; | |
637 | sd->cleanLog = NULL; | |
638 | return; | |
639 | } | |
640 | ||
641 | state->outbuf_offset = 0; | |
cd748f27 | 642 | } |
643 | } | |
644 | ||
d3b3ab85 | 645 | void |
646 | CossSwapDir::writeCleanDone() | |
cd748f27 | 647 | { |
d3b3ab85 | 648 | CossCleanLog *state = (CossCleanLog *)cleanLog; |
62e76326 | 649 | |
a7d542ee | 650 | if (NULL == state) |
62e76326 | 651 | return; |
652 | ||
cd748f27 | 653 | if (state->fd < 0) |
62e76326 | 654 | return; |
655 | ||
d0ef8ea8 | 656 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
62e76326 | 657 | debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", |
658 | state->newLog, xstrerror()); | |
659 | debug(50, 0) ("storeCossDirWriteCleanLogs: Current swap logfile " | |
660 | "not replaced.\n"); | |
661 | file_close(state->fd); | |
662 | state->fd = -1; | |
663 | ::unlink(state->newLog); | |
cd748f27 | 664 | } |
62e76326 | 665 | |
cd748f27 | 666 | safe_free(state->outbuf); |
667 | /* | |
668 | * You can't rename open files on Microsoft "operating systems" | |
669 | * so we have to close before renaming. | |
670 | */ | |
d3b3ab85 | 671 | closeLog(); |
3d37fe17 | 672 | /* save the fd value for a later test */ |
d3b3ab85 | 673 | int anfd = state->fd; |
cd748f27 | 674 | /* rename */ |
62e76326 | 675 | |
cd748f27 | 676 | if (state->fd >= 0) { |
3d37fe17 | 677 | #if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) |
62e76326 | 678 | file_close(state->fd); |
679 | state->fd = -1; | |
680 | ||
681 | if (unlink(state->cur) < 0) | |
682 | debug(50, 0) ("storeCossDirWriteCleanLogs: unlinkd failed: %s, %s\n", | |
683 | xstrerror(), state->cur); | |
684 | ||
cd748f27 | 685 | #endif |
62e76326 | 686 | |
687 | xrename(state->newLog, state->cur); | |
cd748f27 | 688 | } |
62e76326 | 689 | |
cd748f27 | 690 | /* touch a timestamp file if we're not still validating */ |
691 | if (store_dirs_rebuilding) | |
62e76326 | 692 | (void) 0; |
d3b3ab85 | 693 | else if (anfd < 0) |
62e76326 | 694 | (void) 0; |
cd748f27 | 695 | else |
62e76326 | 696 | file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); |
697 | ||
cd748f27 | 698 | /* close */ |
699 | safe_free(state->cur); | |
62e76326 | 700 | |
e6ccf245 | 701 | safe_free(state->newLog); |
62e76326 | 702 | |
cd748f27 | 703 | safe_free(state->cln); |
62e76326 | 704 | |
cd748f27 | 705 | if (state->fd >= 0) |
62e76326 | 706 | file_close(state->fd); |
707 | ||
cd748f27 | 708 | state->fd = -1; |
62e76326 | 709 | |
d3b3ab85 | 710 | delete state; |
62e76326 | 711 | |
d3b3ab85 | 712 | cleanLog = NULL; |
cd748f27 | 713 | } |
714 | ||
061c7171 | 715 | static void |
716 | FreeObject(void *address) | |
717 | { | |
718 | StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address); | |
719 | delete anObject; | |
720 | } | |
721 | ||
d3b3ab85 | 722 | void |
723 | CossSwapDir::logEntry(const StoreEntry & e, int op) const | |
cd748f27 | 724 | { |
51ee7c82 | 725 | StoreSwapLogData *s = new StoreSwapLogData; |
cd748f27 | 726 | s->op = (char) op; |
d3b3ab85 | 727 | s->swap_filen = e.swap_filen; |
728 | s->timestamp = e.timestamp; | |
729 | s->lastref = e.lastref; | |
730 | s->expires = e.expires; | |
731 | s->lastmod = e.lastmod; | |
732 | s->swap_file_sz = e.swap_file_sz; | |
733 | s->refcount = e.refcount; | |
734 | s->flags = e.flags; | |
735 | xmemcpy(s->key, e.key, MD5_DIGEST_CHARS); | |
736 | file_write(swaplog_fd, | |
62e76326 | 737 | -1, |
738 | s, | |
51ee7c82 | 739 | sizeof(StoreSwapLogData), |
62e76326 | 740 | NULL, |
741 | NULL, | |
061c7171 | 742 | &FreeObject); |
cd748f27 | 743 | } |
744 | ||
d3b3ab85 | 745 | void |
746 | CossSwapDir::newFileSystem() | |
cd748f27 | 747 | { |
d3b3ab85 | 748 | debug(47, 3) ("Creating swap space in %s\n", path); |
749 | debug (47,0)("COSS autocreation is not implemented. Please create the file manually\n"); | |
cd748f27 | 750 | } |
751 | ||
a4b8110e | 752 | /* we are shutting down, flush all membufs to disk */ |
d3b3ab85 | 753 | CossSwapDir::~CossSwapDir() |
cd748f27 | 754 | { |
d3b3ab85 | 755 | sync(); /* This'll call a_file_syncqueue() */ |
756 | a_file_closequeue(&aq); | |
757 | file_close(fd); | |
758 | fd = -1; | |
cd748f27 | 759 | |
d3b3ab85 | 760 | closeLog(); |
cd748f27 | 761 | n_coss_dirs--; |
762 | } | |
763 | ||
764 | /* | |
765 | * storeCossDirCheckObj | |
766 | * | |
767 | * This routine is called by storeDirSelectSwapDir to see if the given | |
768 | * object is able to be stored on this filesystem. COSS filesystems will | |
769 | * not store everything. We don't check for maxobjsize here since its | |
770 | * done by the upper layers. | |
771 | */ | |
772 | int | |
d3b3ab85 | 773 | CossSwapDir::canStore(StoreEntry const &e)const |
cd748f27 | 774 | { |
0dbabb56 | 775 | int loadav; |
776 | ||
a4b8110e | 777 | /* Check if the object is a special object, we can't cache these */ |
62e76326 | 778 | |
d3b3ab85 | 779 | if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) |
62e76326 | 780 | return -1; |
cd748f27 | 781 | |
a4b8110e | 782 | /* Otherwise, we're ok */ |
0dbabb56 | 783 | /* Return load, cs->aq.aq_numpending out of MAX_ASYNCOP */ |
d3b3ab85 | 784 | loadav = aq.aq_numpending * 1000 / MAX_ASYNCOP; |
62e76326 | 785 | |
0dbabb56 | 786 | return loadav; |
cd748f27 | 787 | } |
a4b8110e | 788 | |
0e23343f | 789 | /* |
790 | * storeCossDirCallback - do the IO completions | |
791 | */ | |
d3b3ab85 | 792 | int |
793 | CossSwapDir::callback() | |
0e23343f | 794 | { |
d3b3ab85 | 795 | return a_file_callback(&aq); |
0e23343f | 796 | } |
797 | ||
cd748f27 | 798 | /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ |
799 | ||
d3b3ab85 | 800 | void |
801 | CossSwapDir::statfs(StoreEntry & sentry) const | |
cd748f27 | 802 | { |
d3b3ab85 | 803 | storeAppendPrintf(&sentry, "\n"); |
804 | storeAppendPrintf(&sentry, "Maximum Size: %d KB\n", max_size); | |
805 | storeAppendPrintf(&sentry, "Current Size: %d KB\n", cur_size); | |
806 | storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", | |
62e76326 | 807 | 100.0 * cur_size / max_size); |
d3b3ab85 | 808 | storeAppendPrintf(&sentry, "Number of object collisions: %d\n", (int) numcollisions); |
cd748f27 | 809 | #if 0 |
810 | /* is this applicable? I Hope not .. */ | |
811 | storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", | |
62e76326 | 812 | SD->map->n_files_in_map, SD->map->max_n_files, |
813 | percent(SD->map->n_files_in_map, SD->map->max_n_files)); | |
cd748f27 | 814 | #endif |
62e76326 | 815 | |
d3b3ab85 | 816 | storeAppendPrintf(&sentry, "Pending operations: %d out of %d\n", aq.aq_numpending, MAX_ASYNCOP); |
817 | storeAppendPrintf(&sentry, "Flags:"); | |
62e76326 | 818 | |
d3b3ab85 | 819 | if (flags.selected) |
62e76326 | 820 | storeAppendPrintf(&sentry, " SELECTED"); |
821 | ||
d3b3ab85 | 822 | if (flags.read_only) |
62e76326 | 823 | storeAppendPrintf(&sentry, " READ-ONLY"); |
824 | ||
d3b3ab85 | 825 | storeAppendPrintf(&sentry, "\n"); |
cd748f27 | 826 | } |
827 | ||
d3b3ab85 | 828 | void |
829 | CossSwapDir::parse(int anIndex, char *aPath) | |
cd748f27 | 830 | { |
cd748f27 | 831 | unsigned int i; |
832 | unsigned int size; | |
1a224843 | 833 | off_t max_offset; |
cd748f27 | 834 | |
835 | i = GetInteger(); | |
836 | size = i << 10; /* Mbytes to Kbytes */ | |
62e76326 | 837 | |
cd748f27 | 838 | if (size <= 0) |
62e76326 | 839 | fatal("storeCossDirParse: invalid size value"); |
cd748f27 | 840 | |
d3b3ab85 | 841 | index = anIndex; |
62e76326 | 842 | |
d3b3ab85 | 843 | path = xstrdup(aPath); |
62e76326 | 844 | |
d3b3ab85 | 845 | max_size = size; |
846 | ||
c94af0b9 | 847 | /* Enforce maxobjsize being set to something */ |
d3b3ab85 | 848 | if (max_objsize == -1) |
62e76326 | 849 | fatal("COSS requires max-size to be set to something other than -1!\n"); |
1a224843 | 850 | |
851 | if (max_objsize > COSS_MEMBUF_SZ) | |
852 | fatalf("COSS max-size option must be less than COSS_MEMBUF_SZ (%d)\n", | |
853 | COSS_MEMBUF_SZ); | |
854 | ||
855 | /* | |
856 | * check that we won't overflow sfileno later. 0xFFFFFF is the | |
857 | * largest possible sfileno, assuming sfileno is a 25-bit | |
858 | * signed integer, as defined in structs.h. | |
859 | */ | |
860 | max_offset = (off_t) 0xFFFFFF << blksz_bits; | |
861 | ||
862 | if (max_size > (max_offset>>10)) { | |
863 | debug(47,0)("COSS block-size = %d bytes\n", 1<<blksz_bits); | |
4989878c | 864 | debugs(47,0, "COSS largest file offset = " << (max_offset >> 10) << " KB"); |
1a224843 | 865 | debug(47,0)("COSS cache_dir size = %d KB\n", max_size); |
866 | fatal("COSS cache_dir size exceeds largest offset\n"); | |
867 | } | |
cd748f27 | 868 | } |
869 | ||
870 | ||
d3b3ab85 | 871 | void |
872 | CossSwapDir::reconfigure(int index, char *path) | |
cd748f27 | 873 | { |
cd748f27 | 874 | unsigned int i; |
875 | unsigned int size; | |
cd748f27 | 876 | |
877 | i = GetInteger(); | |
878 | size = i << 10; /* Mbytes to Kbytes */ | |
62e76326 | 879 | |
cd748f27 | 880 | if (size <= 0) |
62e76326 | 881 | fatal("storeCossDirParse: invalid size value"); |
cd748f27 | 882 | |
d3b3ab85 | 883 | if (size == (size_t)max_size) |
62e76326 | 884 | debug(3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size); |
cd748f27 | 885 | else { |
62e76326 | 886 | debug(3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size); |
887 | max_size = size; | |
cd748f27 | 888 | } |
62e76326 | 889 | |
c94af0b9 | 890 | /* Enforce maxobjsize being set to something */ |
d3b3ab85 | 891 | if (max_objsize == -1) |
62e76326 | 892 | fatal("COSS requires max-size to be set to something other than -1!\n"); |
cd748f27 | 893 | } |
894 | ||
895 | void | |
d3b3ab85 | 896 | CossSwapDir::dump(StoreEntry &entry)const |
cd748f27 | 897 | { |
59b2d47f | 898 | storeAppendPrintf(&entry, " %d", max_size >> 20); |
899 | dumpOptions(&entry); | |
cd748f27 | 900 | } |
901 | ||
1a224843 | 902 | |
903 | CossSwapDir::CossSwapDir() : SwapDir ("coss"), fd (-1), swaplog_fd(-1), count(0), current_membuf (NULL), current_offset(0), numcollisions(0) | |
cd748f27 | 904 | { |
1a224843 | 905 | membufs.head = NULL; |
906 | membufs.tail = NULL; | |
907 | cossindex.head = NULL; | |
908 | cossindex.tail = NULL; | |
909 | blksz_mask = (1 << blksz_bits) - 1; | |
910 | } | |
a4b8110e | 911 | |
1a224843 | 912 | bool |
913 | CossSwapDir::optionBlockSizeParse(const char *option, const char *value, int reconfiguring) | |
914 | { | |
915 | int blksz = atoi(value); | |
62e76326 | 916 | |
1a224843 | 917 | if (blksz == (1 << blksz_bits)) |
918 | /* no change */ | |
919 | return true; | |
920 | ||
921 | if (reconfiguring) { | |
922 | debug(47, 0) ("WARNING: cannot change COSS block-size while" | |
923 | " Squid is running\n"); | |
924 | return false; | |
cd748f27 | 925 | } |
62e76326 | 926 | |
1a224843 | 927 | int nbits = 0; |
928 | int check = blksz; | |
929 | ||
930 | while (check > 1) { | |
931 | nbits++; | |
932 | check >>= 1; | |
cd748f27 | 933 | } |
62e76326 | 934 | |
1a224843 | 935 | check = 1 << nbits; |
936 | ||
937 | if (check != blksz) | |
938 | fatal("COSS block-size must be a power of 2\n"); | |
939 | ||
940 | if (nbits > 13) | |
941 | fatal("COSS block-size must be 8192 or smaller\n"); | |
942 | ||
943 | blksz_bits = nbits; | |
944 | ||
945 | blksz_mask = (1 << blksz_bits) - 1; | |
946 | ||
947 | return true; | |
cd748f27 | 948 | } |
62e76326 | 949 | |
1a224843 | 950 | void |
951 | CossSwapDir::optionBlockSizeDump(StoreEntry * e) const | |
d3b3ab85 | 952 | { |
1a224843 | 953 | storeAppendPrintf(e, " block-size=%d", 1 << blksz_bits); |
d3b3ab85 | 954 | } |
955 | ||
1a224843 | 956 | SwapDirOption * |
957 | CossSwapDir::getOptionTree() const | |
958 | { | |
959 | SwapDirOption *parentResult = SwapDir::getOptionTree(); | |
960 | ||
961 | SwapDirOptionVector *result = new SwapDirOptionVector(); | |
962 | ||
963 | result->options.push_back(parentResult); | |
964 | ||
965 | result->options.push_back( | |
966 | new SwapDirOptionAdapter<CossSwapDir>(*const_cast<CossSwapDir *>(this), | |
967 | &CossSwapDir::optionBlockSizeParse, | |
968 | &CossSwapDir::optionBlockSizeDump)); | |
969 | ||
970 | return result; | |
971 | } |