]>
Commit | Line | Data |
---|---|---|
a4b8110e | 1 | |
cd748f27 | 2 | /* |
332dafa2 | 3 | * $Id: store_dir_coss.cc,v 1.40 2002/10/15 08:03:33 robertc Exp $ |
cd748f27 | 4 | * |
17bb3486 | 5 | * DEBUG: section 47 Store COSS Directory Routines |
cd748f27 | 6 | * AUTHOR: Eric Stern |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
cd748f27 | 9 | * ---------------------------------------------------------- |
10 | * | |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
cd748f27 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
36 | #include "squid.h" | |
e6ccf245 | 37 | #include "Store.h" |
0b30d52d | 38 | #include <aio.h> |
cd748f27 | 39 | |
0b30d52d | 40 | #include "async_io.h" |
cd748f27 | 41 | #include "store_coss.h" |
42 | ||
43 | #define STORE_META_BUFSZ 4096 | |
44 | ||
cd748f27 | 45 | int n_coss_dirs = 0; |
46 | /* static int last_coss_pick_index = -1; */ | |
47 | int coss_initialised = 0; | |
a4b8110e | 48 | MemPool *coss_state_pool = NULL; |
6a566b9c | 49 | MemPool *coss_index_pool = NULL; |
cd748f27 | 50 | |
51 | typedef struct _RebuildState RebuildState; | |
52 | struct _RebuildState { | |
53 | SwapDir *sd; | |
54 | int n_read; | |
55 | FILE *log; | |
56 | int speed; | |
cd748f27 | 57 | struct { |
cd748f27 | 58 | unsigned int clean:1; |
cd748f27 | 59 | } flags; |
cd748f27 | 60 | struct _store_rebuild_data counts; |
61 | }; | |
62 | ||
63 | static char *storeCossDirSwapLogFile(SwapDir *, const char *); | |
64 | static EVH storeCossRebuildFromSwapLog; | |
a4b8110e | 65 | static StoreEntry *storeCossAddDiskRestore(SwapDir * SD, const cache_key * key, |
cd748f27 | 66 | int file_number, |
67 | size_t swap_file_sz, | |
68 | time_t expires, | |
69 | time_t timestamp, | |
70 | time_t lastref, | |
71 | time_t lastmod, | |
a9245686 | 72 | u_int32_t refcount, |
73 | u_int16_t flags, | |
cd748f27 | 74 | int clean); |
75 | static void storeCossDirRebuild(SwapDir * sd); | |
76 | static void storeCossDirCloseTmpSwapLog(SwapDir * sd); | |
77 | static FILE *storeCossDirOpenTmpSwapLog(SwapDir *, int *, int *); | |
78 | static STLOGOPEN storeCossDirOpenSwapLog; | |
79 | static STINIT storeCossDirInit; | |
6a566b9c | 80 | static STLOGCLEANSTART storeCossDirWriteCleanStart; |
81 | static STLOGCLEANNEXTENTRY storeCossDirCleanLogNextEntry; | |
cd748f27 | 82 | static STLOGCLEANWRITE storeCossDirWriteCleanEntry; |
6a566b9c | 83 | static STLOGCLEANDONE storeCossDirWriteCleanDone; |
cd748f27 | 84 | static STLOGCLOSE storeCossDirCloseSwapLog; |
85 | static STLOGWRITE storeCossDirSwapLog; | |
86 | static STNEWFS storeCossDirNewfs; | |
87 | static STCHECKOBJ storeCossDirCheckObj; | |
6a566b9c | 88 | static STFREE storeCossDirShutdown; |
89 | static STFSPARSE storeCossDirParse; | |
90 | static STFSRECONFIGURE storeCossDirReconfigure; | |
91 | static STDUMP storeCossDirDump; | |
0e23343f | 92 | static STCALLBACK storeCossDirCallback; |
cd748f27 | 93 | |
2d72d4fd | 94 | /* The "only" externally visible function */ |
95 | STSETUP storeFsSetup_coss; | |
96 | ||
cd748f27 | 97 | static char * |
98 | storeCossDirSwapLogFile(SwapDir * sd, const char *ext) | |
99 | { | |
100 | LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); | |
101 | LOCAL_ARRAY(char, pathtmp, SQUID_MAXPATHLEN); | |
102 | LOCAL_ARRAY(char, digit, 32); | |
103 | char *pathtmp2; | |
104 | if (Config.Log.swap) { | |
a4b8110e | 105 | xstrncpy(pathtmp, sd->path, SQUID_MAXPATHLEN - 64); |
cb6bd0bd | 106 | pathtmp2 = pathtmp; |
107 | while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) | |
108 | *pathtmp2 = '.'; | |
a4b8110e | 109 | while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') |
110 | pathtmp[strlen(pathtmp) - 1] = '\0'; | |
111 | for (pathtmp2 = pathtmp; *pathtmp2 == '.'; pathtmp2++); | |
112 | snprintf(path, SQUID_MAXPATHLEN - 64, Config.Log.swap, pathtmp2); | |
113 | if (strncmp(path, Config.Log.swap, SQUID_MAXPATHLEN - 64) == 0) { | |
114 | strcat(path, "."); | |
115 | snprintf(digit, 32, "%02d", sd->index); | |
116 | strncat(path, digit, 3); | |
117 | } | |
cd748f27 | 118 | } else { |
a4b8110e | 119 | xstrncpy(path, sd->path, SQUID_MAXPATHLEN - 64); |
120 | strcat(path, "/swap.state"); | |
cd748f27 | 121 | } |
122 | if (ext) | |
a4b8110e | 123 | strncat(path, ext, 16); |
cd748f27 | 124 | return path; |
125 | } | |
126 | ||
127 | static void | |
128 | storeCossDirOpenSwapLog(SwapDir * sd) | |
129 | { | |
a4b8110e | 130 | CossInfo *cs = (CossInfo *) sd->fsdata; |
cd748f27 | 131 | char *path; |
132 | int fd; | |
133 | path = storeCossDirSwapLogFile(sd, NULL); | |
3d37fe17 | 134 | fd = file_open(path, O_WRONLY | O_CREAT | O_BINARY); |
cd748f27 | 135 | if (fd < 0) { |
99cce4cb | 136 | debug(47, 1) ("%s: %s\n", path, xstrerror()); |
cd748f27 | 137 | fatal("storeCossDirOpenSwapLog: Failed to open swap log."); |
138 | } | |
99cce4cb | 139 | debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd); |
cd748f27 | 140 | cs->swaplog_fd = fd; |
141 | } | |
142 | ||
143 | static void | |
144 | storeCossDirCloseSwapLog(SwapDir * sd) | |
145 | { | |
a4b8110e | 146 | CossInfo *cs = (CossInfo *) sd->fsdata; |
cd748f27 | 147 | if (cs->swaplog_fd < 0) /* not open */ |
148 | return; | |
149 | file_close(cs->swaplog_fd); | |
99cce4cb | 150 | debug(47, 3) ("Cache COSS Dir #%d log closed on FD %d\n", |
cd748f27 | 151 | sd->index, cs->swaplog_fd); |
152 | cs->swaplog_fd = -1; | |
153 | } | |
154 | ||
155 | static void | |
156 | storeCossDirInit(SwapDir * sd) | |
157 | { | |
a4b8110e | 158 | CossInfo *cs = (CossInfo *) sd->fsdata; |
0b30d52d | 159 | a_file_setupqueue(&cs->aq); |
cd748f27 | 160 | storeCossDirOpenSwapLog(sd); |
161 | storeCossDirRebuild(sd); | |
162 | cs->fd = file_open(sd->path, O_RDWR | O_CREAT); | |
697c29aa | 163 | if (cs->fd < 0) { |
99cce4cb | 164 | debug(47, 1) ("%s: %s\n", sd->path, xstrerror()); |
b671cc68 | 165 | fatal("storeCossDirInit: Failed to open a COSS directory."); |
697c29aa | 166 | } |
cd748f27 | 167 | n_coss_dirs++; |
3abcea5e | 168 | (void) storeDirGetBlkSize(sd->path, &sd->fs.blksize); |
cd748f27 | 169 | } |
170 | ||
6a566b9c | 171 | void |
e282852a | 172 | storeCossRemove(SwapDir * sd, StoreEntry * e) |
6a566b9c | 173 | { |
174 | CossInfo *cs = (CossInfo *) sd->fsdata; | |
e6ccf245 | 175 | CossIndexNode *coss_node = (CossIndexNode *)e->repl.data; |
6a566b9c | 176 | e->repl.data = NULL; |
177 | dlinkDelete(&coss_node->node, &cs->index); | |
178 | memPoolFree(coss_index_pool, coss_node); | |
179 | cs->count -= 1; | |
180 | } | |
181 | ||
182 | void | |
e282852a | 183 | storeCossAdd(SwapDir * sd, StoreEntry * e) |
6a566b9c | 184 | { |
185 | CossInfo *cs = (CossInfo *) sd->fsdata; | |
e6ccf245 | 186 | CossIndexNode *coss_node = (CossIndexNode *)memPoolAlloc(coss_index_pool); |
6a566b9c | 187 | assert(!e->repl.data); |
188 | e->repl.data = coss_node; | |
189 | dlinkAdd(e, &coss_node->node, &cs->index); | |
190 | cs->count += 1; | |
191 | } | |
192 | ||
193 | static void | |
194 | storeCossRebuildComplete(void *data) | |
195 | { | |
e6ccf245 | 196 | RebuildState *rb = (RebuildState *)data; |
6a566b9c | 197 | SwapDir *sd = rb->sd; |
198 | storeCossStartMembuf(sd); | |
199 | store_dirs_rebuilding--; | |
200 | storeCossDirCloseTmpSwapLog(rb->sd); | |
201 | storeRebuildComplete(&rb->counts); | |
202 | cbdataFree(rb); | |
203 | } | |
204 | ||
cd748f27 | 205 | static void |
206 | storeCossRebuildFromSwapLog(void *data) | |
207 | { | |
e6ccf245 | 208 | RebuildState *rb = (RebuildState *)data; |
cd748f27 | 209 | StoreEntry *e = NULL; |
210 | storeSwapLogData s; | |
211 | size_t ss = sizeof(storeSwapLogData); | |
212 | int count; | |
213 | double x; | |
214 | assert(rb != NULL); | |
215 | /* load a number of objects per invocation */ | |
216 | for (count = 0; count < rb->speed; count++) { | |
217 | if (fread(&s, ss, 1, rb->log) != 1) { | |
99cce4cb | 218 | debug(47, 1) ("Done reading %s swaplog (%d entries)\n", |
cd748f27 | 219 | rb->sd->path, rb->n_read); |
220 | fclose(rb->log); | |
221 | rb->log = NULL; | |
6a566b9c | 222 | storeCossRebuildComplete(rb); |
cd748f27 | 223 | return; |
224 | } | |
225 | rb->n_read++; | |
226 | if (s.op <= SWAP_LOG_NOP) | |
227 | continue; | |
228 | if (s.op >= SWAP_LOG_MAX) | |
229 | continue; | |
99cce4cb | 230 | debug(47, 3) ("storeCossRebuildFromSwapLog: %s %s %08X\n", |
cd748f27 | 231 | swap_log_op_str[(int) s.op], |
232 | storeKeyText(s.key), | |
233 | s.swap_filen); | |
234 | if (s.op == SWAP_LOG_ADD) { | |
235 | (void) 0; | |
236 | } else if (s.op == SWAP_LOG_DEL) { | |
237 | if ((e = storeGet(s.key)) != NULL) { | |
238 | /* | |
239 | * Make sure we don't unlink the file, it might be | |
240 | * in use by a subsequent entry. Also note that | |
241 | * we don't have to subtract from store_swap_size | |
242 | * because adding to store_swap_size happens in | |
243 | * the cleanup procedure. | |
244 | */ | |
245 | storeExpireNow(e); | |
246 | storeReleaseRequest(e); | |
247 | if (e->swap_filen > -1) { | |
248 | e->swap_filen = -1; | |
249 | } | |
a4b8110e | 250 | storeRelease(e); |
251 | /* Fake an unlink here, this is a bad hack :( */ | |
273eab74 | 252 | storeCossRemove(rb->sd, e); |
cd748f27 | 253 | rb->counts.objcount--; |
254 | rb->counts.cancelcount++; | |
255 | } | |
256 | continue; | |
257 | } else { | |
258 | x = log(++rb->counts.bad_log_op) / log(10.0); | |
259 | if (0.0 == x - (double) (int) x) | |
99cce4cb | 260 | debug(47, 1) ("WARNING: %d invalid swap log entries found\n", |
cd748f27 | 261 | rb->counts.bad_log_op); |
262 | rb->counts.invalid++; | |
263 | continue; | |
264 | } | |
5a797987 | 265 | if ((++rb->counts.scancount & 0xFFF) == 0) { |
266 | struct stat sb; | |
267 | if (0 == fstat(fileno(rb->log), &sb)) | |
273eab74 | 268 | storeRebuildProgress(rb->sd->index, |
5a797987 | 269 | (int) sb.st_size / ss, rb->n_read); |
270 | } | |
cd748f27 | 271 | if (EBIT_TEST(s.flags, KEY_PRIVATE)) { |
272 | rb->counts.badflags++; | |
273 | continue; | |
274 | } | |
275 | e = storeGet(s.key); | |
276 | if (e) { | |
277 | /* key already exists, current entry is newer */ | |
278 | /* keep old, ignore new */ | |
279 | rb->counts.dupcount++; | |
280 | continue; | |
281 | } | |
282 | /* update store_swap_size */ | |
283 | rb->counts.objcount++; | |
a4b8110e | 284 | e = storeCossAddDiskRestore(rb->sd, s.key, |
cd748f27 | 285 | s.swap_filen, |
286 | s.swap_file_sz, | |
287 | s.expires, | |
288 | s.timestamp, | |
289 | s.lastref, | |
290 | s.lastmod, | |
291 | s.refcount, | |
292 | s.flags, | |
293 | (int) rb->flags.clean); | |
294 | storeDirSwapLog(e, SWAP_LOG_ADD); | |
295 | } | |
296 | eventAdd("storeCossRebuild", storeCossRebuildFromSwapLog, rb, 0.0, 1); | |
297 | } | |
298 | ||
299 | /* Add a new object to the cache with empty memory copy and pointer to disk | |
300 | * use to rebuild store from disk. */ | |
301 | static StoreEntry * | |
a4b8110e | 302 | storeCossAddDiskRestore(SwapDir * SD, const cache_key * key, |
cd748f27 | 303 | int file_number, |
304 | size_t swap_file_sz, | |
305 | time_t expires, | |
306 | time_t timestamp, | |
307 | time_t lastref, | |
308 | time_t lastmod, | |
a9245686 | 309 | u_int32_t refcount, |
310 | u_int16_t flags, | |
cd748f27 | 311 | int clean) |
312 | { | |
313 | StoreEntry *e = NULL; | |
99cce4cb | 314 | debug(47, 5) ("storeCossAddDiskRestore: %s, fileno=%08X\n", storeKeyText(key), file_number); |
cd748f27 | 315 | /* if you call this you'd better be sure file_number is not |
316 | * already in use! */ | |
317 | e = new_StoreEntry(STORE_ENTRY_WITHOUT_MEMOBJ, NULL, NULL); | |
318 | e->store_status = STORE_OK; | |
319 | e->swap_dirn = SD->index; | |
320 | storeSetMemStatus(e, NOT_IN_MEMORY); | |
321 | e->swap_status = SWAPOUT_DONE; | |
322 | e->swap_filen = file_number; | |
323 | e->swap_file_sz = swap_file_sz; | |
324 | e->lock_count = 0; | |
cd748f27 | 325 | e->lastref = lastref; |
326 | e->timestamp = timestamp; | |
327 | e->expires = expires; | |
328 | e->lastmod = lastmod; | |
329 | e->refcount = refcount; | |
330 | e->flags = flags; | |
331 | EBIT_SET(e->flags, ENTRY_CACHABLE); | |
332 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
333 | EBIT_CLR(e->flags, KEY_PRIVATE); | |
334 | e->ping_status = PING_NONE; | |
335 | EBIT_CLR(e->flags, ENTRY_VALIDATED); | |
336 | storeHashInsert(e, key); /* do it after we clear KEY_PRIVATE */ | |
6a566b9c | 337 | storeCossAdd(SD, e); |
cd748f27 | 338 | e->swap_filen = storeCossAllocate(SD, e, COSS_ALLOC_NOTIFY); |
339 | return e; | |
340 | } | |
341 | ||
28c60158 | 342 | CBDATA_TYPE(RebuildState); |
cd748f27 | 343 | static void |
344 | storeCossDirRebuild(SwapDir * sd) | |
345 | { | |
28c60158 | 346 | RebuildState *rb; |
cd748f27 | 347 | int clean = 0; |
348 | int zero = 0; | |
349 | FILE *fp; | |
350 | EVH *func = NULL; | |
28c60158 | 351 | CBDATA_INIT_TYPE(RebuildState); |
72711e31 | 352 | rb = cbdataAlloc(RebuildState); |
cd748f27 | 353 | rb->sd = sd; |
354 | rb->speed = opt_foreground_rebuild ? 1 << 30 : 50; | |
6a566b9c | 355 | func = storeCossRebuildFromSwapLog; |
356 | rb->flags.clean = (unsigned int) clean; | |
cd748f27 | 357 | /* |
358 | * If the swap.state file exists in the cache_dir, then | |
359 | * we'll use storeCossRebuildFromSwapLog(). | |
360 | */ | |
361 | fp = storeCossDirOpenTmpSwapLog(sd, &clean, &zero); | |
99cce4cb | 362 | debug(47, 1) ("Rebuilding COSS storage in %s (%s)\n", |
cd748f27 | 363 | sd->path, clean ? "CLEAN" : "DIRTY"); |
6a566b9c | 364 | rb->log = fp; |
365 | store_dirs_rebuilding++; | |
cd748f27 | 366 | if (!clean || fp == NULL) { |
a4b8110e | 367 | /* COSS cannot yet rebuild from a dirty state. If the log |
cd748f27 | 368 | * is dirty then the COSS contents is thrown away. |
369 | * Why? I guess it is because some contents will be lost, | |
370 | * and COSS cannot verify this.. | |
371 | */ | |
372 | if (fp != NULL) | |
373 | fclose(fp); | |
a4b8110e | 374 | /* |
cd748f27 | 375 | * XXX Make sure we don't trigger an assertion if this is the first |
376 | * storedir, since if we are, this call will cause storeRebuildComplete | |
377 | * to prematurely complete the rebuild process, and then some other | |
378 | * storedir will try to rebuild and eventually die. | |
379 | */ | |
6a566b9c | 380 | eventAdd("storeCossRebuildComplete", storeCossRebuildComplete, rb, 0.0, 0); |
a4b8110e | 381 | return; |
cd748f27 | 382 | } |
cd748f27 | 383 | eventAdd("storeCossRebuild", func, rb, 0.0, 1); |
384 | } | |
385 | ||
386 | static void | |
387 | storeCossDirCloseTmpSwapLog(SwapDir * sd) | |
388 | { | |
a4b8110e | 389 | CossInfo *cs = (CossInfo *) sd->fsdata; |
cd748f27 | 390 | char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); |
391 | char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); | |
392 | int fd; | |
393 | file_close(cs->swaplog_fd); | |
3d37fe17 | 394 | #if defined (_SQUID_OS2_) || defined (_SQUID_CYGWIN_) |
cd748f27 | 395 | if (unlink(swaplog_path) < 0) { |
396 | debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); | |
397 | fatal("storeCossDirCloseTmpSwapLog: unlink failed"); | |
398 | } | |
399 | #endif | |
400 | if (xrename(new_path, swaplog_path) < 0) { | |
401 | fatal("storeCossDirCloseTmpSwapLog: rename failed"); | |
402 | } | |
3d37fe17 | 403 | fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); |
cd748f27 | 404 | if (fd < 0) { |
405 | debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); | |
406 | fatal("storeCossDirCloseTmpSwapLog: Failed to open swap log."); | |
407 | } | |
408 | safe_free(swaplog_path); | |
409 | safe_free(new_path); | |
410 | cs->swaplog_fd = fd; | |
411 | debug(47, 3) ("Cache COSS Dir #%d log opened on FD %d\n", sd->index, fd); | |
412 | } | |
413 | ||
414 | static FILE * | |
415 | storeCossDirOpenTmpSwapLog(SwapDir * sd, int *clean_flag, int *zero_flag) | |
416 | { | |
a4b8110e | 417 | CossInfo *cs = (CossInfo *) sd->fsdata; |
cd748f27 | 418 | char *swaplog_path = xstrdup(storeCossDirSwapLogFile(sd, NULL)); |
419 | char *clean_path = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); | |
420 | char *new_path = xstrdup(storeCossDirSwapLogFile(sd, ".new")); | |
421 | struct stat log_sb; | |
422 | struct stat clean_sb; | |
423 | FILE *fp; | |
424 | int fd; | |
425 | if (stat(swaplog_path, &log_sb) < 0) { | |
99cce4cb | 426 | debug(50, 1) ("Cache COSS Dir #%d: No log file\n", sd->index); |
cd748f27 | 427 | safe_free(swaplog_path); |
428 | safe_free(clean_path); | |
429 | safe_free(new_path); | |
430 | return NULL; | |
431 | } | |
432 | *zero_flag = log_sb.st_size == 0 ? 1 : 0; | |
433 | /* close the existing write-only FD */ | |
434 | if (cs->swaplog_fd >= 0) | |
435 | file_close(cs->swaplog_fd); | |
436 | /* open a write-only FD for the new log */ | |
3d37fe17 | 437 | fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); |
cd748f27 | 438 | if (fd < 0) { |
439 | debug(50, 1) ("%s: %s\n", new_path, xstrerror()); | |
440 | fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); | |
441 | } | |
442 | cs->swaplog_fd = fd; | |
443 | /* open a read-only stream of the old log */ | |
3d37fe17 | 444 | fp = fopen(swaplog_path, "rb"); |
cd748f27 | 445 | if (fp == NULL) { |
446 | debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); | |
447 | fatal("Failed to open swap log for reading"); | |
448 | } | |
449 | memset(&clean_sb, '\0', sizeof(struct stat)); | |
450 | if (stat(clean_path, &clean_sb) < 0) | |
451 | *clean_flag = 0; | |
452 | else if (clean_sb.st_mtime < log_sb.st_mtime) | |
453 | *clean_flag = 0; | |
454 | else | |
455 | *clean_flag = 1; | |
456 | safeunlink(clean_path, 1); | |
457 | safe_free(swaplog_path); | |
458 | safe_free(clean_path); | |
459 | safe_free(new_path); | |
460 | return fp; | |
461 | } | |
462 | ||
463 | struct _clean_state { | |
464 | char *cur; | |
e6ccf245 | 465 | char *newLog; |
cd748f27 | 466 | char *cln; |
467 | char *outbuf; | |
468 | off_t outbuf_offset; | |
469 | int fd; | |
6a566b9c | 470 | dlink_node *current; |
cd748f27 | 471 | }; |
472 | ||
473 | #define CLEAN_BUF_SZ 16384 | |
474 | /* | |
475 | * Begin the process to write clean cache state. For COSS this means | |
476 | * opening some log files and allocating write buffers. Return 0 if | |
477 | * we succeed, and assign the 'func' and 'data' return pointers. | |
478 | */ | |
479 | static int | |
6a566b9c | 480 | storeCossDirWriteCleanStart(SwapDir * sd) |
cd748f27 | 481 | { |
6a566b9c | 482 | CossInfo *cs = (CossInfo *) sd->fsdata; |
e6ccf245 | 483 | struct _clean_state *state = (struct _clean_state *)xcalloc(1, sizeof(*state)); |
1f56cfef | 484 | #if HAVE_FCHMOD |
cd748f27 | 485 | struct stat sb; |
1f56cfef | 486 | #endif |
e6ccf245 | 487 | state->newLog = xstrdup(storeCossDirSwapLogFile(sd, ".clean")); |
488 | state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); | |
81928dfe | 489 | if (state->fd < 0) { |
e6ccf245 | 490 | xfree(state->newLog); |
81928dfe | 491 | xfree(state); |
492 | return -1; | |
493 | } | |
cd748f27 | 494 | sd->log.clean.write = NULL; |
495 | sd->log.clean.state = NULL; | |
496 | state->cur = xstrdup(storeCossDirSwapLogFile(sd, NULL)); | |
cd748f27 | 497 | state->cln = xstrdup(storeCossDirSwapLogFile(sd, ".last-clean")); |
e6ccf245 | 498 | state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); |
cd748f27 | 499 | state->outbuf_offset = 0; |
cd748f27 | 500 | unlink(state->cln); |
6a566b9c | 501 | state->current = cs->index.tail; |
99cce4cb | 502 | debug(50, 3) ("storeCOssDirWriteCleanLogs: opened %s, FD %d\n", |
e6ccf245 | 503 | state->newLog, state->fd); |
cd748f27 | 504 | #if HAVE_FCHMOD |
505 | if (stat(state->cur, &sb) == 0) | |
506 | fchmod(state->fd, sb.st_mode); | |
507 | #endif | |
508 | sd->log.clean.write = storeCossDirWriteCleanEntry; | |
509 | sd->log.clean.state = state; | |
6a566b9c | 510 | |
cd748f27 | 511 | return 0; |
512 | } | |
513 | ||
6a566b9c | 514 | static const StoreEntry * |
515 | storeCossDirCleanLogNextEntry(SwapDir * sd) | |
516 | { | |
e6ccf245 | 517 | struct _clean_state *state = (struct _clean_state *)sd->log.clean.state; |
6a566b9c | 518 | const StoreEntry *entry; |
519 | if (!state) | |
520 | return NULL; | |
521 | if (!state->current) | |
522 | return NULL; | |
e282852a | 523 | entry = (const StoreEntry *) state->current->data; |
6a566b9c | 524 | state->current = state->current->prev; |
525 | return entry; | |
526 | } | |
527 | ||
cd748f27 | 528 | /* |
529 | * "write" an entry to the clean log file. | |
530 | */ | |
531 | static void | |
e282852a | 532 | storeCossDirWriteCleanEntry(SwapDir * sd, const StoreEntry * e) |
cd748f27 | 533 | { |
534 | storeSwapLogData s; | |
535 | static size_t ss = sizeof(storeSwapLogData); | |
e6ccf245 | 536 | struct _clean_state *state = (struct _clean_state *)sd->log.clean.state; |
cd748f27 | 537 | memset(&s, '\0', ss); |
538 | s.op = (char) SWAP_LOG_ADD; | |
539 | s.swap_filen = e->swap_filen; | |
540 | s.timestamp = e->timestamp; | |
541 | s.lastref = e->lastref; | |
542 | s.expires = e->expires; | |
543 | s.lastmod = e->lastmod; | |
544 | s.swap_file_sz = e->swap_file_sz; | |
545 | s.refcount = e->refcount; | |
546 | s.flags = e->flags; | |
332dafa2 | 547 | xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); |
cd748f27 | 548 | xmemcpy(state->outbuf + state->outbuf_offset, &s, ss); |
549 | state->outbuf_offset += ss; | |
550 | /* buffered write */ | |
551 | if (state->outbuf_offset + ss > CLEAN_BUF_SZ) { | |
d0ef8ea8 | 552 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
cd748f27 | 553 | debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", |
e6ccf245 | 554 | state->newLog, xstrerror()); |
99cce4cb | 555 | debug(50, 0) ("storeCossDirWriteCleanLogs: Current swap logfile not replaced.\n"); |
cd748f27 | 556 | file_close(state->fd); |
557 | state->fd = -1; | |
e6ccf245 | 558 | unlink(state->newLog); |
cd748f27 | 559 | safe_free(state); |
560 | sd->log.clean.state = NULL; | |
561 | sd->log.clean.write = NULL; | |
82c308b4 | 562 | return; |
cd748f27 | 563 | } |
564 | state->outbuf_offset = 0; | |
565 | } | |
566 | } | |
567 | ||
568 | static void | |
6a566b9c | 569 | storeCossDirWriteCleanDone(SwapDir * sd) |
cd748f27 | 570 | { |
3d37fe17 | 571 | int fd; |
e6ccf245 | 572 | struct _clean_state *state = (struct _clean_state *)sd->log.clean.state; |
a7d542ee | 573 | if (NULL == state) |
574 | return; | |
cd748f27 | 575 | if (state->fd < 0) |
576 | return; | |
d0ef8ea8 | 577 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
cd748f27 | 578 | debug(50, 0) ("storeCossDirWriteCleanLogs: %s: write: %s\n", |
e6ccf245 | 579 | state->newLog, xstrerror()); |
99cce4cb | 580 | debug(50, 0) ("storeCossDirWriteCleanLogs: Current swap logfile " |
cd748f27 | 581 | "not replaced.\n"); |
582 | file_close(state->fd); | |
583 | state->fd = -1; | |
e6ccf245 | 584 | unlink(state->newLog); |
cd748f27 | 585 | } |
586 | safe_free(state->outbuf); | |
587 | /* | |
588 | * You can't rename open files on Microsoft "operating systems" | |
589 | * so we have to close before renaming. | |
590 | */ | |
591 | storeCossDirCloseSwapLog(sd); | |
3d37fe17 | 592 | /* save the fd value for a later test */ |
593 | fd = state->fd; | |
cd748f27 | 594 | /* rename */ |
595 | if (state->fd >= 0) { | |
3d37fe17 | 596 | #if defined(_SQUID_OS2_) || defined (_SQUID_CYGWIN_) |
cd748f27 | 597 | file_close(state->fd); |
598 | state->fd = -1; | |
3d37fe17 | 599 | if (unlink(state->cur) < 0) |
cd748f27 | 600 | debug(50, 0) ("storeCossDirWriteCleanLogs: unlinkd failed: %s, %s\n", |
3d37fe17 | 601 | xstrerror(), state->cur); |
cd748f27 | 602 | #endif |
e6ccf245 | 603 | xrename(state->newLog, state->cur); |
cd748f27 | 604 | } |
605 | /* touch a timestamp file if we're not still validating */ | |
606 | if (store_dirs_rebuilding) | |
607 | (void) 0; | |
3d37fe17 | 608 | else if (fd < 0) |
cd748f27 | 609 | (void) 0; |
610 | else | |
3d37fe17 | 611 | file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); |
cd748f27 | 612 | /* close */ |
613 | safe_free(state->cur); | |
e6ccf245 | 614 | safe_free(state->newLog); |
cd748f27 | 615 | safe_free(state->cln); |
616 | if (state->fd >= 0) | |
617 | file_close(state->fd); | |
618 | state->fd = -1; | |
619 | safe_free(state); | |
620 | sd->log.clean.state = NULL; | |
621 | sd->log.clean.write = NULL; | |
622 | } | |
623 | ||
58cd5bbd | 624 | static void |
625 | storeSwapLogDataFree(void *s) | |
626 | { | |
627 | memFree(s, MEM_SWAP_LOG_DATA); | |
628 | } | |
629 | ||
cd748f27 | 630 | static void |
631 | storeCossDirSwapLog(const SwapDir * sd, const StoreEntry * e, int op) | |
632 | { | |
a4b8110e | 633 | CossInfo *cs = (CossInfo *) sd->fsdata; |
e6ccf245 | 634 | storeSwapLogData *s = (storeSwapLogData *)memAllocate(MEM_SWAP_LOG_DATA); |
cd748f27 | 635 | s->op = (char) op; |
636 | s->swap_filen = e->swap_filen; | |
637 | s->timestamp = e->timestamp; | |
638 | s->lastref = e->lastref; | |
639 | s->expires = e->expires; | |
640 | s->lastmod = e->lastmod; | |
641 | s->swap_file_sz = e->swap_file_sz; | |
642 | s->refcount = e->refcount; | |
643 | s->flags = e->flags; | |
332dafa2 | 644 | xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); |
cd748f27 | 645 | file_write(cs->swaplog_fd, |
646 | -1, | |
647 | s, | |
648 | sizeof(storeSwapLogData), | |
649 | NULL, | |
650 | NULL, | |
58cd5bbd | 651 | (FREE *) storeSwapLogDataFree); |
cd748f27 | 652 | } |
653 | ||
654 | static void | |
655 | storeCossDirNewfs(SwapDir * sd) | |
656 | { | |
657 | debug(47, 3) ("Creating swap space in %s\n", sd->path); | |
658 | } | |
659 | ||
a4b8110e | 660 | /* we are shutting down, flush all membufs to disk */ |
cd748f27 | 661 | static void |
a4b8110e | 662 | storeCossDirShutdown(SwapDir * SD) |
cd748f27 | 663 | { |
a4b8110e | 664 | CossInfo *cs = (CossInfo *) SD->fsdata; |
cd748f27 | 665 | |
b671cc68 | 666 | storeCossSync(SD); /* This'll call a_file_syncqueue() */ |
0b30d52d | 667 | a_file_closequeue(&cs->aq); |
cd748f27 | 668 | file_close(cs->fd); |
669 | cs->fd = -1; | |
670 | ||
671 | if (cs->swaplog_fd > -1) { | |
672 | file_close(cs->swaplog_fd); | |
673 | cs->swaplog_fd = -1; | |
674 | } | |
cd748f27 | 675 | n_coss_dirs--; |
676 | } | |
677 | ||
678 | /* | |
679 | * storeCossDirCheckObj | |
680 | * | |
681 | * This routine is called by storeDirSelectSwapDir to see if the given | |
682 | * object is able to be stored on this filesystem. COSS filesystems will | |
683 | * not store everything. We don't check for maxobjsize here since its | |
684 | * done by the upper layers. | |
685 | */ | |
686 | int | |
a4b8110e | 687 | storeCossDirCheckObj(SwapDir * SD, const StoreEntry * e) |
cd748f27 | 688 | { |
0dbabb56 | 689 | CossInfo *cs = (CossInfo *) SD->fsdata; |
690 | int loadav; | |
691 | ||
a4b8110e | 692 | /* Check if the object is a special object, we can't cache these */ |
693 | if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) | |
694 | return -1; | |
cd748f27 | 695 | |
a4b8110e | 696 | /* Otherwise, we're ok */ |
0dbabb56 | 697 | /* Return load, cs->aq.aq_numpending out of MAX_ASYNCOP */ |
698 | loadav = cs->aq.aq_numpending * 1000 / MAX_ASYNCOP; | |
699 | return loadav; | |
cd748f27 | 700 | } |
a4b8110e | 701 | |
0e23343f | 702 | |
703 | /* | |
704 | * storeCossDirCallback - do the IO completions | |
705 | */ | |
706 | static int | |
b671cc68 | 707 | storeCossDirCallback(SwapDir * SD) |
0e23343f | 708 | { |
709 | CossInfo *cs = (CossInfo *) SD->fsdata; | |
710 | ||
711 | return a_file_callback(&cs->aq); | |
712 | } | |
713 | ||
cd748f27 | 714 | /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ |
715 | ||
2d72d4fd | 716 | static void |
a4b8110e | 717 | storeCossDirStats(SwapDir * SD, StoreEntry * sentry) |
cd748f27 | 718 | { |
a4b8110e | 719 | CossInfo *cs = (CossInfo *) SD->fsdata; |
720 | ||
cd748f27 | 721 | storeAppendPrintf(sentry, "\n"); |
722 | storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); | |
723 | storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); | |
724 | storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", | |
a4b8110e | 725 | 100.0 * SD->cur_size / SD->max_size); |
726 | storeAppendPrintf(sentry, "Number of object collisions: %d\n", (int) cs->numcollisions); | |
cd748f27 | 727 | #if 0 |
728 | /* is this applicable? I Hope not .. */ | |
729 | storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", | |
a4b8110e | 730 | SD->map->n_files_in_map, SD->map->max_n_files, |
731 | percent(SD->map->n_files_in_map, SD->map->max_n_files)); | |
cd748f27 | 732 | #endif |
0dbabb56 | 733 | storeAppendPrintf(sentry, "Pending operations: %d out of %d\n", cs->aq.aq_numpending, MAX_ASYNCOP); |
cd748f27 | 734 | storeAppendPrintf(sentry, "Flags:"); |
735 | if (SD->flags.selected) | |
a4b8110e | 736 | storeAppendPrintf(sentry, " SELECTED"); |
cd748f27 | 737 | if (SD->flags.read_only) |
a4b8110e | 738 | storeAppendPrintf(sentry, " READ-ONLY"); |
cd748f27 | 739 | storeAppendPrintf(sentry, "\n"); |
740 | } | |
741 | ||
742 | static void | |
a4b8110e | 743 | storeCossDirParse(SwapDir * sd, int index, char *path) |
cd748f27 | 744 | { |
cd748f27 | 745 | unsigned int i; |
746 | unsigned int size; | |
cd748f27 | 747 | CossInfo *cs; |
748 | ||
749 | i = GetInteger(); | |
750 | size = i << 10; /* Mbytes to Kbytes */ | |
751 | if (size <= 0) | |
752 | fatal("storeCossDirParse: invalid size value"); | |
cd748f27 | 753 | |
e6ccf245 | 754 | cs = (CossInfo *)xmalloc(sizeof(CossInfo)); |
cd748f27 | 755 | if (cs == NULL) |
756 | fatal("storeCossDirParse: couldn't xmalloc() CossInfo!\n"); | |
757 | ||
758 | sd->index = index; | |
759 | sd->path = xstrdup(path); | |
760 | sd->max_size = size; | |
761 | sd->fsdata = cs; | |
762 | ||
763 | cs->fd = -1; | |
764 | cs->swaplog_fd = -1; | |
cd748f27 | 765 | |
766 | sd->init = storeCossDirInit; | |
767 | sd->newfs = storeCossDirNewfs; | |
768 | sd->dump = storeCossDirDump; | |
769 | sd->freefs = storeCossDirShutdown; | |
770 | sd->dblcheck = NULL; | |
771 | sd->statfs = storeCossDirStats; | |
772 | sd->maintainfs = NULL; | |
773 | sd->checkobj = storeCossDirCheckObj; | |
a4b8110e | 774 | sd->refobj = NULL; /* LRU is done in storeCossRead */ |
cd748f27 | 775 | sd->unrefobj = NULL; |
0e23343f | 776 | sd->callback = storeCossDirCallback; |
6a566b9c | 777 | sd->sync = storeCossSync; |
cd748f27 | 778 | |
779 | sd->obj.create = storeCossCreate; | |
780 | sd->obj.open = storeCossOpen; | |
781 | sd->obj.close = storeCossClose; | |
782 | sd->obj.read = storeCossRead; | |
783 | sd->obj.write = storeCossWrite; | |
784 | sd->obj.unlink = storeCossUnlink; | |
785 | ||
786 | sd->log.open = storeCossDirOpenSwapLog; | |
787 | sd->log.close = storeCossDirCloseSwapLog; | |
788 | sd->log.write = storeCossDirSwapLog; | |
6a566b9c | 789 | sd->log.clean.start = storeCossDirWriteCleanStart; |
cd748f27 | 790 | sd->log.clean.write = storeCossDirWriteCleanEntry; |
6a566b9c | 791 | sd->log.clean.nextentry = storeCossDirCleanLogNextEntry; |
792 | sd->log.clean.done = storeCossDirWriteCleanDone; | |
cd748f27 | 793 | |
794 | cs->current_offset = 0; | |
795 | cs->fd = -1; | |
796 | cs->swaplog_fd = -1; | |
797 | cs->numcollisions = 0; | |
c94af0b9 | 798 | cs->membufs.head = cs->membufs.tail = NULL; /* set when the rebuild completes */ |
799 | cs->current_membuf = NULL; | |
6a566b9c | 800 | cs->index.head = NULL; |
801 | cs->index.tail = NULL; | |
8e8d4f30 | 802 | |
803 | parse_cachedir_options(sd, NULL, 0); | |
c94af0b9 | 804 | /* Enforce maxobjsize being set to something */ |
805 | if (sd->max_objsize == -1) | |
806 | fatal("COSS requires max-size to be set to something other than -1!\n"); | |
cd748f27 | 807 | } |
808 | ||
809 | ||
810 | static void | |
a4b8110e | 811 | storeCossDirReconfigure(SwapDir * sd, int index, char *path) |
cd748f27 | 812 | { |
cd748f27 | 813 | unsigned int i; |
814 | unsigned int size; | |
cd748f27 | 815 | |
816 | i = GetInteger(); | |
817 | size = i << 10; /* Mbytes to Kbytes */ | |
818 | if (size <= 0) | |
819 | fatal("storeCossDirParse: invalid size value"); | |
cd748f27 | 820 | |
e6ccf245 | 821 | if (size == (size_t)sd->max_size) |
a4b8110e | 822 | debug(3, 1) ("Cache COSS dir '%s' size remains unchanged at %d KB\n", path, size); |
cd748f27 | 823 | else { |
a4b8110e | 824 | debug(3, 1) ("Cache COSS dir '%s' size changed to %d KB\n", path, size); |
825 | sd->max_size = size; | |
cd748f27 | 826 | } |
8e8d4f30 | 827 | parse_cachedir_options(sd, NULL, 1); |
c94af0b9 | 828 | /* Enforce maxobjsize being set to something */ |
829 | if (sd->max_objsize == -1) | |
830 | fatal("COSS requires max-size to be set to something other than -1!\n"); | |
cd748f27 | 831 | } |
832 | ||
833 | void | |
25a77f03 | 834 | storeCossDirDump(StoreEntry * entry, SwapDir * s) |
cd748f27 | 835 | { |
25a77f03 | 836 | storeAppendPrintf(entry, " %d", |
cd748f27 | 837 | s->max_size >> 20); |
4464fd13 | 838 | dump_cachedir_options(entry, NULL, s); |
cd748f27 | 839 | } |
840 | ||
6a566b9c | 841 | #if OLD_UNUSED_CODE |
cd748f27 | 842 | SwapDir * |
843 | storeCossDirPick(void) | |
844 | { | |
a4b8110e | 845 | int i, choosenext = 0; |
cd748f27 | 846 | SwapDir *SD; |
a4b8110e | 847 | |
cd748f27 | 848 | if (n_coss_dirs == 0) |
a4b8110e | 849 | return NULL; |
cd748f27 | 850 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
851 | SD = &Config.cacheSwap.swapDirs[i]; | |
a4b8110e | 852 | if (SD->type == SWAPDIR_COSS) { |
853 | if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { | |
854 | last_coss_pick_index = i; | |
855 | return SD; | |
856 | } else if (choosenext) { | |
857 | last_coss_pick_index = i; | |
858 | return SD; | |
859 | } else if (last_coss_pick_index == i) { | |
860 | choosenext = 1; | |
861 | } | |
862 | } | |
cd748f27 | 863 | } |
864 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { | |
865 | SD = &Config.cacheSwap.swapDirs[i]; | |
a4b8110e | 866 | if (SD->type == SWAPDIR_COSS) { |
867 | if ((last_coss_pick_index == -1) || (n_coss_dirs == 1)) { | |
868 | last_coss_pick_index = i; | |
869 | return SD; | |
870 | } else if (choosenext) { | |
871 | last_coss_pick_index = i; | |
872 | return SD; | |
873 | } else if (last_coss_pick_index == i) { | |
874 | choosenext = 1; | |
875 | } | |
876 | } | |
cd748f27 | 877 | } |
a4b8110e | 878 | return NULL; |
cd748f27 | 879 | } |
880 | #endif | |
881 | ||
882 | /* | |
883 | * initial setup/done code | |
884 | */ | |
885 | static void | |
886 | storeCossDirDone(void) | |
887 | { | |
d96ceb8e | 888 | memPoolDestroy(&coss_state_pool); |
889 | /* memPoolDestroy(&coss_index_pool); XXX Should be here? */ | |
cd748f27 | 890 | coss_initialised = 0; |
891 | } | |
892 | ||
893 | void | |
a4b8110e | 894 | storeFsSetup_coss(storefs_entry_t * storefs) |
cd748f27 | 895 | { |
896 | assert(!coss_initialised); | |
897 | ||
898 | storefs->parsefunc = storeCossDirParse; | |
899 | storefs->reconfigurefunc = storeCossDirReconfigure; | |
900 | storefs->donefunc = storeCossDirDone; | |
cd748f27 | 901 | coss_state_pool = memPoolCreate("COSS IO State data", sizeof(CossState)); |
6a566b9c | 902 | coss_index_pool = memPoolCreate("COSS index data", sizeof(CossIndexNode)); |
cd748f27 | 903 | coss_initialised = 1; |
904 | } |