]>
Commit | Line | Data |
---|---|---|
f1dc9b30 | 1 | |
2 | /* | |
9ca759c5 | 3 | * $Id: store_dir.cc,v 1.71 1998/06/08 17:00:06 wessels Exp $ |
f1dc9b30 | 4 | * |
5 | * DEBUG: section 47 Store Directory Routines | |
6 | * AUTHOR: Duane Wessels | |
7 | * | |
8 | * SQUID Internet Object Cache http://squid.nlanr.net/Squid/ | |
9 | * -------------------------------------------------------- | |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from the | |
12 | * Internet community. Development is led by Duane Wessels of the | |
13 | * National Laboratory for Applied Network Research and funded by | |
14 | * the National Science Foundation. | |
15 | * | |
16 | * This program is free software; you can redistribute it and/or modify | |
17 | * it under the terms of the GNU General Public License as published by | |
18 | * the Free Software Foundation; either version 2 of the License, or | |
19 | * (at your option) any later version. | |
20 | * | |
21 | * This program is distributed in the hope that it will be useful, | |
22 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
23 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
24 | * GNU General Public License for more details. | |
25 | * | |
26 | * You should have received a copy of the GNU General Public License | |
27 | * along with this program; if not, write to the Free Software | |
28 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
29 | * | |
30 | */ | |
31 | ||
596dddc1 | 32 | #include "squid.h" |
596dddc1 | 33 | |
34 | #define SWAP_DIR_SHIFT 24 | |
35 | #define SWAP_FILE_MASK 0x00FFFFFF | |
36 | #define DefaultLevelOneDirs 16 | |
37 | #define DefaultLevelTwoDirs 256 | |
38 | ||
85407535 | 39 | static char *storeSwapSubDir(int dirn, int subdirn); |
a2899918 | 40 | static int storeDirSelectSwapDir(void); |
85407535 | 41 | static int storeVerifyDirectory(const char *path); |
d78185ed | 42 | static int storeCreateDirectory(const char *path, int); |
85407535 | 43 | static void storeCreateSwapSubDirs(int j); |
44 | ||
596dddc1 | 45 | /* return full name to swapfile */ |
46 | char * | |
47 | storeSwapFullPath(int fn, char *fullpath) | |
48 | { | |
49 | LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); | |
f1dc9b30 | 50 | int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured; |
596dddc1 | 51 | int filn = fn & SWAP_FILE_MASK; |
27a4d8b1 | 52 | int L1 = Config.cacheSwap.swapDirs[dirn].l1; |
53 | int L2 = Config.cacheSwap.swapDirs[dirn].l2; | |
596dddc1 | 54 | if (!fullpath) |
55 | fullpath = fullfilename; | |
56 | fullpath[0] = '\0'; | |
6cf028ab | 57 | snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X/%08X", |
58 | Config.cacheSwap.swapDirs[dirn].path, | |
27a4d8b1 | 59 | ((filn / L2) / L2) % L1, |
60 | (filn / L2) % L2, | |
6cf028ab | 61 | filn); |
596dddc1 | 62 | return fullpath; |
63 | } | |
64 | ||
85407535 | 65 | static char * |
66 | storeSwapSubDir(int dirn, int subdirn) | |
67 | { | |
68 | LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); | |
69 | SwapDir *SD; | |
70 | assert(0 <= dirn && dirn < Config.cacheSwap.n_configured); | |
71 | SD = &Config.cacheSwap.swapDirs[dirn]; | |
72 | assert(0 <= subdirn && subdirn < SD->l1); | |
73 | snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%02X", | |
74 | Config.cacheSwap.swapDirs[dirn].path, | |
75 | subdirn); | |
76 | return fullfilename; | |
77 | } | |
78 | ||
0c04c389 | 79 | char * |
80 | storeSwapSubSubDir(int fn, char *fullpath) | |
81 | { | |
82 | LOCAL_ARRAY(char, fullfilename, SQUID_MAXPATHLEN); | |
f1dc9b30 | 83 | int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured; |
0c04c389 | 84 | int filn = fn & SWAP_FILE_MASK; |
27a4d8b1 | 85 | int L1 = Config.cacheSwap.swapDirs[dirn].l1; |
86 | int L2 = Config.cacheSwap.swapDirs[dirn].l2; | |
0c04c389 | 87 | if (!fullpath) |
88 | fullpath = fullfilename; | |
89 | fullpath[0] = '\0'; | |
56878878 | 90 | snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", |
6cf028ab | 91 | Config.cacheSwap.swapDirs[dirn].path, |
27a4d8b1 | 92 | ((filn / L2) / L2) % L1, |
93 | (filn / L2) % L2); | |
0c04c389 | 94 | return fullpath; |
95 | } | |
96 | ||
68c2e6f7 | 97 | /* |
98 | * Does swapfile number 'fn' belong in cachedir #F0, | |
99 | * level1 dir #F1, level2 dir #F2? | |
100 | * | |
101 | * This is called by storeDirClean(), but placed here because | |
102 | * the algorithm needs to match storeSwapSubSubDir(). | |
b109de6b | 103 | * |
104 | * Don't check that (fn >> SWAP_DIR_SHIFT) == F0 because | |
105 | * 'fn' may not have the directory bits set. | |
68c2e6f7 | 106 | */ |
107 | int | |
108 | storeFilenoBelongsHere(int fn, int F0, int F1, int F2) | |
109 | { | |
b109de6b | 110 | int D1, D2; |
68c2e6f7 | 111 | int L1, L2; |
112 | int filn = fn & SWAP_FILE_MASK; | |
b109de6b | 113 | assert(F0 < Config.cacheSwap.n_configured); |
114 | L1 = Config.cacheSwap.swapDirs[F0].l1; | |
115 | L2 = Config.cacheSwap.swapDirs[F0].l2; | |
68c2e6f7 | 116 | D1 = ((filn / L2) / L2) % L1; |
117 | if (F1 != D1) | |
118 | return 0; | |
119 | D2 = (filn / L2) % L2; | |
120 | if (F2 != D2) | |
121 | return 0; | |
122 | return 1; | |
123 | } | |
124 | ||
d78185ed | 125 | static int |
126 | storeCreateDirectory(const char *path, int should_exist) | |
85407535 | 127 | { |
d78185ed | 128 | int created = 0; |
85407535 | 129 | struct stat st; |
9ca759c5 | 130 | getCurrentTime(); |
d78185ed | 131 | if (0 == stat(path, &st)) { |
132 | if (S_ISDIR(st.st_mode)) { | |
133 | debug(20, should_exist ? 3 : 1) ("%s exists\n", path); | |
134 | } else { | |
137ee196 | 135 | fatalf("Swap directory %s is not a directory.", path); |
d78185ed | 136 | } |
137 | } else if (0 == mkdir(path, 0755)) { | |
138 | debug(20, should_exist ? 1 : 3) ("%s created\n", path); | |
139 | created = 1; | |
85407535 | 140 | } else { |
137ee196 | 141 | fatalf("Failed to make swap directory %s: %s", |
85407535 | 142 | path, xstrerror()); |
85407535 | 143 | } |
d78185ed | 144 | return created; |
85407535 | 145 | } |
146 | ||
596dddc1 | 147 | static int |
85407535 | 148 | storeVerifyDirectory(const char *path) |
596dddc1 | 149 | { |
150 | struct stat sb; | |
85407535 | 151 | if (stat(path, &sb) < 0) { |
152 | debug(20, 0) ("%s: %s\n", path, xstrerror()); | |
153 | return -1; | |
047146c3 | 154 | } |
85407535 | 155 | if (S_ISDIR(sb.st_mode) == 0) { |
156 | debug(20, 0) ("%s is not a directory\n", path); | |
157 | return -1; | |
596dddc1 | 158 | } |
85407535 | 159 | return 0; |
596dddc1 | 160 | } |
161 | ||
85407535 | 162 | /* |
163 | * This function is called by storeInit(). If this returns < 0, | |
164 | * then Squid exits, complains about swap directories not | |
165 | * existing, and instructs the admin to run 'squid -z' | |
166 | */ | |
596dddc1 | 167 | int |
85407535 | 168 | storeVerifyCacheDirs(void) |
169 | { | |
170 | int i; | |
171 | int j; | |
172 | const char *path; | |
173 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { | |
174 | path = Config.cacheSwap.swapDirs[i].path; | |
175 | if (storeVerifyDirectory(path) < 0) | |
176 | return -1; | |
177 | for (j = 0; j < Config.cacheSwap.swapDirs[i].l1; j++) { | |
178 | path = storeSwapSubDir(i, j); | |
179 | if (storeVerifyDirectory(path) < 0) | |
180 | return -1; | |
181 | } | |
182 | } | |
183 | return 0; | |
184 | } | |
185 | ||
186 | void | |
187 | storeCreateSwapDirectories(void) | |
596dddc1 | 188 | { |
189 | int i; | |
190 | const char *path = NULL; | |
f1dc9b30 | 191 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
192 | path = Config.cacheSwap.swapDirs[i].path; | |
85407535 | 193 | debug(47, 3) ("Creating swap space in %s\n", path); |
194 | storeCreateDirectory(path, 0); | |
047146c3 | 195 | storeCreateSwapSubDirs(i); |
596dddc1 | 196 | } |
596dddc1 | 197 | } |
198 | ||
85407535 | 199 | static void |
596dddc1 | 200 | storeCreateSwapSubDirs(int j) |
201 | { | |
202 | int i, k; | |
d78185ed | 203 | int should_exist; |
f1dc9b30 | 204 | SwapDir *SD = &Config.cacheSwap.swapDirs[j]; |
596dddc1 | 205 | LOCAL_ARRAY(char, name, MAXPATHLEN); |
206 | for (i = 0; i < SD->l1; i++) { | |
56878878 | 207 | snprintf(name, MAXPATHLEN, "%s/%02X", SD->path, i); |
d78185ed | 208 | if (storeCreateDirectory(name, 0)) |
209 | should_exist = 0; | |
210 | else | |
211 | should_exist = 1; | |
f1dc9b30 | 212 | debug(47, 1) ("Making directories in %s\n", name); |
596dddc1 | 213 | for (k = 0; k < SD->l2; k++) { |
042461c3 | 214 | snprintf(name, MAXPATHLEN, "%s/%02X/%02X", SD->path, i, k); |
d78185ed | 215 | storeCreateDirectory(name, should_exist); |
596dddc1 | 216 | } |
217 | } | |
218 | } | |
219 | ||
a2899918 | 220 | #if OLD_CODE |
85407535 | 221 | static int |
596dddc1 | 222 | storeMostFreeSwapDir(void) |
223 | { | |
c932b107 | 224 | double least_used = 1.0; |
225 | double this_used; | |
596dddc1 | 226 | int dirn = 0; |
227 | int i; | |
c932b107 | 228 | SwapDir *SD; |
f1dc9b30 | 229 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
230 | SD = &Config.cacheSwap.swapDirs[i]; | |
c932b107 | 231 | this_used = (double) SD->cur_size / SD->max_size; |
232 | if (this_used > least_used) | |
596dddc1 | 233 | continue; |
c932b107 | 234 | if (SD->read_only) |
596dddc1 | 235 | continue; |
c932b107 | 236 | least_used = this_used; |
596dddc1 | 237 | dirn = i; |
238 | } | |
239 | return dirn; | |
240 | } | |
a2899918 | 241 | #endif |
242 | ||
243 | /* | |
244 | *Spread load across least 3/4 of the store directories | |
245 | */ | |
246 | static int | |
247 | storeDirSelectSwapDir(void) | |
248 | { | |
249 | double least_used = 1.0; | |
250 | int dirn; | |
251 | int i, j; | |
252 | SwapDir *SD; | |
253 | static int nleast = 0; | |
254 | static int nconf = 0; | |
255 | static int *dirq = NULL; | |
256 | static double *diru = NULL; | |
257 | /* | |
258 | * Handle simplest case of a single swap directory immediately | |
259 | */ | |
260 | if (Config.cacheSwap.n_configured == 1) | |
261 | return 0; | |
262 | /* | |
263 | * Initialise dirq on the first call or on change of number of dirs | |
264 | */ | |
265 | if (nconf != Config.cacheSwap.n_configured) { | |
266 | nconf = Config.cacheSwap.n_configured; | |
267 | nleast = (nconf * 3) / 4; | |
268 | if (dirq != NULL) | |
269 | xfree(dirq); | |
270 | dirq = (int *) xmalloc(sizeof(int) * nleast); | |
271 | if (diru != NULL) | |
272 | xfree(diru); | |
273 | diru = (double *) xmalloc(sizeof(double) * nconf); | |
274 | for (j = 0; j < nleast; j++) | |
275 | dirq[j] = -1; | |
276 | } | |
277 | /* | |
278 | * Scan for a non-negative dirn in the dirq array and return that one | |
279 | */ | |
280 | dirn = -1; | |
281 | for (j = 0; j < nleast; j++) { | |
282 | dirn = dirq[j]; | |
283 | if (dirn < 0) | |
284 | continue; | |
285 | dirq[j] = -1; | |
286 | break; | |
287 | } | |
288 | /* | |
289 | * If we found a valid dirn return it | |
290 | */ | |
291 | if (dirn >= 0) | |
292 | return dirn; | |
293 | /* | |
294 | * Now for the real guts of the algorithm - building the dirq array | |
295 | */ | |
296 | for (i = 0; i < nconf; i++) { | |
297 | diru[i] = 1.1; | |
298 | SD = &Config.cacheSwap.swapDirs[i]; | |
299 | if (SD->read_only) | |
300 | continue; | |
301 | diru[i] = (double) SD->cur_size; | |
302 | diru[i] /= SD->max_size; | |
303 | } | |
304 | for (j = 0; j < nleast; j++) { | |
305 | dirq[j] = -1; | |
306 | least_used = 1.0; | |
307 | dirn = -1; | |
308 | for (i = 0; i < nconf; i++) { | |
309 | if (diru[i] < least_used) { | |
310 | least_used = diru[i]; | |
311 | dirn = i; | |
312 | } | |
313 | } | |
314 | if (dirn < 0) | |
315 | break; | |
316 | dirq[j] = dirn; | |
317 | diru[dirn] = 1.1; | |
318 | } | |
319 | /* | |
320 | * Setup default return of 0 if no least found | |
321 | */ | |
322 | if (dirq[0] < 0) | |
323 | dirq[0] = 0; | |
324 | dirn = dirq[0]; | |
325 | dirq[0] = -1; | |
326 | return dirn; | |
327 | } | |
596dddc1 | 328 | |
e3ef2b09 | 329 | int |
330 | storeDirValidFileno(int fn) | |
331 | { | |
332 | int dirn = fn >> SWAP_DIR_SHIFT; | |
333 | int filn = fn & SWAP_FILE_MASK; | |
e3ef2b09 | 334 | if (dirn > Config.cacheSwap.n_configured) |
335 | return 0; | |
8c128028 | 336 | if (dirn < 0) |
337 | return 0; | |
338 | if (filn < 0) | |
339 | return 0; | |
340 | if (filn > Config.cacheSwap.swapDirs[dirn].map->max_n_files) | |
e3ef2b09 | 341 | return 0; |
342 | return 1; | |
343 | } | |
344 | ||
0c04c389 | 345 | int |
346 | storeDirMapBitTest(int fn) | |
347 | { | |
348 | int dirn = fn >> SWAP_DIR_SHIFT; | |
349 | int filn = fn & SWAP_FILE_MASK; | |
f1dc9b30 | 350 | return file_map_bit_test(Config.cacheSwap.swapDirs[dirn].map, filn); |
0c04c389 | 351 | } |
352 | ||
596dddc1 | 353 | void |
354 | storeDirMapBitSet(int fn) | |
355 | { | |
356 | int dirn = fn >> SWAP_DIR_SHIFT; | |
357 | int filn = fn & SWAP_FILE_MASK; | |
f1dc9b30 | 358 | file_map_bit_set(Config.cacheSwap.swapDirs[dirn].map, filn); |
596dddc1 | 359 | } |
360 | ||
361 | void | |
362 | storeDirMapBitReset(int fn) | |
363 | { | |
364 | int dirn = fn >> SWAP_DIR_SHIFT; | |
365 | int filn = fn & SWAP_FILE_MASK; | |
f1dc9b30 | 366 | file_map_bit_reset(Config.cacheSwap.swapDirs[dirn].map, filn); |
596dddc1 | 367 | } |
368 | ||
369 | int | |
370 | storeDirMapAllocate(void) | |
371 | { | |
a2899918 | 372 | int dirn = storeDirSelectSwapDir(); |
f1dc9b30 | 373 | SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; |
596dddc1 | 374 | int filn = file_map_allocate(SD->map, SD->suggest); |
6cf028ab | 375 | SD->suggest = filn + 1; |
a7e9d816 | 376 | return (dirn << SWAP_DIR_SHIFT) | (filn & SWAP_FILE_MASK); |
596dddc1 | 377 | } |
378 | ||
379 | char * | |
380 | storeSwapDir(int dirn) | |
381 | { | |
6d80d9a6 | 382 | assert(0 <= dirn && dirn < Config.cacheSwap.n_configured); |
f1dc9b30 | 383 | return Config.cacheSwap.swapDirs[dirn].path; |
596dddc1 | 384 | } |
4683e377 | 385 | |
5608850b | 386 | int |
387 | storeDirNumber(int swap_file_number) | |
388 | { | |
389 | return swap_file_number >> SWAP_DIR_SHIFT; | |
390 | } | |
391 | ||
c932b107 | 392 | int |
393 | storeDirProperFileno(int dirn, int fn) | |
394 | { | |
395 | return (dirn << SWAP_DIR_SHIFT) | (fn & SWAP_FILE_MASK); | |
396 | } | |
397 | ||
b109de6b | 398 | /* |
399 | * An entry written to the swap log MUST have the following | |
400 | * properties. | |
401 | * 1. It MUST be a public key. It does no good to log | |
402 | * a public ADD, change the key, then log a private | |
403 | * DEL. So we need to log a DEL before we change a | |
404 | * key from public to private. | |
405 | * 2. It MUST have a valid (> -1) swap_file_number. | |
406 | */ | |
4683e377 | 407 | void |
5830cdb3 | 408 | storeDirSwapLog(const StoreEntry * e, int op) |
4683e377 | 409 | { |
6a76db70 | 410 | storeSwapLogData *s; |
4683e377 | 411 | int dirn; |
4683e377 | 412 | dirn = e->swap_file_number >> SWAP_DIR_SHIFT; |
f1dc9b30 | 413 | assert(dirn < Config.cacheSwap.n_configured); |
b109de6b | 414 | assert(!EBIT_TEST(e->flag, KEY_PRIVATE)); |
415 | assert(e->swap_file_number >= 0); | |
6c57e268 | 416 | /* |
417 | * icons and such; don't write them to the swap log | |
418 | */ | |
419 | if (EBIT_TEST(e->flag, ENTRY_SPECIAL)) | |
420 | return; | |
b109de6b | 421 | assert(op > SWAP_LOG_NOP && op < SWAP_LOG_MAX); |
07304bf9 | 422 | debug(20, 3) ("storeDirSwapLog: %s %s %08X\n", |
b109de6b | 423 | swap_log_op_str[op], |
07304bf9 | 424 | storeKeyText(e->key), |
425 | e->swap_file_number); | |
6a76db70 | 426 | s = xcalloc(1, sizeof(storeSwapLogData)); |
5830cdb3 | 427 | s->op = (char) op; |
428 | s->swap_file_number = e->swap_file_number; | |
429 | s->timestamp = e->timestamp; | |
430 | s->lastref = e->lastref; | |
431 | s->expires = e->expires; | |
432 | s->lastmod = e->lastmod; | |
07304bf9 | 433 | s->swap_file_sz = e->swap_file_sz; |
5830cdb3 | 434 | s->refcount = e->refcount; |
435 | s->flags = e->flag; | |
436 | xmemcpy(s->key, e->key, MD5_DIGEST_CHARS); | |
f1dc9b30 | 437 | file_write(Config.cacheSwap.swapDirs[dirn].swaplog_fd, |
d377699f | 438 | -1, |
5830cdb3 | 439 | s, |
07304bf9 | 440 | sizeof(storeSwapLogData), |
5608850b | 441 | NULL, |
442 | NULL, | |
443 | xfree); | |
4683e377 | 444 | } |
445 | ||
5608850b | 446 | char * |
447 | storeDirSwapLogFile(int dirn, const char *ext) | |
4683e377 | 448 | { |
449 | LOCAL_ARRAY(char, path, SQUID_MAXPATHLEN); | |
450 | LOCAL_ARRAY(char, digit, 32); | |
451 | if (Config.Log.swap) { | |
452 | xstrncpy(path, Config.Log.swap, SQUID_MAXPATHLEN - 64); | |
453 | strcat(path, "."); | |
042461c3 | 454 | snprintf(digit, 32, "%02d", dirn); |
5608850b | 455 | strncat(path, digit, 3); |
4683e377 | 456 | } else { |
457 | xstrncpy(path, storeSwapDir(dirn), SQUID_MAXPATHLEN - 64); | |
d0d3ec94 | 458 | strcat(path, "/swap.state"); |
4683e377 | 459 | } |
5608850b | 460 | if (ext) |
461 | strncat(path, ext, 16); | |
4683e377 | 462 | return path; |
463 | } | |
464 | ||
465 | void | |
466 | storeDirOpenSwapLogs(void) | |
467 | { | |
468 | int i; | |
4683e377 | 469 | char *path; |
5608850b | 470 | int fd; |
4683e377 | 471 | SwapDir *SD; |
f1dc9b30 | 472 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
473 | SD = &Config.cacheSwap.swapDirs[i]; | |
5608850b | 474 | path = storeDirSwapLogFile(i, NULL); |
6cf028ab | 475 | fd = file_open(path, O_WRONLY | O_CREAT, NULL, NULL, NULL); |
4683e377 | 476 | if (fd < 0) { |
a3d5953d | 477 | debug(50, 1) ("%s: %s\n", path, xstrerror()); |
4683e377 | 478 | fatal("storeDirOpenSwapLogs: Failed to open swap log."); |
479 | } | |
f1dc9b30 | 480 | debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", i, fd); |
4683e377 | 481 | SD->swaplog_fd = fd; |
482 | } | |
483 | } | |
5608850b | 484 | |
485 | void | |
486 | storeDirCloseSwapLogs(void) | |
487 | { | |
488 | int i; | |
489 | SwapDir *SD; | |
f1dc9b30 | 490 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
491 | SD = &Config.cacheSwap.swapDirs[i]; | |
edd2eb63 | 492 | if (SD->swaplog_fd < 0) /* not open */ |
493 | continue; | |
5608850b | 494 | file_close(SD->swaplog_fd); |
f1dc9b30 | 495 | debug(47, 3) ("Cache Dir #%d log closed on FD %d\n", i, SD->swaplog_fd); |
e7a22b88 | 496 | SD->swaplog_fd = -1; |
5608850b | 497 | } |
498 | } | |
499 | ||
500 | FILE * | |
25535cbe | 501 | storeDirOpenTmpSwapLog(int dirn, int *clean_flag, int *zero_flag) |
5608850b | 502 | { |
503 | char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL)); | |
504 | char *clean_path = xstrdup(storeDirSwapLogFile(dirn, ".last-clean")); | |
505 | char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new")); | |
506 | struct stat log_sb; | |
507 | struct stat clean_sb; | |
f1dc9b30 | 508 | SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; |
5608850b | 509 | FILE *fp; |
510 | int fd; | |
511 | if (stat(swaplog_path, &log_sb) < 0) { | |
f1dc9b30 | 512 | debug(47, 1) ("Cache Dir #%d: No log file\n", dirn); |
5608850b | 513 | safe_free(swaplog_path); |
514 | safe_free(clean_path); | |
515 | safe_free(new_path); | |
516 | return NULL; | |
517 | } | |
25535cbe | 518 | *zero_flag = log_sb.st_size == 0 ? 1 : 0; |
5608850b | 519 | /* close the existing write-only FD */ |
95d15928 | 520 | if (SD->swaplog_fd >= 0) |
4f92c80c | 521 | file_close(SD->swaplog_fd); |
5608850b | 522 | /* open a write-only FD for the new log */ |
6cf028ab | 523 | fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL); |
5608850b | 524 | if (fd < 0) { |
a3d5953d | 525 | debug(50, 1) ("%s: %s\n", new_path, xstrerror()); |
5608850b | 526 | fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); |
527 | } | |
528 | SD->swaplog_fd = fd; | |
529 | /* open a read-only stream of the old log */ | |
530 | fp = fopen(swaplog_path, "r"); | |
531 | if (fp == NULL) { | |
a3d5953d | 532 | debug(50, 0) ("%s: %s\n", swaplog_path, xstrerror()); |
5608850b | 533 | fatal("Failed to open swap log for reading"); |
534 | } | |
f88211e8 | 535 | memset(&clean_sb, '\0', sizeof(struct stat)); |
5608850b | 536 | if (stat(clean_path, &clean_sb) < 0) |
537 | *clean_flag = 0; | |
538 | else if (clean_sb.st_mtime < log_sb.st_mtime) | |
539 | *clean_flag = 0; | |
540 | else | |
541 | *clean_flag = 1; | |
542 | safeunlink(clean_path, 1); | |
543 | safe_free(swaplog_path); | |
544 | safe_free(clean_path); | |
545 | safe_free(new_path); | |
546 | return fp; | |
547 | } | |
548 | ||
549 | void | |
550 | storeDirCloseTmpSwapLog(int dirn) | |
551 | { | |
552 | char *swaplog_path = xstrdup(storeDirSwapLogFile(dirn, NULL)); | |
553 | char *new_path = xstrdup(storeDirSwapLogFile(dirn, ".new")); | |
f1dc9b30 | 554 | SwapDir *SD = &Config.cacheSwap.swapDirs[dirn]; |
5608850b | 555 | int fd; |
df087e68 | 556 | file_close(SD->swaplog_fd); |
5608850b | 557 | if (rename(new_path, swaplog_path) < 0) { |
a3d5953d | 558 | debug(50, 0) ("%s,%s: %s\n", new_path, swaplog_path, xstrerror()); |
5608850b | 559 | fatal("storeDirCloseTmpSwapLog: rename failed"); |
560 | } | |
6cf028ab | 561 | fd = file_open(swaplog_path, O_WRONLY | O_CREAT, NULL, NULL, NULL); |
5608850b | 562 | if (fd < 0) { |
a3d5953d | 563 | debug(50, 1) ("%s: %s\n", swaplog_path, xstrerror()); |
5608850b | 564 | fatal("storeDirCloseTmpSwapLog: Failed to open swap log."); |
565 | } | |
e7a22b88 | 566 | safe_free(swaplog_path); |
567 | safe_free(new_path); | |
5608850b | 568 | SD->swaplog_fd = fd; |
f1dc9b30 | 569 | debug(47, 3) ("Cache Dir #%d log opened on FD %d\n", dirn, fd); |
5608850b | 570 | } |
76fefb77 | 571 | |
572 | void | |
c932b107 | 573 | storeDirUpdateSwapSize(int fn, size_t size, int sign) |
76fefb77 | 574 | { |
f1dc9b30 | 575 | int dirn = (fn >> SWAP_DIR_SHIFT) % Config.cacheSwap.n_configured; |
c932b107 | 576 | int k = ((size + 1023) >> 10) * sign; |
f1dc9b30 | 577 | Config.cacheSwap.swapDirs[dirn].cur_size += k; |
76fefb77 | 578 | store_swap_size += k; |
0faf70d0 | 579 | if (sign > 0) |
580 | n_disk_objects++; | |
581 | else if (sign < 0) | |
582 | n_disk_objects--; | |
76fefb77 | 583 | } |
c932b107 | 584 | |
585 | void | |
3e347513 | 586 | storeDirStats(StoreEntry * sentry) |
c932b107 | 587 | { |
588 | int i; | |
589 | SwapDir *SD; | |
590 | storeAppendPrintf(sentry, "Store Directory Statistics:\n"); | |
3f6c0fb2 | 591 | storeAppendPrintf(sentry, "Store Entries : %d\n", |
592 | memInUse(MEM_STOREENTRY)); | |
593 | storeAppendPrintf(sentry, "Maximum Swap Size : %8d KB\n", | |
594 | Config.Swap.maxSize); | |
595 | storeAppendPrintf(sentry, "Current Store Swap Size: %8d KB\n", | |
596 | store_swap_size); | |
dd8037ab | 597 | storeAppendPrintf(sentry, "Current Capacity : %d%% used, %d%% free\n", |
598 | percent((int) store_swap_size, (int) Config.Swap.maxSize), | |
38d04788 | 599 | percent((int) (Config.Swap.maxSize - store_swap_size), (int) Config.Swap.maxSize)); |
f1dc9b30 | 600 | for (i = 0; i < Config.cacheSwap.n_configured; i++) { |
601 | SD = &Config.cacheSwap.swapDirs[i]; | |
c932b107 | 602 | storeAppendPrintf(sentry, "\n"); |
603 | storeAppendPrintf(sentry, "Store Directory #%d: %s\n", i, SD->path); | |
604 | storeAppendPrintf(sentry, "First level subdirectories: %d\n", SD->l1); | |
605 | storeAppendPrintf(sentry, "Second level subdirectories: %d\n", SD->l2); | |
606 | storeAppendPrintf(sentry, "Maximum Size: %d KB\n", SD->max_size); | |
607 | storeAppendPrintf(sentry, "Current Size: %d KB\n", SD->cur_size); | |
608 | storeAppendPrintf(sentry, "Percent Used: %0.2f%%\n", | |
3e347513 | 609 | 100.0 * SD->cur_size / SD->max_size); |
0ce5d801 | 610 | storeAppendPrintf(sentry, "Filemap bits in use: %d of %d (%d%%)\n", |
611 | SD->map->n_files_in_map, SD->map->max_n_files, | |
612 | percent(SD->map->n_files_in_map, SD->map->max_n_files)); | |
c932b107 | 613 | } |
614 | } | |
e3ef2b09 | 615 | |
5d406e78 | 616 | int |
617 | storeDirMapBitsInUse(void) | |
618 | { | |
619 | int i; | |
620 | int n = 0; | |
621 | for (i = 0; i < Config.cacheSwap.n_configured; i++) | |
622 | n += Config.cacheSwap.swapDirs[i].map->n_files_in_map; | |
623 | return n; | |
624 | } | |
625 | ||
e3ef2b09 | 626 | /* |
627 | * storeDirWriteCleanLogs | |
628 | * | |
629 | * Writes a "clean" swap log file from in-memory metadata. | |
630 | */ | |
631 | #define CLEAN_BUF_SZ 16384 | |
632 | int | |
633 | storeDirWriteCleanLogs(int reopen) | |
634 | { | |
635 | StoreEntry *e = NULL; | |
636 | int *fd; | |
e3ef2b09 | 637 | int n = 0; |
638 | time_t start, stop, r; | |
639 | struct stat sb; | |
640 | char **cur; | |
641 | char **new; | |
642 | char **cln; | |
643 | int dirn; | |
ea524965 | 644 | int N = Config.cacheSwap.n_configured; |
e3ef2b09 | 645 | dlink_node *m; |
7be811df | 646 | char **outbuf; |
647 | off_t *outbufoffset; | |
9748816a | 648 | storeSwapLogData s; |
649 | size_t ss = sizeof(storeSwapLogData); | |
e3ef2b09 | 650 | if (store_rebuilding) { |
651 | debug(20, 1) ("Not currently OK to rewrite swap log.\n"); | |
652 | debug(20, 1) ("storeDirWriteCleanLogs: Operation aborted.\n"); | |
e3ef2b09 | 653 | return 0; |
654 | } | |
655 | debug(20, 1) ("storeDirWriteCleanLogs: Starting...\n"); | |
656 | start = squid_curtime; | |
ea524965 | 657 | fd = xcalloc(N, sizeof(int)); |
658 | cur = xcalloc(N, sizeof(char *)); | |
659 | new = xcalloc(N, sizeof(char *)); | |
660 | cln = xcalloc(N, sizeof(char *)); | |
661 | for (dirn = 0; dirn < N; dirn++) { | |
e3ef2b09 | 662 | fd[dirn] = -1; |
663 | cur[dirn] = xstrdup(storeDirSwapLogFile(dirn, NULL)); | |
664 | new[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".clean")); | |
665 | cln[dirn] = xstrdup(storeDirSwapLogFile(dirn, ".last-clean")); | |
666 | unlink(new[dirn]); | |
667 | unlink(cln[dirn]); | |
668 | fd[dirn] = file_open(new[dirn], | |
669 | O_WRONLY | O_CREAT | O_TRUNC, | |
670 | NULL, | |
671 | NULL, | |
672 | NULL); | |
673 | if (fd[dirn] < 0) { | |
674 | debug(50, 0) ("storeDirWriteCleanLogs: %s: %s\n", new[dirn], xstrerror()); | |
675 | continue; | |
676 | } | |
27a4d8b1 | 677 | debug(20, 3) ("storeDirWriteCleanLogs: opened %s, FD %d\n", |
678 | new[dirn], fd[dirn]); | |
e3ef2b09 | 679 | #if HAVE_FCHMOD |
680 | if (stat(cur[dirn], &sb) == 0) | |
681 | fchmod(fd[dirn], sb.st_mode); | |
682 | #endif | |
683 | } | |
ea524965 | 684 | outbuf = xcalloc(N, sizeof(char *)); |
685 | outbufoffset = xcalloc(N, sizeof(*outbufoffset)); | |
686 | for (dirn = 0; dirn < N; dirn++) { | |
687 | outbuf[dirn] = xcalloc(CLEAN_BUF_SZ, 1); | |
7be811df | 688 | outbufoffset[dirn] = 0; |
e3ef2b09 | 689 | } |
690 | for (m = store_list.tail; m; m = m->prev) { | |
691 | e = m->data; | |
692 | if (e->swap_file_number < 0) | |
693 | continue; | |
694 | if (e->swap_status != SWAPOUT_DONE) | |
695 | continue; | |
07304bf9 | 696 | if (e->swap_file_sz <= 0) |
e3ef2b09 | 697 | continue; |
698 | if (EBIT_TEST(e->flag, RELEASE_REQUEST)) | |
699 | continue; | |
700 | if (EBIT_TEST(e->flag, KEY_PRIVATE)) | |
701 | continue; | |
6c57e268 | 702 | if (EBIT_TEST(e->flag, ENTRY_SPECIAL)) |
703 | continue; | |
e3ef2b09 | 704 | dirn = storeDirNumber(e->swap_file_number); |
ea524965 | 705 | assert(dirn < N); |
e3ef2b09 | 706 | if (fd[dirn] < 0) |
707 | continue; | |
9748816a | 708 | memset(&s, '\0', ss); |
709 | s.op = (char) SWAP_LOG_ADD; | |
710 | s.swap_file_number = e->swap_file_number; | |
711 | s.timestamp = e->timestamp; | |
712 | s.lastref = e->lastref; | |
713 | s.expires = e->expires; | |
714 | s.lastmod = e->lastmod; | |
715 | s.swap_file_sz = e->swap_file_sz; | |
716 | s.refcount = e->refcount; | |
717 | s.flags = e->flag; | |
718 | xmemcpy(&s.key, e->key, MD5_DIGEST_CHARS); | |
719 | xmemcpy(outbuf[dirn] + outbufoffset[dirn], &s, ss); | |
720 | outbufoffset[dirn] += ss; | |
e3ef2b09 | 721 | /* buffered write */ |
9748816a | 722 | if (outbufoffset[dirn] + ss > CLEAN_BUF_SZ) { |
7be811df | 723 | if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) { |
25354045 | 724 | debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", |
25535cbe | 725 | new[dirn], xstrerror()); |
e3ef2b09 | 726 | debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); |
727 | file_close(fd[dirn]); | |
728 | fd[dirn] = -1; | |
25354045 | 729 | unlink(new[dirn]); |
e3ef2b09 | 730 | continue; |
731 | } | |
7be811df | 732 | outbufoffset[dirn] = 0; |
e3ef2b09 | 733 | } |
bcf3a738 | 734 | if ((++n & 0xFFFF) == 0) { |
e3ef2b09 | 735 | getCurrentTime(); |
7be811df | 736 | debug(20, 1) (" %7d entries written so far.\n", n); |
e3ef2b09 | 737 | } |
738 | } | |
e3ef2b09 | 739 | /* flush */ |
ea524965 | 740 | for (dirn = 0; dirn < N; dirn++) { |
7be811df | 741 | if (outbufoffset[dirn] == 0) |
742 | continue; | |
25354045 | 743 | if (fd[dirn] < 0) |
744 | continue; | |
7be811df | 745 | if (write(fd[dirn], outbuf[dirn], outbufoffset[dirn]) < 0) { |
25354045 | 746 | debug(50, 0) ("storeDirWriteCleanLogs: %s: write: %s\n", |
747 | new[dirn], xstrerror()); | |
7be811df | 748 | debug(20, 0) ("storeDirWriteCleanLogs: Current swap logfile not replaced.\n"); |
749 | file_close(fd[dirn]); | |
750 | fd[dirn] = -1; | |
25354045 | 751 | unlink(new[dirn]); |
7be811df | 752 | continue; |
e3ef2b09 | 753 | } |
7be811df | 754 | safe_free(outbuf[dirn]); |
e3ef2b09 | 755 | } |
7be811df | 756 | safe_free(outbuf); |
757 | safe_free(outbufoffset); | |
1ab2fda1 | 758 | #ifdef _SQUID_MSWIN_ |
759 | /* | |
760 | * You can't rename open files on Microsoft "operating systems" | |
761 | * so we close before renaming. | |
762 | */ | |
763 | storeDirCloseSwapLogs(); | |
764 | #endif | |
25354045 | 765 | /* rename */ |
ea524965 | 766 | for (dirn = 0; dirn < N; dirn++) { |
25354045 | 767 | if (fd[dirn] < 0) |
768 | continue; | |
e3ef2b09 | 769 | if (rename(new[dirn], cur[dirn]) < 0) { |
770 | debug(50, 0) ("storeDirWriteCleanLogs: rename failed: %s, %s -> %s\n", | |
771 | xstrerror(), new[dirn], cur[dirn]); | |
772 | } | |
773 | } | |
1ab2fda1 | 774 | #ifndef _SQUID_MSWIN_ |
e3ef2b09 | 775 | storeDirCloseSwapLogs(); |
1ab2fda1 | 776 | #endif |
e3ef2b09 | 777 | if (reopen) |
778 | storeDirOpenSwapLogs(); | |
779 | stop = squid_curtime; | |
780 | r = stop - start; | |
7be811df | 781 | debug(20, 1) (" Finished. Wrote %d entries.\n", n); |
5f6ac48b | 782 | debug(20, 1) (" Took %d seconds (%6.1f entries/sec).\n", |
783 | r > 0 ? (int) r : 0, | |
784 | (double) n / (r > 0 ? r : 1)); | |
e3ef2b09 | 785 | /* touch a timestamp file if we're not still validating */ |
25354045 | 786 | if (!store_rebuilding) { |
ea524965 | 787 | for (dirn = 0; dirn < N; dirn++) { |
25354045 | 788 | if (fd[dirn] < 0) |
27a4d8b1 | 789 | continue; |
e3ef2b09 | 790 | file_close(file_open(cln[dirn], |
27a4d8b1 | 791 | O_WRONLY | O_CREAT | O_TRUNC, NULL, NULL, NULL)); |
792 | } | |
25354045 | 793 | } |
794 | /* close */ | |
ea524965 | 795 | for (dirn = 0; dirn < N; dirn++) { |
e3ef2b09 | 796 | safe_free(cur[dirn]); |
797 | safe_free(new[dirn]); | |
798 | safe_free(cln[dirn]); | |
25354045 | 799 | if (fd[dirn] < 0) |
800 | continue; | |
801 | file_close(fd[dirn]); | |
802 | fd[dirn] = -1; | |
e3ef2b09 | 803 | } |
804 | safe_free(cur); | |
805 | safe_free(new); | |
806 | safe_free(cln); | |
807 | safe_free(fd); | |
808 | return n; | |
809 | } | |
810 | #undef CLEAN_BUF_SZ |