]>
Commit | Line | Data |
---|---|---|
cd748f27 | 1 | |
2 | /* | |
262a0e14 | 3 | * $Id$ |
cd748f27 | 4 | * |
5 | * DEBUG: section 47 Store Directory Routines | |
6 | * AUTHOR: Duane Wessels | |
7 | * | |
2b6662ba | 8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
cd748f27 | 9 | * ---------------------------------------------------------- |
10 | * | |
2b6662ba | 11 | * Squid is the result of efforts by numerous individuals from |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
cd748f27 | 19 | * |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
26ac0430 | 24 | * |
cd748f27 | 25 | * This program is distributed in the hope that it will be useful, |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
26ac0430 | 29 | * |
cd748f27 | 30 | * You should have received a copy of the GNU General Public License |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
34 | */ | |
35 | ||
f7f3304a | 36 | #include "squid-old.h" |
e6ccf245 | 37 | #include "Store.h" |
528b2c61 | 38 | #include "fde.h" |
4d6d905e | 39 | #include "ufscommon.h" |
51ee7c82 | 40 | #include "StoreSwapLogData.h" |
b9ae18aa | 41 | #include "ConfigOption.h" |
42 | #include "DiskIO/DiskIOStrategy.h" | |
43 | #include "DiskIO/DiskIOModule.h" | |
75f8f9a2 | 44 | #include "FileMap.h" |
c8f4eac4 | 45 | #include "Parsing.h" |
a98bcbee | 46 | #include "SquidMath.h" |
985c86bc | 47 | #include "SquidTime.h" |
e4f1fdae | 48 | #include "StatCounters.h" |
d3b3ab85 | 49 | #include "SwapDir.h" |
4b981814 | 50 | #include "swap_log_op.h" |
27e059d4 | 51 | |
d3b3ab85 | 52 | int UFSSwapDir::NumberOfUFSDirs = 0; |
53 | int *UFSSwapDir::UFSDirToGlobalDirMapping = NULL; | |
cd748f27 | 54 | |
cd748f27 | 55 | /* |
56 | * storeUfsDirCheckObj | |
57 | * | |
58 | * This routine is called by storeDirSelectSwapDir to see if the given | |
59 | * object is able to be stored on this filesystem. UFS filesystems will | |
60 | * happily store anything as long as the LRU time isn't too small. | |
61 | */ | |
aa1a691e AR |
62 | bool |
63 | UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const | |
cd748f27 | 64 | { |
aa1a691e AR |
65 | if (!SwapDir::canStore(e, diskSpaceNeeded, load)) |
66 | return false; | |
67 | ||
59b2d47f | 68 | if (IO->shedLoad()) |
aa1a691e | 69 | return false; |
62e76326 | 70 | |
aa1a691e AR |
71 | load = IO->load(); |
72 | return true; | |
cd748f27 | 73 | } |
74 | ||
8e8d4f30 | 75 | |
59b2d47f | 76 | /* ========== LOCAL FUNCTIONS ABOVE, GLOBAL FUNCTIONS BELOW ========== */ |
d3b3ab85 | 77 | |
78 | void | |
59b2d47f | 79 | UFSSwapDir::parseSizeL1L2() |
cd748f27 | 80 | { |
6da430a4 AJ |
81 | int i = GetInteger(); |
82 | if (i <= 0) | |
59b2d47f | 83 | fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); |
cd748f27 | 84 | |
a0a5b161 | 85 | const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes |
6da430a4 | 86 | |
cd748f27 | 87 | /* just reconfigure it */ |
0931cb03 | 88 | if (reconfiguring) { |
cc34568d DK |
89 | if (size == maxSize()) |
90 | debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); | |
0931cb03 | 91 | else |
cc34568d | 92 | debugs(3, 1, "Cache dir '" << path << "' size changed to " << i << " MB"); |
0931cb03 | 93 | } |
62e76326 | 94 | |
d3b3ab85 | 95 | max_size = size; |
59b2d47f | 96 | |
97 | l1 = GetInteger(); | |
98 | ||
99 | if (l1 <= 0) | |
100 | fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value"); | |
101 | ||
102 | l2 = GetInteger(); | |
103 | ||
104 | if (l2 <= 0) | |
105 | fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); | |
cd748f27 | 106 | } |
107 | ||
59b2d47f | 108 | /* |
109 | * storeUfsDirReconfigure | |
110 | * | |
26ac0430 | 111 | * This routine is called when the given swapdir needs reconfiguring |
59b2d47f | 112 | */ |
113 | ||
cd748f27 | 114 | void |
c6059970 | 115 | UFSSwapDir::reconfigure() |
cd748f27 | 116 | { |
59b2d47f | 117 | parseSizeL1L2(); |
118 | parseOptions(1); | |
cd748f27 | 119 | } |
120 | ||
cd748f27 | 121 | /* |
122 | * storeUfsDirParse | |
123 | * | |
124 | * Called when a *new* fs is being setup. | |
125 | */ | |
d3b3ab85 | 126 | void |
59b2d47f | 127 | UFSSwapDir::parse (int anIndex, char *aPath) |
cd748f27 | 128 | { |
59b2d47f | 129 | index = anIndex; |
59b2d47f | 130 | path = xstrdup(aPath); |
62e76326 | 131 | |
1cd79a58 | 132 | parseSizeL1L2(); |
133 | ||
59b2d47f | 134 | /* Initialise replacement policy stuff */ |
135 | repl = createRemovalPolicy(Config.replPolicy); | |
62e76326 | 136 | |
59b2d47f | 137 | parseOptions(0); |
138 | } | |
62e76326 | 139 | |
b9ae18aa | 140 | void |
141 | UFSSwapDir::changeIO(DiskIOModule *module) | |
142 | { | |
143 | DiskIOStrategy *anIO = module->createStrategy(); | |
144 | safe_free(ioType); | |
145 | ioType = xstrdup(module->type()); | |
146 | ||
147 | delete IO->io; | |
148 | IO->io = anIO; | |
149 | /* Change the IO Options */ | |
150 | ||
91c414dc | 151 | if (currentIOOptions && currentIOOptions->options.size() > 2) |
b9ae18aa | 152 | delete currentIOOptions->options.pop_back(); |
153 | ||
154 | /* TODO: factor out these 4 lines */ | |
155 | ConfigOption *ioOptions = IO->io->getOptionTree(); | |
156 | ||
157 | if (ioOptions) | |
158 | currentIOOptions->options.push_back(ioOptions); | |
159 | } | |
160 | ||
161 | bool | |
18ec8500 | 162 | UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) |
59b2d47f | 163 | { |
b9ae18aa | 164 | if (strcmp(option, "IOEngine") != 0) |
165 | return false; | |
62e76326 | 166 | |
18ec8500 | 167 | if (isaReconfig) |
b9ae18aa | 168 | /* silently ignore this */ |
169 | return true; | |
62e76326 | 170 | |
b9ae18aa | 171 | if (!value) |
172 | self_destruct(); | |
d3b3ab85 | 173 | |
b9ae18aa | 174 | DiskIOModule *module = DiskIOModule::Find(value); |
62e76326 | 175 | |
b9ae18aa | 176 | if (!module) |
177 | self_destruct(); | |
8e8d4f30 | 178 | |
b9ae18aa | 179 | changeIO(module); |
180 | ||
181 | return true; | |
182 | } | |
183 | ||
184 | void | |
185 | UFSSwapDir::optionIODump(StoreEntry * e) const | |
186 | { | |
187 | storeAppendPrintf(e, " IOEngine=%s", ioType); | |
188 | } | |
189 | ||
190 | ConfigOption * | |
191 | UFSSwapDir::getOptionTree() const | |
192 | { | |
193 | ConfigOption *parentResult = SwapDir::getOptionTree(); | |
194 | ||
e2a8733e | 195 | if (currentIOOptions == NULL) |
196 | currentIOOptions = new ConfigOptionVector(); | |
197 | ||
b9ae18aa | 198 | currentIOOptions->options.push_back(parentResult); |
e2a8733e | 199 | |
b9ae18aa | 200 | currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump)); |
201 | ||
5f2d056f | 202 | if (ConfigOption *ioOptions = IO->io->getOptionTree()) |
b9ae18aa | 203 | currentIOOptions->options.push_back(ioOptions); |
204 | ||
e2a8733e | 205 | ConfigOption* result = currentIOOptions; |
206 | ||
207 | currentIOOptions = NULL; | |
208 | ||
209 | return result; | |
cd748f27 | 210 | } |
211 | ||
212 | /* | |
213 | * Initial setup / end destruction | |
214 | */ | |
d3b3ab85 | 215 | void |
216 | UFSSwapDir::init() | |
217 | { | |
d295d770 | 218 | debugs(47, 3, "Initialising UFS SwapDir engine."); |
219 | /* Parsing must be finished by now - force to NULL, don't delete */ | |
b9ae18aa | 220 | currentIOOptions = NULL; |
d3b3ab85 | 221 | static int started_clean_event = 0; |
222 | static const char *errmsg = | |
62e76326 | 223 | "\tFailed to verify one of the swap directories, Check cache.log\n" |
224 | "\tfor details. Run 'squid -z' to create swap directories\n" | |
225 | "\tif needed, or if running Squid for the first time."; | |
59b2d47f | 226 | IO->init(); |
62e76326 | 227 | |
d3b3ab85 | 228 | if (verifyCacheDirs()) |
62e76326 | 229 | fatal(errmsg); |
230 | ||
d3b3ab85 | 231 | openLog(); |
62e76326 | 232 | |
d3b3ab85 | 233 | rebuild(); |
62e76326 | 234 | |
d3b3ab85 | 235 | if (!started_clean_event) { |
62e76326 | 236 | eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1); |
237 | started_clean_event = 1; | |
d3b3ab85 | 238 | } |
62e76326 | 239 | |
d3b3ab85 | 240 | (void) storeDirGetBlkSize(path, &fs.blksize); |
241 | } | |
242 | ||
243 | void | |
c8f4eac4 | 244 | UFSSwapDir::create() |
d3b3ab85 | 245 | { |
d295d770 | 246 | debugs(47, 3, "Creating swap space in " << path); |
d3b3ab85 | 247 | createDirectory(path, 0); |
248 | createSwapSubDirs(); | |
249 | } | |
250 | ||
75f8f9a2 | 251 | UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) |
b7717b61 | 252 | { |
253 | /* modulename is only set to disk modules that are built, by configure, | |
254 | * so the Find call should never return NULL here. | |
255 | */ | |
256 | IO = new UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); | |
257 | } | |
d3b3ab85 | 258 | |
259 | UFSSwapDir::~UFSSwapDir() | |
260 | { | |
261 | if (swaplog_fd > -1) { | |
62e76326 | 262 | file_close(swaplog_fd); |
263 | swaplog_fd = -1; | |
d3b3ab85 | 264 | } |
62e76326 | 265 | |
75f8f9a2 | 266 | delete map; |
62e76326 | 267 | |
d3b3ab85 | 268 | if (IO) |
00d77d6b | 269 | delete IO; |
62e76326 | 270 | |
d3b3ab85 | 271 | IO = NULL; |
b9ae18aa | 272 | |
273 | safe_free(ioType); | |
d3b3ab85 | 274 | } |
275 | ||
d3b3ab85 | 276 | void |
277 | UFSSwapDir::dumpEntry(StoreEntry &e) const | |
278 | { | |
bf8fe701 | 279 | debugs(47, 0, "UFSSwapDir::dumpEntry: FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); |
280 | debugs(47, 0, "UFSSwapDir::dumpEntry: PATH " << fullPath(e.swap_filen, NULL) ); | |
3900307b | 281 | e.dump(0); |
d3b3ab85 | 282 | } |
283 | ||
284 | /* | |
285 | * UFSSwapDir::doubleCheck | |
286 | * | |
287 | * This is called by storeCleanup() if -S was given on the command line. | |
288 | */ | |
289 | bool | |
290 | UFSSwapDir::doubleCheck(StoreEntry & e) | |
291 | { | |
62e76326 | 292 | |
d3b3ab85 | 293 | struct stat sb; |
62e76326 | 294 | |
c8f4eac4 | 295 | if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { |
bf8fe701 | 296 | debugs(47, 0, "UFSSwapDir::doubleCheck: MISSING SWAP FILE"); |
62e76326 | 297 | dumpEntry(e); |
298 | return true; | |
d3b3ab85 | 299 | } |
62e76326 | 300 | |
d3b3ab85 | 301 | if ((off_t)e.swap_file_sz != sb.st_size) { |
bf8fe701 | 302 | debugs(47, 0, "UFSSwapDir::doubleCheck: SIZE MISMATCH"); |
4a7a3d56 | 303 | debugs(47, 0, "UFSSwapDir::doubleCheck: ENTRY SIZE: " << e.swap_file_sz << ", FILE SIZE: " << sb.st_size); |
62e76326 | 304 | dumpEntry(e); |
305 | return true; | |
d3b3ab85 | 306 | } |
62e76326 | 307 | |
d3b3ab85 | 308 | return false; |
309 | } | |
310 | ||
311 | void | |
312 | UFSSwapDir::statfs(StoreEntry & sentry) const | |
313 | { | |
314 | int totl_kb = 0; | |
315 | int free_kb = 0; | |
316 | int totl_in = 0; | |
317 | int free_in = 0; | |
318 | int x; | |
319 | storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); | |
320 | storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); | |
c91ca3ce | 321 | storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10); |
cc34568d | 322 | storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); |
d3b3ab85 | 323 | storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", |
cc34568d | 324 | Math::doublePercent(currentSize(), maxSize())); |
d3b3ab85 | 325 | storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", |
75f8f9a2 FC |
326 | map->numFilesInMap(), map->capacity(), |
327 | Math::intPercent(map->numFilesInMap(), map->capacity())); | |
d3b3ab85 | 328 | x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in); |
62e76326 | 329 | |
d3b3ab85 | 330 | if (0 == x) { |
62e76326 | 331 | storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", |
332 | totl_kb - free_kb, | |
333 | totl_kb, | |
a98bcbee | 334 | Math::intPercent(totl_kb - free_kb, totl_kb)); |
62e76326 | 335 | storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", |
336 | totl_in - free_in, | |
337 | totl_in, | |
a98bcbee | 338 | Math::intPercent(totl_in - free_in, totl_in)); |
d3b3ab85 | 339 | } |
62e76326 | 340 | |
d3b3ab85 | 341 | storeAppendPrintf(&sentry, "Flags:"); |
62e76326 | 342 | |
d3b3ab85 | 343 | if (flags.selected) |
62e76326 | 344 | storeAppendPrintf(&sentry, " SELECTED"); |
345 | ||
d3b3ab85 | 346 | if (flags.read_only) |
62e76326 | 347 | storeAppendPrintf(&sentry, " READ-ONLY"); |
348 | ||
d3b3ab85 | 349 | storeAppendPrintf(&sentry, "\n"); |
59b2d47f | 350 | |
351 | IO->statfs(sentry); | |
d3b3ab85 | 352 | } |
353 | ||
354 | void | |
c8f4eac4 | 355 | UFSSwapDir::maintain() |
d3b3ab85 | 356 | { |
650c4b88 | 357 | /* We can't delete objects while rebuilding swap */ |
358 | ||
bef81ea5 | 359 | /* XXX FIXME each store should start maintaining as it comes online. */ |
360 | ||
361 | if (StoreController::store_dirs_rebuilding) | |
650c4b88 | 362 | return; |
363 | ||
d3b3ab85 | 364 | StoreEntry *e = NULL; |
650c4b88 | 365 | |
d3b3ab85 | 366 | int removed = 0; |
650c4b88 | 367 | |
d3b3ab85 | 368 | RemovalPurgeWalker *walker; |
62e76326 | 369 | |
cc34568d | 370 | double f = (double) (currentSize() - minSize()) / (maxSize() - minSize()); |
650c4b88 | 371 | |
372 | f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; | |
373 | ||
374 | int max_scan = (int) (f * 400.0 + 100.0); | |
375 | ||
376 | int max_remove = (int) (f * 70.0 + 10.0); | |
377 | ||
378 | /* | |
379 | * This is kinda cheap, but so we need this priority hack? | |
380 | */ | |
62e76326 | 381 | |
bf8fe701 | 382 | debugs(47, 3, "storeMaintainSwapSpace: f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); |
650c4b88 | 383 | |
d3b3ab85 | 384 | walker = repl->PurgeInit(repl, max_scan); |
62e76326 | 385 | |
d3b3ab85 | 386 | while (1) { |
cc34568d | 387 | if (currentSize() < minSize()) |
62e76326 | 388 | break; |
389 | ||
390 | if (removed >= max_remove) | |
391 | break; | |
392 | ||
393 | e = walker->Next(walker); | |
394 | ||
395 | if (!e) | |
396 | break; /* no more objects */ | |
397 | ||
cb4185f1 | 398 | ++removed; |
62e76326 | 399 | |
5f33b71d | 400 | e->release(); |
d3b3ab85 | 401 | } |
62e76326 | 402 | |
d3b3ab85 | 403 | walker->Done(walker); |
26ac0430 AJ |
404 | debugs(47, (removed ? 2 : 3), "UFSSwapDir::maintain: " << path << |
405 | " removed " << removed << "/" << max_remove << " f=" << | |
bf8fe701 | 406 | std::setprecision(4) << f << " max_scan=" << max_scan); |
d3b3ab85 | 407 | } |
408 | ||
409 | /* | |
410 | * UFSSwapDir::reference | |
411 | * | |
412 | * This routine is called whenever an object is referenced, so we can | |
413 | * maintain replacement information within the storage fs. | |
414 | */ | |
415 | void | |
416 | UFSSwapDir::reference(StoreEntry &e) | |
417 | { | |
bf8fe701 | 418 | debugs(47, 3, "UFSSwapDir::reference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); |
62e76326 | 419 | |
d3b3ab85 | 420 | if (repl->Referenced) |
62e76326 | 421 | repl->Referenced(repl, &e, &e.repl); |
422 | } | |
d3b3ab85 | 423 | |
424 | /* | |
425 | * UFSSwapDir::dereference | |
426 | * This routine is called whenever the last reference to an object is | |
427 | * removed, to maintain replacement information within the storage fs. | |
428 | */ | |
4c973beb | 429 | bool |
d3b3ab85 | 430 | UFSSwapDir::dereference(StoreEntry & e) |
431 | { | |
bf8fe701 | 432 | debugs(47, 3, "UFSSwapDir::dereference: referencing " << &e << " " << e.swap_dirn << "/" << e.swap_filen); |
62e76326 | 433 | |
d3b3ab85 | 434 | if (repl->Dereferenced) |
62e76326 | 435 | repl->Dereferenced(repl, &e, &e.repl); |
4c973beb AR |
436 | |
437 | return true; // keep e in the global store_table | |
d3b3ab85 | 438 | } |
439 | ||
440 | StoreIOState::Pointer | |
18ec8500 | 441 | UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) |
d3b3ab85 | 442 | { |
18ec8500 | 443 | return IO->create (this, &e, file_callback, aCallback, callback_data); |
d3b3ab85 | 444 | } |
445 | ||
446 | StoreIOState::Pointer | |
18ec8500 | 447 | UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) |
d3b3ab85 | 448 | { |
18ec8500 | 449 | return IO->open (this, &e, file_callback, aCallback, callback_data); |
d3b3ab85 | 450 | } |
451 | ||
452 | int | |
453 | UFSSwapDir::mapBitTest(sfileno filn) | |
454 | { | |
75f8f9a2 | 455 | return map->testBit(filn); |
d3b3ab85 | 456 | } |
457 | ||
458 | void | |
459 | UFSSwapDir::mapBitSet(sfileno filn) | |
460 | { | |
75f8f9a2 | 461 | map->setBit(filn); |
d3b3ab85 | 462 | } |
463 | ||
464 | void | |
465 | UFSSwapDir::mapBitReset(sfileno filn) | |
466 | { | |
467 | /* | |
75f8f9a2 FC |
468 | * We have to test the bit before calling clearBit as |
469 | * it doesn't do bounds checking and blindly assumes | |
d3b3ab85 | 470 | * filn is a valid file number, but it might not be because |
471 | * the map is dynamic in size. Also clearing an already clear | |
472 | * bit puts the map counter of-of-whack. | |
473 | */ | |
62e76326 | 474 | |
75f8f9a2 FC |
475 | if (map->testBit(filn)) |
476 | map->clearBit(filn); | |
d3b3ab85 | 477 | } |
478 | ||
479 | int | |
480 | UFSSwapDir::mapBitAllocate() | |
481 | { | |
482 | int fn; | |
75f8f9a2 FC |
483 | fn = map->allocate(suggest); |
484 | map->setBit(fn); | |
d3b3ab85 | 485 | suggest = fn + 1; |
486 | return fn; | |
487 | } | |
488 | ||
d3b3ab85 | 489 | char * |
490 | UFSSwapDir::swapSubDir(int subdirn)const | |
491 | { | |
ef364f64 | 492 | LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); |
d3b3ab85 | 493 | assert(0 <= subdirn && subdirn < l1); |
ef364f64 | 494 | snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn); |
d3b3ab85 | 495 | return fullfilename; |
496 | } | |
497 | ||
498 | int | |
e4ae841b | 499 | UFSSwapDir::createDirectory(const char *aPath, int should_exist) |
d3b3ab85 | 500 | { |
501 | int created = 0; | |
62e76326 | 502 | |
d3b3ab85 | 503 | struct stat st; |
504 | getCurrentTime(); | |
62e76326 | 505 | |
e4ae841b | 506 | if (0 == ::stat(aPath, &st)) { |
62e76326 | 507 | if (S_ISDIR(st.st_mode)) { |
e4ae841b | 508 | debugs(47, (should_exist ? 3 : 1), aPath << " exists"); |
62e76326 | 509 | } else { |
e4ae841b | 510 | fatalf("Swap directory %s is not a directory.", aPath); |
62e76326 | 511 | } |
e4ae841b | 512 | } else if (0 == mkdir(aPath, 0755)) { |
e4ae841b | 513 | debugs(47, (should_exist ? 1 : 3), aPath << " created"); |
62e76326 | 514 | created = 1; |
d3b3ab85 | 515 | } else { |
62e76326 | 516 | fatalf("Failed to make swap directory %s: %s", |
e4ae841b | 517 | aPath, xstrerror()); |
d3b3ab85 | 518 | } |
62e76326 | 519 | |
d3b3ab85 | 520 | return created; |
521 | } | |
522 | ||
523 | bool | |
e4ae841b | 524 | UFSSwapDir::pathIsDirectory(const char *aPath)const |
d3b3ab85 | 525 | { |
62e76326 | 526 | |
d3b3ab85 | 527 | struct stat sb; |
62e76326 | 528 | |
e4ae841b FC |
529 | if (::stat(aPath, &sb) < 0) { |
530 | debugs(47, 0, "" << aPath << ": " << xstrerror()); | |
62e76326 | 531 | return false; |
d3b3ab85 | 532 | } |
62e76326 | 533 | |
d3b3ab85 | 534 | if (S_ISDIR(sb.st_mode) == 0) { |
e4ae841b | 535 | debugs(47, 0, "" << aPath << " is not a directory"); |
62e76326 | 536 | return false; |
d3b3ab85 | 537 | } |
62e76326 | 538 | |
d3b3ab85 | 539 | return true; |
540 | } | |
541 | ||
542 | /* | |
543 | * This function is called by commonUfsDirInit(). If this returns < 0, | |
544 | * then Squid exits, complains about swap directories not | |
545 | * existing, and instructs the admin to run 'squid -z' | |
546 | */ | |
547 | bool | |
548 | UFSSwapDir::verifyCacheDirs() | |
549 | { | |
550 | if (!pathIsDirectory(path)) | |
62e76326 | 551 | return true; |
552 | ||
cb4185f1 | 553 | for (int j = 0; j < l1; ++j) { |
62e76326 | 554 | char const *aPath = swapSubDir(j); |
555 | ||
556 | if (!pathIsDirectory(aPath)) | |
557 | return true; | |
d3b3ab85 | 558 | } |
62e76326 | 559 | |
d3b3ab85 | 560 | return false; |
561 | } | |
562 | ||
563 | void | |
564 | UFSSwapDir::createSwapSubDirs() | |
565 | { | |
d3b3ab85 | 566 | LOCAL_ARRAY(char, name, MAXPATHLEN); |
62e76326 | 567 | |
cb4185f1 | 568 | for (int i = 0; i < l1; ++i) { |
62e76326 | 569 | snprintf(name, MAXPATHLEN, "%s/%02X", path, i); |
570 | ||
59b2d47f | 571 | int should_exist; |
572 | ||
62e76326 | 573 | if (createDirectory(name, 0)) |
574 | should_exist = 0; | |
575 | else | |
576 | should_exist = 1; | |
577 | ||
bf8fe701 | 578 | debugs(47, 1, "Making directories in " << name); |
62e76326 | 579 | |
cb4185f1 | 580 | for (int k = 0; k < l2; ++k) { |
62e76326 | 581 | snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); |
582 | createDirectory(name, should_exist); | |
583 | } | |
d3b3ab85 | 584 | } |
585 | } | |
586 | ||
587 | char * | |
588 | UFSSwapDir::logFile(char const *ext) const | |
589 | { | |
ef364f64 AJ |
590 | LOCAL_ARRAY(char, lpath, MAXPATHLEN); |
591 | LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); | |
d3b3ab85 | 592 | LOCAL_ARRAY(char, digit, 32); |
593 | char *pathtmp2; | |
62e76326 | 594 | |
d3b3ab85 | 595 | if (Config.Log.swap) { |
ef364f64 | 596 | xstrncpy(pathtmp, path, MAXPATHLEN - 64); |
62e76326 | 597 | pathtmp2 = pathtmp; |
598 | ||
599 | while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) | |
600 | *pathtmp2 = '.'; | |
601 | ||
602 | while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') | |
603 | pathtmp[strlen(pathtmp) - 1] = '\0'; | |
604 | ||
cb4185f1 | 605 | for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2); |
ef364f64 | 606 | snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); |
62e76326 | 607 | |
ef364f64 | 608 | if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) { |
62e76326 | 609 | strcat(lpath, "."); |
610 | snprintf(digit, 32, "%02d", index); | |
611 | strncat(lpath, digit, 3); | |
612 | } | |
d3b3ab85 | 613 | } else { |
ef364f64 | 614 | xstrncpy(lpath, path, MAXPATHLEN - 64); |
62e76326 | 615 | strcat(lpath, "/swap.state"); |
d3b3ab85 | 616 | } |
62e76326 | 617 | |
d3b3ab85 | 618 | if (ext) |
62e76326 | 619 | strncat(lpath, ext, 16); |
620 | ||
d3b3ab85 | 621 | return lpath; |
622 | } | |
623 | ||
624 | void | |
625 | UFSSwapDir::openLog() | |
626 | { | |
627 | char *logPath; | |
628 | logPath = logFile(); | |
629 | swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); | |
62e76326 | 630 | |
d3b3ab85 | 631 | if (swaplog_fd < 0) { |
bf8fe701 | 632 | debugs(50, 1, "" << logPath << ": " << xstrerror()); |
62e76326 | 633 | fatal("commonUfsDirOpenSwapLog: Failed to open swap log."); |
d3b3ab85 | 634 | } |
62e76326 | 635 | |
bf8fe701 | 636 | debugs(50, 3, "Cache Dir #" << index << " log opened on FD " << swaplog_fd); |
62e76326 | 637 | |
d3b3ab85 | 638 | if (0 == NumberOfUFSDirs) |
62e76326 | 639 | assert(NULL == UFSDirToGlobalDirMapping); |
640 | ||
d3b3ab85 | 641 | ++NumberOfUFSDirs; |
62e76326 | 642 | |
d3b3ab85 | 643 | assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured); |
644 | } | |
645 | ||
646 | void | |
647 | UFSSwapDir::closeLog() | |
648 | { | |
649 | if (swaplog_fd < 0) /* not open */ | |
62e76326 | 650 | return; |
651 | ||
d3b3ab85 | 652 | file_close(swaplog_fd); |
62e76326 | 653 | |
bf8fe701 | 654 | debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd); |
62e76326 | 655 | |
d3b3ab85 | 656 | swaplog_fd = -1; |
62e76326 | 657 | |
59b2d47f | 658 | --NumberOfUFSDirs; |
62e76326 | 659 | |
d3b3ab85 | 660 | assert(NumberOfUFSDirs >= 0); |
62e76326 | 661 | |
eb5ac42b | 662 | if (0 == NumberOfUFSDirs) |
62e76326 | 663 | safe_free(UFSDirToGlobalDirMapping); |
d3b3ab85 | 664 | } |
665 | ||
666 | bool | |
667 | UFSSwapDir::validL1(int anInt) const | |
668 | { | |
669 | return anInt < l1; | |
670 | } | |
671 | ||
672 | bool | |
673 | UFSSwapDir::validL2(int anInt) const | |
674 | { | |
675 | return anInt < l2; | |
676 | } | |
677 | ||
678 | /* Add a new object to the cache with empty memory copy and pointer to disk | |
679 | * use to rebuild store from disk. */ | |
680 | StoreEntry * | |
681 | UFSSwapDir::addDiskRestore(const cache_key * key, | |
62e76326 | 682 | sfileno file_number, |
47f6e231 | 683 | uint64_t swap_file_sz, |
62e76326 | 684 | time_t expires, |
685 | time_t timestamp, | |
686 | time_t lastref, | |
687 | time_t lastmod, | |
09aabd84 FC |
688 | uint32_t refcount, |
689 | uint16_t newFlags, | |
62e76326 | 690 | int clean) |
d3b3ab85 | 691 | { |
692 | StoreEntry *e = NULL; | |
bf8fe701 | 693 | debugs(47, 5, "commonUfsAddDiskRestore: " << storeKeyText(key) << |
694 | ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); | |
62e76326 | 695 | /* if you call this you'd better be sure file_number is not |
d3b3ab85 | 696 | * already in use! */ |
c8f4eac4 | 697 | e = new StoreEntry(); |
d3b3ab85 | 698 | e->store_status = STORE_OK; |
3900307b | 699 | e->setMemStatus(NOT_IN_MEMORY); |
d3b3ab85 | 700 | e->swap_status = SWAPOUT_DONE; |
701 | e->swap_filen = file_number; | |
702 | e->swap_dirn = index; | |
703 | e->swap_file_sz = swap_file_sz; | |
704 | e->lock_count = 0; | |
705 | e->lastref = lastref; | |
706 | e->timestamp = timestamp; | |
707 | e->expires = expires; | |
708 | e->lastmod = lastmod; | |
709 | e->refcount = refcount; | |
e4ae841b | 710 | e->flags = newFlags; |
d3b3ab85 | 711 | EBIT_SET(e->flags, ENTRY_CACHABLE); |
712 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
713 | EBIT_CLR(e->flags, KEY_PRIVATE); | |
714 | e->ping_status = PING_NONE; | |
715 | EBIT_CLR(e->flags, ENTRY_VALIDATED); | |
716 | mapBitSet(e->swap_filen); | |
a86ff82a DK |
717 | cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); |
718 | ++n_disk_objects; | |
3900307b | 719 | e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ |
d3b3ab85 | 720 | replacementAdd (e); |
721 | return e; | |
722 | } | |
723 | ||
c6e51aff AR |
724 | void |
725 | UFSSwapDir::undoAddDiskRestore(StoreEntry *e) | |
726 | { | |
727 | debugs(47, 5, HERE << *e); | |
728 | replacementRemove(e); // checks swap_dirn so do it before we invalidate it | |
729 | // Do not unlink the file as it might be used by a subsequent entry. | |
730 | mapBitReset(e->swap_filen); | |
731 | e->swap_filen = -1; | |
732 | e->swap_dirn = -1; | |
733 | cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz); | |
734 | --n_disk_objects; | |
735 | } | |
736 | ||
d3b3ab85 | 737 | void |
738 | UFSSwapDir::rebuild() | |
739 | { | |
bef81ea5 | 740 | ++StoreController::store_dirs_rebuilding; |
c8f4eac4 | 741 | eventAdd("storeRebuild", RebuildState::RebuildStep, new RebuildState(this), 0.0, 1); |
d3b3ab85 | 742 | } |
743 | ||
744 | void | |
745 | UFSSwapDir::closeTmpSwapLog() | |
746 | { | |
747 | char *swaplog_path = xstrdup(logFile(NULL)); | |
748 | char *new_path = xstrdup(logFile(".new")); | |
749 | int fd; | |
750 | file_close(swaplog_fd); | |
62e76326 | 751 | |
d3b3ab85 | 752 | if (xrename(new_path, swaplog_path) < 0) { |
2066f5a1 | 753 | debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); |
c031952f | 754 | fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); |
d3b3ab85 | 755 | } |
62e76326 | 756 | |
d3b3ab85 | 757 | fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); |
62e76326 | 758 | |
d3b3ab85 | 759 | if (fd < 0) { |
2066f5a1 | 760 | debugs(50, DBG_IMPORTANT, "ERROR: " << swaplog_path << ": " << xstrerror()); |
c031952f | 761 | fatalf("Failed to open swap log %s", swaplog_path); |
d3b3ab85 | 762 | } |
62e76326 | 763 | |
d3b3ab85 | 764 | safe_free(swaplog_path); |
765 | safe_free(new_path); | |
766 | swaplog_fd = fd; | |
bf8fe701 | 767 | debugs(47, 3, "Cache Dir #" << index << " log opened on FD " << fd); |
d3b3ab85 | 768 | } |
769 | ||
770 | FILE * | |
771 | UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) | |
772 | { | |
773 | char *swaplog_path = xstrdup(logFile(NULL)); | |
774 | char *clean_path = xstrdup(logFile(".last-clean")); | |
775 | char *new_path = xstrdup(logFile(".new")); | |
62e76326 | 776 | |
d3b3ab85 | 777 | struct stat log_sb; |
62e76326 | 778 | |
d3b3ab85 | 779 | struct stat clean_sb; |
780 | FILE *fp; | |
781 | int fd; | |
62e76326 | 782 | |
c8f4eac4 | 783 | if (::stat(swaplog_path, &log_sb) < 0) { |
bf8fe701 | 784 | debugs(47, 1, "Cache Dir #" << index << ": No log file"); |
62e76326 | 785 | safe_free(swaplog_path); |
786 | safe_free(clean_path); | |
787 | safe_free(new_path); | |
788 | return NULL; | |
d3b3ab85 | 789 | } |
62e76326 | 790 | |
d3b3ab85 | 791 | *zero_flag = log_sb.st_size == 0 ? 1 : 0; |
792 | /* close the existing write-only FD */ | |
62e76326 | 793 | |
d3b3ab85 | 794 | if (swaplog_fd >= 0) |
62e76326 | 795 | file_close(swaplog_fd); |
796 | ||
d3b3ab85 | 797 | /* open a write-only FD for the new log */ |
798 | fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); | |
26ac0430 | 799 | |
d3b3ab85 | 800 | if (fd < 0) { |
bf8fe701 | 801 | debugs(50, 1, "" << new_path << ": " << xstrerror()); |
62e76326 | 802 | fatal("storeDirOpenTmpSwapLog: Failed to open swap log."); |
d3b3ab85 | 803 | } |
26ac0430 | 804 | |
d3b3ab85 | 805 | swaplog_fd = fd; |
47f6e231 | 806 | |
95ec9c9d | 807 | { |
155d0ed6 AR |
808 | const StoreSwapLogHeader header; |
809 | MemBuf buf; | |
810 | buf.init(header.record_size, header.record_size); | |
811 | buf.append(reinterpret_cast<const char*>(&header), sizeof(header)); | |
812 | // Pad to keep in sync with UFSSwapDir::writeCleanStart(). | |
786f6516 AR |
813 | memset(buf.space(), 0, header.gapSize()); |
814 | buf.appended(header.gapSize()); | |
155d0ed6 AR |
815 | file_write(swaplog_fd, -1, buf.content(), buf.contentSize(), |
816 | NULL, NULL, buf.freeFunc()); | |
95ec9c9d | 817 | } |
47f6e231 | 818 | |
d3b3ab85 | 819 | /* open a read-only stream of the old log */ |
820 | fp = fopen(swaplog_path, "rb"); | |
62e76326 | 821 | |
d3b3ab85 | 822 | if (fp == NULL) { |
bf8fe701 | 823 | debugs(50, 0, "" << swaplog_path << ": " << xstrerror()); |
62e76326 | 824 | fatal("Failed to open swap log for reading"); |
d3b3ab85 | 825 | } |
62e76326 | 826 | |
d3b3ab85 | 827 | memset(&clean_sb, '\0', sizeof(struct stat)); |
62e76326 | 828 | |
c8f4eac4 | 829 | if (::stat(clean_path, &clean_sb) < 0) |
62e76326 | 830 | *clean_flag = 0; |
d3b3ab85 | 831 | else if (clean_sb.st_mtime < log_sb.st_mtime) |
62e76326 | 832 | *clean_flag = 0; |
d3b3ab85 | 833 | else |
62e76326 | 834 | *clean_flag = 1; |
835 | ||
d3b3ab85 | 836 | safeunlink(clean_path, 1); |
62e76326 | 837 | |
d3b3ab85 | 838 | safe_free(swaplog_path); |
62e76326 | 839 | |
d3b3ab85 | 840 | safe_free(clean_path); |
62e76326 | 841 | |
d3b3ab85 | 842 | safe_free(new_path); |
62e76326 | 843 | |
d3b3ab85 | 844 | return fp; |
845 | } | |
846 | ||
62e76326 | 847 | class UFSCleanLog : public SwapDir::CleanLog |
848 | { | |
849 | ||
850 | public: | |
d3b3ab85 | 851 | UFSCleanLog(SwapDir *); |
852 | virtual const StoreEntry *nextEntry(); | |
853 | virtual void write(StoreEntry const &); | |
854 | char *cur; | |
855 | char *newLog; | |
856 | char *cln; | |
857 | char *outbuf; | |
858 | off_t outbuf_offset; | |
859 | int fd; | |
860 | RemovalPolicyWalker *walker; | |
861 | SwapDir *sd; | |
862 | }; | |
863 | ||
864 | #define CLEAN_BUF_SZ 16384 | |
865 | ||
d3b3ab85 | 866 | |
867 | UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : cur(NULL),newLog(NULL),cln(NULL),outbuf(NULL), | |
62e76326 | 868 | outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) |
869 | {} | |
d3b3ab85 | 870 | |
59b2d47f | 871 | /* |
872 | * Begin the process to write clean cache state. For AUFS this means | |
873 | * opening some log files and allocating write buffers. Return 0 if | |
874 | * we succeed, and assign the 'func' and 'data' return pointers. | |
875 | */ | |
d3b3ab85 | 876 | int |
877 | UFSSwapDir::writeCleanStart() | |
878 | { | |
879 | UFSCleanLog *state = new UFSCleanLog(this); | |
47f6e231 | 880 | StoreSwapLogHeader header; |
d3b3ab85 | 881 | #if HAVE_FCHMOD |
62e76326 | 882 | |
d3b3ab85 | 883 | struct stat sb; |
884 | #endif | |
62e76326 | 885 | |
d3b3ab85 | 886 | cleanLog = NULL; |
887 | state->newLog = xstrdup(logFile(".clean")); | |
888 | state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); | |
62e76326 | 889 | |
d3b3ab85 | 890 | if (state->fd < 0) { |
62e76326 | 891 | xfree(state->newLog); |
892 | delete state; | |
893 | return -1; | |
d3b3ab85 | 894 | } |
62e76326 | 895 | |
d3b3ab85 | 896 | state->cur = xstrdup(logFile(NULL)); |
897 | state->cln = xstrdup(logFile(".last-clean")); | |
898 | state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); | |
899 | state->outbuf_offset = 0; | |
47f6e231 | 900 | /*copy the header */ |
41d00cd3 | 901 | memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); |
155d0ed6 | 902 | // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog(). |
786f6516 | 903 | memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize()); |
47f6e231 | 904 | state->outbuf_offset += header.record_size; |
905 | ||
d3b3ab85 | 906 | state->walker = repl->WalkInit(repl); |
907 | ::unlink(state->cln); | |
bf8fe701 | 908 | debugs(47, 3, "storeDirWriteCleanLogs: opened " << state->newLog << ", FD " << state->fd); |
d3b3ab85 | 909 | #if HAVE_FCHMOD |
62e76326 | 910 | |
c8f4eac4 | 911 | if (::stat(state->cur, &sb) == 0) |
62e76326 | 912 | fchmod(state->fd, sb.st_mode); |
913 | ||
d3b3ab85 | 914 | #endif |
26ac0430 | 915 | |
62e76326 | 916 | |
d3b3ab85 | 917 | cleanLog = state; |
918 | return 0; | |
919 | } | |
920 | ||
921 | /* | |
922 | * Get the next entry that is a candidate for clean log writing | |
923 | */ | |
924 | const StoreEntry * | |
925 | UFSCleanLog::nextEntry() | |
926 | { | |
927 | const StoreEntry *entry = NULL; | |
62e76326 | 928 | |
d3b3ab85 | 929 | if (walker) |
62e76326 | 930 | entry = walker->Next(walker); |
931 | ||
d3b3ab85 | 932 | return entry; |
933 | } | |
934 | ||
935 | /* | |
936 | * "write" an entry to the clean log file. | |
937 | */ | |
938 | void | |
939 | UFSCleanLog::write(StoreEntry const &e) | |
940 | { | |
51ee7c82 | 941 | StoreSwapLogData s; |
942 | static size_t ss = sizeof(StoreSwapLogData); | |
d3b3ab85 | 943 | s.op = (char) SWAP_LOG_ADD; |
944 | s.swap_filen = e.swap_filen; | |
945 | s.timestamp = e.timestamp; | |
946 | s.lastref = e.lastref; | |
947 | s.expires = e.expires; | |
948 | s.lastmod = e.lastmod; | |
949 | s.swap_file_sz = e.swap_file_sz; | |
950 | s.refcount = e.refcount; | |
951 | s.flags = e.flags; | |
41d00cd3 | 952 | memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); |
786f6516 | 953 | s.finalize(); |
41d00cd3 | 954 | memcpy(outbuf + outbuf_offset, &s, ss); |
78fe982f | 955 | outbuf_offset += ss; |
d3b3ab85 | 956 | /* buffered write */ |
62e76326 | 957 | |
e79ee799 | 958 | if (outbuf_offset + ss >= CLEAN_BUF_SZ) { |
78fe982f | 959 | if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { |
960 | /* XXX This error handling should probably move up to the caller */ | |
bf8fe701 | 961 | debugs(50, 0, "storeDirWriteCleanLogs: " << newLog << ": write: " << xstrerror()); |
962 | debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); | |
78fe982f | 963 | file_close(fd); |
964 | fd = -1; | |
965 | unlink(newLog); | |
62e76326 | 966 | sd->cleanLog = NULL; |
78fe982f | 967 | delete this; |
62e76326 | 968 | return; |
969 | } | |
970 | ||
78fe982f | 971 | outbuf_offset = 0; |
d3b3ab85 | 972 | } |
973 | } | |
974 | ||
975 | void | |
976 | UFSSwapDir::writeCleanDone() | |
977 | { | |
978 | UFSCleanLog *state = (UFSCleanLog *)cleanLog; | |
979 | int fd; | |
62e76326 | 980 | |
d3b3ab85 | 981 | if (NULL == state) |
62e76326 | 982 | return; |
983 | ||
d3b3ab85 | 984 | if (state->fd < 0) |
62e76326 | 985 | return; |
986 | ||
d3b3ab85 | 987 | state->walker->Done(state->walker); |
62e76326 | 988 | |
d3b3ab85 | 989 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
bf8fe701 | 990 | debugs(50, 0, "storeDirWriteCleanLogs: " << state->newLog << ": write: " << xstrerror()); |
991 | debugs(50, 0, "storeDirWriteCleanLogs: Current swap logfile not replaced."); | |
62e76326 | 992 | file_close(state->fd); |
993 | state->fd = -1; | |
994 | ::unlink(state->newLog); | |
d3b3ab85 | 995 | } |
62e76326 | 996 | |
d3b3ab85 | 997 | safe_free(state->outbuf); |
998 | /* | |
999 | * You can't rename open files on Microsoft "operating systems" | |
1000 | * so we have to close before renaming. | |
1001 | */ | |
1002 | closeLog(); | |
1003 | /* save the fd value for a later test */ | |
1004 | fd = state->fd; | |
1005 | /* rename */ | |
62e76326 | 1006 | |
d3b3ab85 | 1007 | if (state->fd >= 0) { |
be266cb2 | 1008 | #if _SQUID_OS2_ || _SQUID_WINDOWS_ |
62e76326 | 1009 | file_close(state->fd); |
1010 | state->fd = -1; | |
d3b3ab85 | 1011 | #endif |
62e76326 | 1012 | |
1013 | xrename(state->newLog, state->cur); | |
d3b3ab85 | 1014 | } |
62e76326 | 1015 | |
d3b3ab85 | 1016 | /* touch a timestamp file if we're not still validating */ |
bef81ea5 | 1017 | if (StoreController::store_dirs_rebuilding) |
62e76326 | 1018 | (void) 0; |
d3b3ab85 | 1019 | else if (fd < 0) |
62e76326 | 1020 | (void) 0; |
d3b3ab85 | 1021 | else |
62e76326 | 1022 | file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); |
1023 | ||
d3b3ab85 | 1024 | /* close */ |
1025 | safe_free(state->cur); | |
62e76326 | 1026 | |
d3b3ab85 | 1027 | safe_free(state->newLog); |
62e76326 | 1028 | |
d3b3ab85 | 1029 | safe_free(state->cln); |
62e76326 | 1030 | |
d3b3ab85 | 1031 | if (state->fd >= 0) |
62e76326 | 1032 | file_close(state->fd); |
1033 | ||
d3b3ab85 | 1034 | state->fd = -1; |
62e76326 | 1035 | |
d3b3ab85 | 1036 | delete state; |
62e76326 | 1037 | |
d3b3ab85 | 1038 | cleanLog = NULL; |
1039 | } | |
1040 | ||
061c7171 | 1041 | static void |
1042 | FreeObject(void *address) | |
1043 | { | |
1044 | StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address); | |
1045 | delete anObject; | |
1046 | } | |
1047 | ||
d3b3ab85 | 1048 | void |
1049 | UFSSwapDir::logEntry(const StoreEntry & e, int op) const | |
1050 | { | |
51ee7c82 | 1051 | StoreSwapLogData *s = new StoreSwapLogData; |
d3b3ab85 | 1052 | s->op = (char) op; |
1053 | s->swap_filen = e.swap_filen; | |
1054 | s->timestamp = e.timestamp; | |
1055 | s->lastref = e.lastref; | |
1056 | s->expires = e.expires; | |
1057 | s->lastmod = e.lastmod; | |
1058 | s->swap_file_sz = e.swap_file_sz; | |
1059 | s->refcount = e.refcount; | |
1060 | s->flags = e.flags; | |
41d00cd3 | 1061 | memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); |
786f6516 | 1062 | s->finalize(); |
d3b3ab85 | 1063 | file_write(swaplog_fd, |
62e76326 | 1064 | -1, |
1065 | s, | |
51ee7c82 | 1066 | sizeof(StoreSwapLogData), |
62e76326 | 1067 | NULL, |
1068 | NULL, | |
061c7171 | 1069 | FreeObject); |
d3b3ab85 | 1070 | } |
1071 | ||
1072 | static QS rev_int_sort; | |
1073 | static int | |
1074 | rev_int_sort(const void *A, const void *B) | |
1075 | { | |
1076 | const int *i1 = (const int *)A; | |
1077 | const int *i2 = (const int *)B; | |
1078 | return *i2 - *i1; | |
1079 | } | |
1080 | ||
1081 | int | |
1082 | UFSSwapDir::DirClean(int swap_index) | |
1083 | { | |
bef81ea5 | 1084 | DIR *dir_pointer = NULL; |
62e76326 | 1085 | |
d3b3ab85 | 1086 | LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); |
1087 | LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); | |
62e76326 | 1088 | |
d3b3ab85 | 1089 | int files[20]; |
1090 | int swapfileno; | |
1091 | int fn; /* same as swapfileno, but with dirn bits set */ | |
1092 | int n = 0; | |
1093 | int k = 0; | |
1094 | int N0, N1, N2; | |
1095 | int D0, D1, D2; | |
1096 | UFSSwapDir *SD; | |
1097 | N0 = NumberOfUFSDirs; | |
1098 | D0 = UFSDirToGlobalDirMapping[swap_index % N0]; | |
1099 | SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0)); | |
1100 | assert (SD); | |
1101 | N1 = SD->l1; | |
1102 | D1 = (swap_index / N0) % N1; | |
1103 | N2 = SD->l2; | |
1104 | D2 = ((swap_index / N0) / N1) % N2; | |
ef364f64 | 1105 | snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", |
62e76326 | 1106 | SD->path, D1, D2); |
bf8fe701 | 1107 | debugs(36, 3, "storeDirClean: Cleaning directory " << p1); |
bef81ea5 | 1108 | dir_pointer = opendir(p1); |
62e76326 | 1109 | |
bef81ea5 | 1110 | if (dir_pointer == NULL) { |
62e76326 | 1111 | if (errno == ENOENT) { |
bf8fe701 | 1112 | debugs(36, 0, "storeDirClean: WARNING: Creating " << p1); |
62e76326 | 1113 | if (mkdir(p1, 0777) == 0) |
62e76326 | 1114 | return 0; |
1115 | } | |
1116 | ||
bf8fe701 | 1117 | debugs(50, 0, "storeDirClean: " << p1 << ": " << xstrerror()); |
62e76326 | 1118 | safeunlink(p1, 1); |
1119 | return 0; | |
d3b3ab85 | 1120 | } |
62e76326 | 1121 | |
a8d98451 | 1122 | dirent_t *de; |
bef81ea5 | 1123 | while ((de = readdir(dir_pointer)) != NULL && k < 20) { |
62e76326 | 1124 | if (sscanf(de->d_name, "%X", &swapfileno) != 1) |
1125 | continue; | |
1126 | ||
1127 | fn = swapfileno; /* XXX should remove this cruft ! */ | |
1128 | ||
1129 | if (SD->validFileno(fn, 1)) | |
1130 | if (SD->mapBitTest(fn)) | |
1131 | if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) | |
1132 | continue; | |
1133 | ||
a38ec4b1 FC |
1134 | files[k] = swapfileno; |
1135 | ++k; | |
d3b3ab85 | 1136 | } |
62e76326 | 1137 | |
bef81ea5 | 1138 | closedir(dir_pointer); |
62e76326 | 1139 | |
d3b3ab85 | 1140 | if (k == 0) |
62e76326 | 1141 | return 0; |
1142 | ||
d3b3ab85 | 1143 | qsort(files, k, sizeof(int), rev_int_sort); |
62e76326 | 1144 | |
d3b3ab85 | 1145 | if (k > 10) |
62e76326 | 1146 | k = 10; |
1147 | ||
cb4185f1 | 1148 | for (n = 0; n < k; ++n) { |
bf8fe701 | 1149 | debugs(36, 3, "storeDirClean: Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); |
62e76326 | 1150 | snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); |
62e76326 | 1151 | safeunlink(p2, 0); |
e4f1fdae | 1152 | ++statCounter.swap.files_cleaned; |
d3b3ab85 | 1153 | } |
62e76326 | 1154 | |
bf8fe701 | 1155 | debugs(36, 3, "Cleaned " << k << " unused files from " << p1); |
d3b3ab85 | 1156 | return k; |
1157 | } | |
1158 | ||
1159 | void | |
1160 | UFSSwapDir::CleanEvent(void *unused) | |
1161 | { | |
1162 | static int swap_index = 0; | |
1163 | int i; | |
1164 | int j = 0; | |
1165 | int n = 0; | |
1166 | /* | |
1167 | * Assert that there are UFS cache_dirs configured, otherwise | |
1168 | * we should never be called. | |
1169 | */ | |
1170 | assert(NumberOfUFSDirs); | |
62e76326 | 1171 | |
d3b3ab85 | 1172 | if (NULL == UFSDirToGlobalDirMapping) { |
62e76326 | 1173 | SwapDir *sd; |
1174 | /* | |
1175 | * Initialize the little array that translates UFS cache_dir | |
1176 | * number into the Config.cacheSwap.swapDirs array index. | |
1177 | */ | |
1178 | UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); | |
1179 | ||
cb4185f1 | 1180 | for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) { |
c8f4eac4 | 1181 | /* This is bogus, the controller should just clean each instance once */ |
1182 | sd = dynamic_cast <SwapDir *>(INDEXSD(i)); | |
62e76326 | 1183 | |
1184 | if (!UFSSwapDir::IsUFSDir(sd)) | |
1185 | continue; | |
1186 | ||
1187 | UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd); | |
1188 | ||
1189 | assert (usd); | |
1190 | ||
a38ec4b1 FC |
1191 | UFSDirToGlobalDirMapping[n] = i; |
1192 | ++n; | |
62e76326 | 1193 | |
1194 | j += (usd->l1 * usd->l2); | |
1195 | } | |
1196 | ||
1197 | assert(n == NumberOfUFSDirs); | |
1198 | /* | |
1199 | * Start the commonUfsDirClean() swap_index with a random | |
1200 | * value. j equals the total number of UFS level 2 | |
1201 | * swap directories | |
1202 | */ | |
1203 | swap_index = (int) (squid_random() % j); | |
d3b3ab85 | 1204 | } |
62e76326 | 1205 | |
bef81ea5 | 1206 | /* if the rebuild is finished, start cleaning directories. */ |
1207 | if (0 == StoreController::store_dirs_rebuilding) { | |
62e76326 | 1208 | n = DirClean(swap_index); |
cb4185f1 | 1209 | ++swap_index; |
d3b3ab85 | 1210 | } |
62e76326 | 1211 | |
d3b3ab85 | 1212 | eventAdd("storeDirClean", CleanEvent, NULL, |
62e76326 | 1213 | 15.0 * exp(-0.25 * n), 1); |
d3b3ab85 | 1214 | } |
1215 | ||
1216 | int | |
1217 | UFSSwapDir::IsUFSDir(SwapDir * sd) | |
1218 | { | |
1219 | UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd); | |
1220 | return mySD ? 1 : 0 ; | |
1221 | } | |
1222 | ||
1223 | /* | |
1224 | * Does swapfile number 'fn' belong in cachedir #F0, | |
1225 | * level1 dir #F1, level2 dir #F2? | |
1226 | * XXX: this is broken - it assumes all cache dirs use the same | |
26ac0430 | 1227 | * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - |
d3b3ab85 | 1228 | * if not UFSSwapDir return 0; |
1229 | */ | |
1230 | int | |
1231 | UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) | |
1232 | { | |
1233 | int D1, D2; | |
1234 | int L1, L2; | |
1235 | int filn = fn; | |
1236 | assert(F0 < Config.cacheSwap.n_configured); | |
c8f4eac4 | 1237 | assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0)))); |
d3b3ab85 | 1238 | UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0)); |
62e76326 | 1239 | |
d3b3ab85 | 1240 | if (!sd) |
62e76326 | 1241 | return 0; |
1242 | ||
d3b3ab85 | 1243 | L1 = sd->l1; |
62e76326 | 1244 | |
d3b3ab85 | 1245 | L2 = sd->l2; |
62e76326 | 1246 | |
d3b3ab85 | 1247 | D1 = ((filn / L2) / L2) % L1; |
62e76326 | 1248 | |
d3b3ab85 | 1249 | if (F1 != D1) |
62e76326 | 1250 | return 0; |
1251 | ||
d3b3ab85 | 1252 | D2 = (filn / L2) % L2; |
62e76326 | 1253 | |
d3b3ab85 | 1254 | if (F2 != D2) |
62e76326 | 1255 | return 0; |
1256 | ||
d3b3ab85 | 1257 | return 1; |
1258 | } | |
1259 | ||
bef81ea5 | 1260 | |
d3b3ab85 | 1261 | int |
1262 | UFSSwapDir::validFileno(sfileno filn, int flag) const | |
1263 | { | |
1264 | if (filn < 0) | |
62e76326 | 1265 | return 0; |
1266 | ||
d3b3ab85 | 1267 | /* |
1268 | * If flag is set it means out-of-range file number should | |
1269 | * be considered invalid. | |
1270 | */ | |
1271 | if (flag) | |
75f8f9a2 | 1272 | if (filn > map->capacity()) |
62e76326 | 1273 | return 0; |
1274 | ||
d3b3ab85 | 1275 | return 1; |
1276 | } | |
1277 | ||
59b2d47f | 1278 | |
1279 | ||
d3b3ab85 | 1280 | /* |
1281 | * UFSSwapDir::unlinkFile | |
1282 | * | |
1283 | * This routine unlinks a file and pulls it out of the bitmap. | |
1284 | * It used to be in commonUfsUnlink(), however an interface change | |
1285 | * forced this bit of code here. Eeek. | |
1286 | */ | |
1287 | void | |
1288 | UFSSwapDir::unlinkFile(sfileno f) | |
1289 | { | |
26ac0430 AJ |
1290 | debugs(79, 3, "UFSSwapDir::unlinkFile: unlinking fileno " << std::setfill('0') << |
1291 | std::hex << std::uppercase << std::setw(8) << f << " '" << | |
1292 | fullPath(f,NULL) << "'"); | |
d3b3ab85 | 1293 | /* commonUfsDirMapBitReset(this, f); */ |
59b2d47f | 1294 | IO->unlinkFile(fullPath(f,NULL)); |
1295 | } | |
1296 | ||
c521ad17 DK |
1297 | bool |
1298 | UFSSwapDir::unlinkdUseful() const | |
1299 | { | |
1300 | // unlinkd may be useful only in workers | |
1301 | return IamWorkerProcess() && IO->io->unlinkdUseful(); | |
1302 | } | |
1303 | ||
59b2d47f | 1304 | void |
1305 | UFSSwapDir::unlink(StoreEntry & e) | |
1306 | { | |
bf8fe701 | 1307 | debugs(79, 3, "storeUfsUnlink: dirno " << index << ", fileno "<< |
1308 | std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); | |
c6e51aff | 1309 | if (e.swap_status == SWAPOUT_DONE) { |
a86ff82a DK |
1310 | cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); |
1311 | --n_disk_objects; | |
1312 | } | |
59b2d47f | 1313 | replacementRemove(&e); |
1314 | mapBitReset(e.swap_filen); | |
1315 | UFSSwapDir::unlinkFile(e.swap_filen); | |
d3b3ab85 | 1316 | } |
1317 | ||
1318 | /* | |
1319 | * Add and remove the given StoreEntry from the replacement policy in | |
1320 | * use. | |
1321 | */ | |
1322 | ||
1323 | void | |
1324 | UFSSwapDir::replacementAdd(StoreEntry * e) | |
1325 | { | |
bf8fe701 | 1326 | debugs(47, 4, "UFSSwapDir::replacementAdd: added node " << e << " to dir " << index); |
d3b3ab85 | 1327 | repl->Add(repl, e, &e->repl); |
1328 | } | |
1329 | ||
1330 | ||
1331 | void | |
1332 | UFSSwapDir::replacementRemove(StoreEntry * e) | |
1333 | { | |
c8f4eac4 | 1334 | StorePointer SD; |
62e76326 | 1335 | |
d3b3ab85 | 1336 | if (e->swap_dirn < 0) |
62e76326 | 1337 | return; |
1338 | ||
d3b3ab85 | 1339 | SD = INDEXSD(e->swap_dirn); |
62e76326 | 1340 | |
c8f4eac4 | 1341 | assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this); |
62e76326 | 1342 | |
bf8fe701 | 1343 | debugs(47, 4, "UFSSwapDir::replacementRemove: remove node " << e << " from dir " << index); |
62e76326 | 1344 | |
d3b3ab85 | 1345 | repl->Remove(repl, e, &e->repl); |
1346 | } | |
1347 | ||
1348 | void | |
1349 | UFSSwapDir::dump(StoreEntry & entry) const | |
1350 | { | |
c91ca3ce | 1351 | storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2); |
59b2d47f | 1352 | dumpOptions(&entry); |
d3b3ab85 | 1353 | } |
1354 | ||
1355 | char * | |
1356 | UFSSwapDir::fullPath(sfileno filn, char *fullpath) const | |
1357 | { | |
ef364f64 | 1358 | LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); |
d3b3ab85 | 1359 | int L1 = l1; |
1360 | int L2 = l2; | |
62e76326 | 1361 | |
d3b3ab85 | 1362 | if (!fullpath) |
62e76326 | 1363 | fullpath = fullfilename; |
1364 | ||
d3b3ab85 | 1365 | fullpath[0] = '\0'; |
62e76326 | 1366 | |
ef364f64 | 1367 | snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X", |
62e76326 | 1368 | path, |
1369 | ((filn / L2) / L2) % L1, | |
1370 | (filn / L2) % L2, | |
1371 | filn); | |
1372 | ||
d3b3ab85 | 1373 | return fullpath; |
1374 | } | |
59b2d47f | 1375 | |
1376 | int | |
1377 | UFSSwapDir::callback() | |
1378 | { | |
1379 | return IO->callback(); | |
1380 | } | |
1381 | ||
1382 | void | |
1383 | UFSSwapDir::sync() | |
1384 | { | |
1385 | IO->sync(); | |
1386 | } | |
c8f4eac4 | 1387 | |
da9d3191 DK |
1388 | void |
1389 | UFSSwapDir::swappedOut(const StoreEntry &e) | |
1390 | { | |
a86ff82a DK |
1391 | cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); |
1392 | ++n_disk_objects; | |
da9d3191 DK |
1393 | } |
1394 | ||
c8f4eac4 | 1395 | StoreSearch * |
30abd221 | 1396 | UFSSwapDir::search(String const url, HttpRequest *request) |
c8f4eac4 | 1397 | { |
1398 | if (url.size()) | |
1399 | fatal ("Cannot search by url yet\n"); | |
1400 | ||
1401 | return new StoreSearchUFS (this); | |
1402 | } | |
1403 | ||
1404 | CBDATA_CLASS_INIT(StoreSearchUFS); | |
1405 | StoreSearchUFS::StoreSearchUFS(RefCount<UFSSwapDir> aSwapDir) : sd(aSwapDir), walker (sd->repl->WalkInit(sd->repl)), current (NULL), _done (false) | |
1406 | {} | |
1407 | ||
1408 | /* do not link | |
1409 | StoreSearchUFS::StoreSearchUFS(StoreSearchUFS const &); | |
1410 | */ | |
1411 | ||
1412 | StoreSearchUFS::~StoreSearchUFS() | |
1413 | { | |
1414 | walker->Done(walker); | |
1415 | walker = NULL; | |
1416 | } | |
1417 | ||
1418 | void | |
e4ae841b | 1419 | StoreSearchUFS::next(void (aCallback)(void *cbdata), void *aCallbackArgs) |
c8f4eac4 | 1420 | { |
1421 | next(); | |
e4ae841b | 1422 | aCallback(aCallbackArgs); |
c8f4eac4 | 1423 | } |
1424 | ||
1425 | bool | |
1426 | StoreSearchUFS::next() | |
1427 | { | |
1428 | /* the walker API doesn't make sense. the store entries referred to are already readwrite | |
1429 | * from their hash table entries | |
1430 | */ | |
1431 | ||
1432 | if (walker) | |
1433 | current = const_cast<StoreEntry *>(walker->Next(walker)); | |
1434 | ||
1435 | if (current == NULL) | |
1436 | _done = true; | |
1437 | ||
1438 | return current != NULL; | |
1439 | } | |
1440 | ||
1441 | bool | |
1442 | StoreSearchUFS::error() const | |
1443 | { | |
1444 | return false; | |
1445 | } | |
1446 | ||
1447 | bool | |
1448 | StoreSearchUFS::isDone() const | |
1449 | { | |
1450 | return _done; | |
1451 | } | |
1452 | ||
1453 | StoreEntry * | |
1454 | StoreSearchUFS::currentItem() | |
1455 | { | |
1456 | return current; | |
1457 | } |