]>
Commit | Line | Data |
---|---|---|
cd748f27 | 1 | /* |
cd748f27 | 2 | * DEBUG: section 47 Store Directory Routines |
58373ff8 | 3 | * AUTHOR: Robert Collins |
cd748f27 | 4 | * |
2b6662ba | 5 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
cd748f27 | 6 | * ---------------------------------------------------------- |
7 | * | |
2b6662ba | 8 | * Squid is the result of efforts by numerous individuals from |
9 | * the Internet community; see the CONTRIBUTORS file for full | |
10 | * details. Many organizations have provided support for Squid's | |
11 | * development; see the SPONSORS file for full details. Squid is | |
12 | * Copyrighted (C) 2001 by the Regents of the University of | |
13 | * California; see the COPYRIGHT file for full details. Squid | |
14 | * incorporates software developed and/or copyrighted by other | |
15 | * sources; see the CREDITS file for full details. | |
cd748f27 | 16 | * |
17 | * This program is free software; you can redistribute it and/or modify | |
18 | * it under the terms of the GNU General Public License as published by | |
19 | * the Free Software Foundation; either version 2 of the License, or | |
20 | * (at your option) any later version. | |
26ac0430 | 21 | * |
cd748f27 | 22 | * This program is distributed in the hope that it will be useful, |
23 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
24 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
25 | * GNU General Public License for more details. | |
26ac0430 | 26 | * |
cd748f27 | 27 | * You should have received a copy of the GNU General Public License |
28 | * along with this program; if not, write to the Free Software | |
29 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
cd748f27 | 30 | */ |
31 | ||
f7f3304a | 32 | #include "squid-old.h" |
58373ff8 FC |
33 | |
34 | #define CLEAN_BUF_SZ 16384 | |
35 | ||
b9ae18aa | 36 | #include "ConfigOption.h" |
b9ae18aa | 37 | #include "DiskIO/DiskIOModule.h" |
75f8f9a2 | 38 | #include "FileMap.h" |
58373ff8 | 39 | #include "fde.h" |
c8f4eac4 | 40 | #include "Parsing.h" |
58373ff8 FC |
41 | #include "protos.h" |
42 | #include "RebuildState.h" | |
a98bcbee | 43 | #include "SquidMath.h" |
58373ff8 FC |
44 | #include "DiskIO/DiskIOStrategy.h" |
45 | #include "StoreSearchUFS.h" | |
46 | #include "StoreSwapLogData.h" | |
985c86bc | 47 | #include "SquidTime.h" |
e4f1fdae | 48 | #include "StatCounters.h" |
58373ff8 | 49 | #include "UFSSwapDir.h" |
27e059d4 | 50 | |
cd748f27 | 51 | |
58373ff8 FC |
52 | int Fs::Ufs::UFSSwapDir::NumberOfUFSDirs = 0; |
53 | int *Fs::Ufs::UFSSwapDir::UFSDirToGlobalDirMapping = NULL; | |
54 | ||
55 | class UFSCleanLog : public SwapDir::CleanLog | |
56 | { | |
57 | ||
58 | public: | |
59 | UFSCleanLog(SwapDir *); | |
60 | /** Get the next entry that is a candidate for clean log writing | |
61 | */ | |
62 | virtual const StoreEntry *nextEntry(); | |
63 | /** "write" an entry to the clean log file. | |
64 | */ | |
65 | virtual void write(StoreEntry const &); | |
66 | char *cur; | |
67 | char *newLog; | |
68 | char *cln; | |
69 | char *outbuf; | |
70 | off_t outbuf_offset; | |
71 | int fd; | |
72 | RemovalPolicyWalker *walker; | |
73 | SwapDir *sd; | |
74 | }; | |
75 | ||
76 | UFSCleanLog::UFSCleanLog(SwapDir *aSwapDir) : | |
77 | cur(NULL), newLog(NULL), cln(NULL), outbuf(NULL), | |
78 | outbuf_offset(0), fd(-1),walker(NULL), sd(aSwapDir) | |
79 | {} | |
80 | ||
81 | const StoreEntry * | |
82 | UFSCleanLog::nextEntry() | |
83 | { | |
84 | const StoreEntry *entry = NULL; | |
85 | ||
86 | if (walker) | |
87 | entry = walker->Next(walker); | |
88 | ||
89 | return entry; | |
90 | } | |
91 | ||
92 | void | |
93 | UFSCleanLog::write(StoreEntry const &e) | |
94 | { | |
95 | StoreSwapLogData s; | |
96 | static size_t ss = sizeof(StoreSwapLogData); | |
97 | s.op = (char) SWAP_LOG_ADD; | |
98 | s.swap_filen = e.swap_filen; | |
99 | s.timestamp = e.timestamp; | |
100 | s.lastref = e.lastref; | |
101 | s.expires = e.expires; | |
102 | s.lastmod = e.lastmod; | |
103 | s.swap_file_sz = e.swap_file_sz; | |
104 | s.refcount = e.refcount; | |
105 | s.flags = e.flags; | |
106 | memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH); | |
107 | s.finalize(); | |
108 | memcpy(outbuf + outbuf_offset, &s, ss); | |
109 | outbuf_offset += ss; | |
110 | /* buffered write */ | |
111 | ||
112 | if (outbuf_offset + ss >= CLEAN_BUF_SZ) { | |
113 | if (FD_WRITE_METHOD(fd, outbuf, outbuf_offset) < 0) { | |
114 | /* XXX This error handling should probably move up to the caller */ | |
115 | debugs(50, DBG_CRITICAL, HERE << newLog << ": write: " << xstrerror()); | |
116 | debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); | |
117 | file_close(fd); | |
118 | fd = -1; | |
119 | unlink(newLog); | |
120 | sd->cleanLog = NULL; | |
121 | delete this; | |
122 | return; | |
123 | } | |
124 | ||
125 | outbuf_offset = 0; | |
126 | } | |
127 | } | |
128 | ||
aa1a691e | 129 | bool |
58373ff8 | 130 | Fs::Ufs::UFSSwapDir::canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const |
cd748f27 | 131 | { |
aa1a691e AR |
132 | if (!SwapDir::canStore(e, diskSpaceNeeded, load)) |
133 | return false; | |
134 | ||
59b2d47f | 135 | if (IO->shedLoad()) |
aa1a691e | 136 | return false; |
62e76326 | 137 | |
aa1a691e AR |
138 | load = IO->load(); |
139 | return true; | |
cd748f27 | 140 | } |
141 | ||
58373ff8 FC |
142 | static void |
143 | FreeObject(void *address) | |
144 | { | |
145 | StoreSwapLogData *anObject = static_cast <StoreSwapLogData *>(address); | |
146 | delete anObject; | |
147 | } | |
8e8d4f30 | 148 | |
58373ff8 FC |
149 | static QS rev_int_sort; |
150 | static int | |
151 | rev_int_sort(const void *A, const void *B) | |
152 | { | |
153 | const int *i1 = (const int *)A; | |
154 | const int *i2 = (const int *)B; | |
155 | return *i2 - *i1; | |
156 | } | |
d3b3ab85 | 157 | |
158 | void | |
58373ff8 | 159 | Fs::Ufs::UFSSwapDir::parseSizeL1L2() |
cd748f27 | 160 | { |
6da430a4 AJ |
161 | int i = GetInteger(); |
162 | if (i <= 0) | |
59b2d47f | 163 | fatal("UFSSwapDir::parseSizeL1L2: invalid size value"); |
cd748f27 | 164 | |
a0a5b161 | 165 | const uint64_t size = static_cast<uint64_t>(i) << 20; // MBytes to Bytes |
6da430a4 | 166 | |
cd748f27 | 167 | /* just reconfigure it */ |
0931cb03 | 168 | if (reconfiguring) { |
cc34568d DK |
169 | if (size == maxSize()) |
170 | debugs(3, 2, "Cache dir '" << path << "' size remains unchanged at " << i << " MB"); | |
0931cb03 | 171 | else |
58373ff8 | 172 | debugs(3, DBG_IMPORTANT, "Cache dir '" << path << "' size changed to " << i << " MB"); |
0931cb03 | 173 | } |
62e76326 | 174 | |
d3b3ab85 | 175 | max_size = size; |
59b2d47f | 176 | |
177 | l1 = GetInteger(); | |
178 | ||
179 | if (l1 <= 0) | |
180 | fatal("UFSSwapDir::parseSizeL1L2: invalid level 1 directories value"); | |
181 | ||
182 | l2 = GetInteger(); | |
183 | ||
184 | if (l2 <= 0) | |
185 | fatal("UFSSwapDir::parseSizeL1L2: invalid level 2 directories value"); | |
cd748f27 | 186 | } |
187 | ||
188 | void | |
58373ff8 | 189 | Fs::Ufs::UFSSwapDir::reconfigure() |
cd748f27 | 190 | { |
59b2d47f | 191 | parseSizeL1L2(); |
192 | parseOptions(1); | |
cd748f27 | 193 | } |
194 | ||
d3b3ab85 | 195 | void |
58373ff8 | 196 | Fs::Ufs::UFSSwapDir::parse (int anIndex, char *aPath) |
cd748f27 | 197 | { |
59b2d47f | 198 | index = anIndex; |
59b2d47f | 199 | path = xstrdup(aPath); |
62e76326 | 200 | |
1cd79a58 | 201 | parseSizeL1L2(); |
202 | ||
59b2d47f | 203 | /* Initialise replacement policy stuff */ |
204 | repl = createRemovalPolicy(Config.replPolicy); | |
62e76326 | 205 | |
59b2d47f | 206 | parseOptions(0); |
207 | } | |
62e76326 | 208 | |
b9ae18aa | 209 | void |
58373ff8 | 210 | Fs::Ufs::UFSSwapDir::changeIO(DiskIOModule *module) |
b9ae18aa | 211 | { |
212 | DiskIOStrategy *anIO = module->createStrategy(); | |
213 | safe_free(ioType); | |
214 | ioType = xstrdup(module->type()); | |
215 | ||
216 | delete IO->io; | |
217 | IO->io = anIO; | |
218 | /* Change the IO Options */ | |
219 | ||
91c414dc | 220 | if (currentIOOptions && currentIOOptions->options.size() > 2) |
b9ae18aa | 221 | delete currentIOOptions->options.pop_back(); |
222 | ||
223 | /* TODO: factor out these 4 lines */ | |
224 | ConfigOption *ioOptions = IO->io->getOptionTree(); | |
225 | ||
226 | if (ioOptions) | |
227 | currentIOOptions->options.push_back(ioOptions); | |
228 | } | |
229 | ||
230 | bool | |
58373ff8 | 231 | Fs::Ufs::UFSSwapDir::optionIOParse(char const *option, const char *value, int isaReconfig) |
59b2d47f | 232 | { |
b9ae18aa | 233 | if (strcmp(option, "IOEngine") != 0) |
234 | return false; | |
62e76326 | 235 | |
18ec8500 | 236 | if (isaReconfig) |
b9ae18aa | 237 | /* silently ignore this */ |
238 | return true; | |
62e76326 | 239 | |
b9ae18aa | 240 | if (!value) |
241 | self_destruct(); | |
d3b3ab85 | 242 | |
b9ae18aa | 243 | DiskIOModule *module = DiskIOModule::Find(value); |
62e76326 | 244 | |
b9ae18aa | 245 | if (!module) |
246 | self_destruct(); | |
8e8d4f30 | 247 | |
b9ae18aa | 248 | changeIO(module); |
249 | ||
250 | return true; | |
251 | } | |
252 | ||
253 | void | |
58373ff8 | 254 | Fs::Ufs::UFSSwapDir::optionIODump(StoreEntry * e) const |
b9ae18aa | 255 | { |
256 | storeAppendPrintf(e, " IOEngine=%s", ioType); | |
257 | } | |
258 | ||
259 | ConfigOption * | |
58373ff8 | 260 | Fs::Ufs::UFSSwapDir::getOptionTree() const |
b9ae18aa | 261 | { |
262 | ConfigOption *parentResult = SwapDir::getOptionTree(); | |
263 | ||
e2a8733e | 264 | if (currentIOOptions == NULL) |
265 | currentIOOptions = new ConfigOptionVector(); | |
266 | ||
b9ae18aa | 267 | currentIOOptions->options.push_back(parentResult); |
e2a8733e | 268 | |
b9ae18aa | 269 | currentIOOptions->options.push_back(new ConfigOptionAdapter<UFSSwapDir>(*const_cast<UFSSwapDir *>(this), &UFSSwapDir::optionIOParse, &UFSSwapDir::optionIODump)); |
270 | ||
5f2d056f | 271 | if (ConfigOption *ioOptions = IO->io->getOptionTree()) |
b9ae18aa | 272 | currentIOOptions->options.push_back(ioOptions); |
273 | ||
e2a8733e | 274 | ConfigOption* result = currentIOOptions; |
275 | ||
276 | currentIOOptions = NULL; | |
277 | ||
278 | return result; | |
cd748f27 | 279 | } |
280 | ||
d3b3ab85 | 281 | void |
58373ff8 | 282 | Fs::Ufs::UFSSwapDir::init() |
d3b3ab85 | 283 | { |
58373ff8 | 284 | debugs(47, 3, HERE << "Initialising UFS SwapDir engine."); |
d295d770 | 285 | /* Parsing must be finished by now - force to NULL, don't delete */ |
b9ae18aa | 286 | currentIOOptions = NULL; |
d3b3ab85 | 287 | static int started_clean_event = 0; |
288 | static const char *errmsg = | |
62e76326 | 289 | "\tFailed to verify one of the swap directories, Check cache.log\n" |
290 | "\tfor details. Run 'squid -z' to create swap directories\n" | |
291 | "\tif needed, or if running Squid for the first time."; | |
59b2d47f | 292 | IO->init(); |
62e76326 | 293 | |
d3b3ab85 | 294 | if (verifyCacheDirs()) |
62e76326 | 295 | fatal(errmsg); |
296 | ||
d3b3ab85 | 297 | openLog(); |
62e76326 | 298 | |
d3b3ab85 | 299 | rebuild(); |
62e76326 | 300 | |
d3b3ab85 | 301 | if (!started_clean_event) { |
62e76326 | 302 | eventAdd("UFS storeDirClean", CleanEvent, NULL, 15.0, 1); |
303 | started_clean_event = 1; | |
d3b3ab85 | 304 | } |
62e76326 | 305 | |
d3b3ab85 | 306 | (void) storeDirGetBlkSize(path, &fs.blksize); |
307 | } | |
308 | ||
309 | void | |
58373ff8 | 310 | Fs::Ufs::UFSSwapDir::create() |
d3b3ab85 | 311 | { |
d295d770 | 312 | debugs(47, 3, "Creating swap space in " << path); |
d3b3ab85 | 313 | createDirectory(path, 0); |
314 | createSwapSubDirs(); | |
315 | } | |
316 | ||
58373ff8 | 317 | Fs::Ufs::UFSSwapDir::UFSSwapDir(char const *aType, const char *anIOType) : SwapDir(aType), IO(NULL), map(new FileMap()), suggest(0), swaplog_fd (-1), currentIOOptions(new ConfigOptionVector()), ioType(xstrdup(anIOType)), cur_size(0), n_disk_objects(0) |
b7717b61 | 318 | { |
319 | /* modulename is only set to disk modules that are built, by configure, | |
320 | * so the Find call should never return NULL here. | |
321 | */ | |
58373ff8 | 322 | IO = new Fs::Ufs::UFSStrategy(DiskIOModule::Find(anIOType)->createStrategy()); |
b7717b61 | 323 | } |
d3b3ab85 | 324 | |
58373ff8 | 325 | Fs::Ufs::UFSSwapDir::~UFSSwapDir() |
d3b3ab85 | 326 | { |
327 | if (swaplog_fd > -1) { | |
62e76326 | 328 | file_close(swaplog_fd); |
329 | swaplog_fd = -1; | |
d3b3ab85 | 330 | } |
62e76326 | 331 | |
75f8f9a2 | 332 | delete map; |
62e76326 | 333 | |
d3b3ab85 | 334 | if (IO) |
00d77d6b | 335 | delete IO; |
62e76326 | 336 | |
d3b3ab85 | 337 | IO = NULL; |
b9ae18aa | 338 | |
339 | safe_free(ioType); | |
d3b3ab85 | 340 | } |
341 | ||
d3b3ab85 | 342 | void |
58373ff8 | 343 | Fs::Ufs::UFSSwapDir::dumpEntry(StoreEntry &e) const |
d3b3ab85 | 344 | { |
58373ff8 FC |
345 | debugs(47, DBG_CRITICAL, HERE << "FILENO "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); |
346 | debugs(47, DBG_CRITICAL, HERE << "PATH " << fullPath(e.swap_filen, NULL) ); | |
3900307b | 347 | e.dump(0); |
d3b3ab85 | 348 | } |
349 | ||
d3b3ab85 | 350 | bool |
58373ff8 | 351 | Fs::Ufs::UFSSwapDir::doubleCheck(StoreEntry & e) |
d3b3ab85 | 352 | { |
62e76326 | 353 | |
d3b3ab85 | 354 | struct stat sb; |
62e76326 | 355 | |
c8f4eac4 | 356 | if (::stat(fullPath(e.swap_filen, NULL), &sb) < 0) { |
58373ff8 | 357 | debugs(47, DBG_CRITICAL, HERE << "WARNING: Missing swap file"); |
62e76326 | 358 | dumpEntry(e); |
359 | return true; | |
d3b3ab85 | 360 | } |
62e76326 | 361 | |
d3b3ab85 | 362 | if ((off_t)e.swap_file_sz != sb.st_size) { |
58373ff8 FC |
363 | debugs(47, DBG_CRITICAL, HERE << "WARNING: Size Mismatch. Entry size: " |
364 | << e.swap_file_sz << ", file size: " << sb.st_size); | |
62e76326 | 365 | dumpEntry(e); |
366 | return true; | |
d3b3ab85 | 367 | } |
62e76326 | 368 | |
d3b3ab85 | 369 | return false; |
370 | } | |
371 | ||
372 | void | |
58373ff8 | 373 | Fs::Ufs::UFSSwapDir::statfs(StoreEntry & sentry) const |
d3b3ab85 | 374 | { |
375 | int totl_kb = 0; | |
376 | int free_kb = 0; | |
377 | int totl_in = 0; | |
378 | int free_in = 0; | |
379 | int x; | |
380 | storeAppendPrintf(&sentry, "First level subdirectories: %d\n", l1); | |
381 | storeAppendPrintf(&sentry, "Second level subdirectories: %d\n", l2); | |
c91ca3ce | 382 | storeAppendPrintf(&sentry, "Maximum Size: %" PRIu64 " KB\n", maxSize() >> 10); |
cc34568d | 383 | storeAppendPrintf(&sentry, "Current Size: %.2f KB\n", currentSize() / 1024.0); |
d3b3ab85 | 384 | storeAppendPrintf(&sentry, "Percent Used: %0.2f%%\n", |
cc34568d | 385 | Math::doublePercent(currentSize(), maxSize())); |
d3b3ab85 | 386 | storeAppendPrintf(&sentry, "Filemap bits in use: %d of %d (%d%%)\n", |
75f8f9a2 FC |
387 | map->numFilesInMap(), map->capacity(), |
388 | Math::intPercent(map->numFilesInMap(), map->capacity())); | |
d3b3ab85 | 389 | x = storeDirGetUFSStats(path, &totl_kb, &free_kb, &totl_in, &free_in); |
62e76326 | 390 | |
d3b3ab85 | 391 | if (0 == x) { |
62e76326 | 392 | storeAppendPrintf(&sentry, "Filesystem Space in use: %d/%d KB (%d%%)\n", |
393 | totl_kb - free_kb, | |
394 | totl_kb, | |
a98bcbee | 395 | Math::intPercent(totl_kb - free_kb, totl_kb)); |
62e76326 | 396 | storeAppendPrintf(&sentry, "Filesystem Inodes in use: %d/%d (%d%%)\n", |
397 | totl_in - free_in, | |
398 | totl_in, | |
a98bcbee | 399 | Math::intPercent(totl_in - free_in, totl_in)); |
d3b3ab85 | 400 | } |
62e76326 | 401 | |
d3b3ab85 | 402 | storeAppendPrintf(&sentry, "Flags:"); |
62e76326 | 403 | |
d3b3ab85 | 404 | if (flags.selected) |
62e76326 | 405 | storeAppendPrintf(&sentry, " SELECTED"); |
406 | ||
d3b3ab85 | 407 | if (flags.read_only) |
62e76326 | 408 | storeAppendPrintf(&sentry, " READ-ONLY"); |
409 | ||
d3b3ab85 | 410 | storeAppendPrintf(&sentry, "\n"); |
59b2d47f | 411 | |
412 | IO->statfs(sentry); | |
d3b3ab85 | 413 | } |
414 | ||
415 | void | |
58373ff8 | 416 | Fs::Ufs::UFSSwapDir::maintain() |
d3b3ab85 | 417 | { |
650c4b88 | 418 | /* We can't delete objects while rebuilding swap */ |
419 | ||
bef81ea5 | 420 | /* XXX FIXME each store should start maintaining as it comes online. */ |
421 | ||
422 | if (StoreController::store_dirs_rebuilding) | |
650c4b88 | 423 | return; |
424 | ||
d3b3ab85 | 425 | StoreEntry *e = NULL; |
650c4b88 | 426 | |
d3b3ab85 | 427 | int removed = 0; |
650c4b88 | 428 | |
d3b3ab85 | 429 | RemovalPurgeWalker *walker; |
62e76326 | 430 | |
cc34568d | 431 | double f = (double) (currentSize() - minSize()) / (maxSize() - minSize()); |
650c4b88 | 432 | |
433 | f = f < 0.0 ? 0.0 : f > 1.0 ? 1.0 : f; | |
434 | ||
435 | int max_scan = (int) (f * 400.0 + 100.0); | |
436 | ||
437 | int max_remove = (int) (f * 70.0 + 10.0); | |
438 | ||
439 | /* | |
440 | * This is kinda cheap, but so we need this priority hack? | |
441 | */ | |
62e76326 | 442 | |
58373ff8 | 443 | debugs(47, 3, HERE << "f=" << f << ", max_scan=" << max_scan << ", max_remove=" << max_remove ); |
650c4b88 | 444 | |
d3b3ab85 | 445 | walker = repl->PurgeInit(repl, max_scan); |
62e76326 | 446 | |
d3b3ab85 | 447 | while (1) { |
cc34568d | 448 | if (currentSize() < minSize()) |
62e76326 | 449 | break; |
450 | ||
451 | if (removed >= max_remove) | |
452 | break; | |
453 | ||
454 | e = walker->Next(walker); | |
455 | ||
456 | if (!e) | |
58373ff8 | 457 | break; /* no more objects */ |
62e76326 | 458 | |
cb4185f1 | 459 | ++removed; |
62e76326 | 460 | |
5f33b71d | 461 | e->release(); |
d3b3ab85 | 462 | } |
62e76326 | 463 | |
d3b3ab85 | 464 | walker->Done(walker); |
58373ff8 | 465 | debugs(47, (removed ? 2 : 3), HERE << path << |
26ac0430 | 466 | " removed " << removed << "/" << max_remove << " f=" << |
bf8fe701 | 467 | std::setprecision(4) << f << " max_scan=" << max_scan); |
d3b3ab85 | 468 | } |
469 | ||
d3b3ab85 | 470 | void |
58373ff8 | 471 | Fs::Ufs::UFSSwapDir::reference(StoreEntry &e) |
d3b3ab85 | 472 | { |
58373ff8 FC |
473 | debugs(47, 3, HERE << "referencing " << &e << " " << |
474 | e.swap_dirn << "/" << e.swap_filen); | |
62e76326 | 475 | |
d3b3ab85 | 476 | if (repl->Referenced) |
62e76326 | 477 | repl->Referenced(repl, &e, &e.repl); |
478 | } | |
d3b3ab85 | 479 | |
4c973beb | 480 | bool |
58373ff8 | 481 | Fs::Ufs::UFSSwapDir::dereference(StoreEntry & e) |
d3b3ab85 | 482 | { |
58373ff8 FC |
483 | debugs(47, 3, HERE << "dereferencing " << &e << " " << |
484 | e.swap_dirn << "/" << e.swap_filen); | |
62e76326 | 485 | |
d3b3ab85 | 486 | if (repl->Dereferenced) |
62e76326 | 487 | repl->Dereferenced(repl, &e, &e.repl); |
4c973beb AR |
488 | |
489 | return true; // keep e in the global store_table | |
d3b3ab85 | 490 | } |
491 | ||
492 | StoreIOState::Pointer | |
58373ff8 | 493 | Fs::Ufs::UFSSwapDir::createStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) |
d3b3ab85 | 494 | { |
18ec8500 | 495 | return IO->create (this, &e, file_callback, aCallback, callback_data); |
d3b3ab85 | 496 | } |
497 | ||
498 | StoreIOState::Pointer | |
58373ff8 | 499 | Fs::Ufs::UFSSwapDir::openStoreIO(StoreEntry &e, StoreIOState::STFNCB * file_callback, StoreIOState::STIOCB * aCallback, void *callback_data) |
d3b3ab85 | 500 | { |
18ec8500 | 501 | return IO->open (this, &e, file_callback, aCallback, callback_data); |
d3b3ab85 | 502 | } |
503 | ||
504 | int | |
58373ff8 | 505 | Fs::Ufs::UFSSwapDir::mapBitTest(sfileno filn) |
d3b3ab85 | 506 | { |
75f8f9a2 | 507 | return map->testBit(filn); |
d3b3ab85 | 508 | } |
509 | ||
510 | void | |
58373ff8 | 511 | Fs::Ufs::UFSSwapDir::mapBitSet(sfileno filn) |
d3b3ab85 | 512 | { |
75f8f9a2 | 513 | map->setBit(filn); |
d3b3ab85 | 514 | } |
515 | ||
516 | void | |
58373ff8 | 517 | Fs::Ufs::UFSSwapDir::mapBitReset(sfileno filn) |
d3b3ab85 | 518 | { |
519 | /* | |
75f8f9a2 FC |
520 | * We have to test the bit before calling clearBit as |
521 | * it doesn't do bounds checking and blindly assumes | |
d3b3ab85 | 522 | * filn is a valid file number, but it might not be because |
523 | * the map is dynamic in size. Also clearing an already clear | |
524 | * bit puts the map counter of-of-whack. | |
525 | */ | |
62e76326 | 526 | |
75f8f9a2 FC |
527 | if (map->testBit(filn)) |
528 | map->clearBit(filn); | |
d3b3ab85 | 529 | } |
530 | ||
531 | int | |
58373ff8 | 532 | Fs::Ufs::UFSSwapDir::mapBitAllocate() |
d3b3ab85 | 533 | { |
534 | int fn; | |
75f8f9a2 FC |
535 | fn = map->allocate(suggest); |
536 | map->setBit(fn); | |
d3b3ab85 | 537 | suggest = fn + 1; |
538 | return fn; | |
539 | } | |
540 | ||
d3b3ab85 | 541 | char * |
58373ff8 | 542 | Fs::Ufs::UFSSwapDir::swapSubDir(int subdirn)const |
d3b3ab85 | 543 | { |
ef364f64 | 544 | LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); |
d3b3ab85 | 545 | assert(0 <= subdirn && subdirn < l1); |
ef364f64 | 546 | snprintf(fullfilename, MAXPATHLEN, "%s/%02X", path, subdirn); |
d3b3ab85 | 547 | return fullfilename; |
548 | } | |
549 | ||
550 | int | |
58373ff8 | 551 | Fs::Ufs::UFSSwapDir::createDirectory(const char *aPath, int should_exist) |
d3b3ab85 | 552 | { |
553 | int created = 0; | |
62e76326 | 554 | |
d3b3ab85 | 555 | struct stat st; |
556 | getCurrentTime(); | |
62e76326 | 557 | |
e4ae841b | 558 | if (0 == ::stat(aPath, &st)) { |
62e76326 | 559 | if (S_ISDIR(st.st_mode)) { |
58373ff8 | 560 | debugs(47, (should_exist ? 3 : DBG_IMPORTANT), aPath << " exists"); |
62e76326 | 561 | } else { |
e4ae841b | 562 | fatalf("Swap directory %s is not a directory.", aPath); |
62e76326 | 563 | } |
58373ff8 FC |
564 | |
565 | #if _SQUID_MSWIN_ | |
566 | ||
567 | } else if (0 == mkdir(aPath)) { | |
568 | #else | |
569 | ||
e4ae841b | 570 | } else if (0 == mkdir(aPath, 0755)) { |
58373ff8 FC |
571 | #endif |
572 | debugs(47, (should_exist ? DBG_IMPORTANT : 3), aPath << " created"); | |
62e76326 | 573 | created = 1; |
d3b3ab85 | 574 | } else { |
62e76326 | 575 | fatalf("Failed to make swap directory %s: %s", |
e4ae841b | 576 | aPath, xstrerror()); |
d3b3ab85 | 577 | } |
62e76326 | 578 | |
d3b3ab85 | 579 | return created; |
580 | } | |
581 | ||
582 | bool | |
58373ff8 | 583 | Fs::Ufs::UFSSwapDir::pathIsDirectory(const char *aPath)const |
d3b3ab85 | 584 | { |
62e76326 | 585 | |
d3b3ab85 | 586 | struct stat sb; |
62e76326 | 587 | |
e4ae841b | 588 | if (::stat(aPath, &sb) < 0) { |
58373ff8 | 589 | debugs(47, DBG_CRITICAL, "ERROR: " << aPath << ": " << xstrerror()); |
62e76326 | 590 | return false; |
d3b3ab85 | 591 | } |
62e76326 | 592 | |
d3b3ab85 | 593 | if (S_ISDIR(sb.st_mode) == 0) { |
58373ff8 | 594 | debugs(47, DBG_CRITICAL, "WARNING: " << aPath << " is not a directory"); |
62e76326 | 595 | return false; |
d3b3ab85 | 596 | } |
62e76326 | 597 | |
d3b3ab85 | 598 | return true; |
599 | } | |
600 | ||
d3b3ab85 | 601 | bool |
58373ff8 | 602 | Fs::Ufs::UFSSwapDir::verifyCacheDirs() |
d3b3ab85 | 603 | { |
604 | if (!pathIsDirectory(path)) | |
62e76326 | 605 | return true; |
606 | ||
cb4185f1 | 607 | for (int j = 0; j < l1; ++j) { |
62e76326 | 608 | char const *aPath = swapSubDir(j); |
609 | ||
610 | if (!pathIsDirectory(aPath)) | |
611 | return true; | |
d3b3ab85 | 612 | } |
62e76326 | 613 | |
d3b3ab85 | 614 | return false; |
615 | } | |
616 | ||
617 | void | |
58373ff8 | 618 | Fs::Ufs::UFSSwapDir::createSwapSubDirs() |
d3b3ab85 | 619 | { |
d3b3ab85 | 620 | LOCAL_ARRAY(char, name, MAXPATHLEN); |
62e76326 | 621 | |
cb4185f1 | 622 | for (int i = 0; i < l1; ++i) { |
62e76326 | 623 | snprintf(name, MAXPATHLEN, "%s/%02X", path, i); |
624 | ||
59b2d47f | 625 | int should_exist; |
626 | ||
62e76326 | 627 | if (createDirectory(name, 0)) |
628 | should_exist = 0; | |
629 | else | |
630 | should_exist = 1; | |
631 | ||
58373ff8 | 632 | debugs(47, DBG_IMPORTANT, "Making directories in " << name); |
62e76326 | 633 | |
cb4185f1 | 634 | for (int k = 0; k < l2; ++k) { |
62e76326 | 635 | snprintf(name, MAXPATHLEN, "%s/%02X/%02X", path, i, k); |
636 | createDirectory(name, should_exist); | |
637 | } | |
d3b3ab85 | 638 | } |
639 | } | |
640 | ||
641 | char * | |
58373ff8 | 642 | Fs::Ufs::UFSSwapDir::logFile(char const *ext) const |
d3b3ab85 | 643 | { |
ef364f64 AJ |
644 | LOCAL_ARRAY(char, lpath, MAXPATHLEN); |
645 | LOCAL_ARRAY(char, pathtmp, MAXPATHLEN); | |
d3b3ab85 | 646 | LOCAL_ARRAY(char, digit, 32); |
647 | char *pathtmp2; | |
62e76326 | 648 | |
d3b3ab85 | 649 | if (Config.Log.swap) { |
ef364f64 | 650 | xstrncpy(pathtmp, path, MAXPATHLEN - 64); |
62e76326 | 651 | pathtmp2 = pathtmp; |
652 | ||
653 | while ((pathtmp2 = strchr(pathtmp2, '/')) != NULL) | |
654 | *pathtmp2 = '.'; | |
655 | ||
656 | while (strlen(pathtmp) && pathtmp[strlen(pathtmp) - 1] == '.') | |
657 | pathtmp[strlen(pathtmp) - 1] = '\0'; | |
658 | ||
cb4185f1 | 659 | for (pathtmp2 = pathtmp; *pathtmp2 == '.'; ++pathtmp2); |
ef364f64 | 660 | snprintf(lpath, MAXPATHLEN - 64, Config.Log.swap, pathtmp2); |
62e76326 | 661 | |
ef364f64 | 662 | if (strncmp(lpath, Config.Log.swap, MAXPATHLEN - 64) == 0) { |
62e76326 | 663 | strcat(lpath, "."); |
664 | snprintf(digit, 32, "%02d", index); | |
665 | strncat(lpath, digit, 3); | |
666 | } | |
d3b3ab85 | 667 | } else { |
ef364f64 | 668 | xstrncpy(lpath, path, MAXPATHLEN - 64); |
62e76326 | 669 | strcat(lpath, "/swap.state"); |
d3b3ab85 | 670 | } |
62e76326 | 671 | |
d3b3ab85 | 672 | if (ext) |
62e76326 | 673 | strncat(lpath, ext, 16); |
674 | ||
d3b3ab85 | 675 | return lpath; |
676 | } | |
677 | ||
678 | void | |
58373ff8 | 679 | Fs::Ufs::UFSSwapDir::openLog() |
d3b3ab85 | 680 | { |
681 | char *logPath; | |
682 | logPath = logFile(); | |
683 | swaplog_fd = file_open(logPath, O_WRONLY | O_CREAT | O_BINARY); | |
62e76326 | 684 | |
d3b3ab85 | 685 | if (swaplog_fd < 0) { |
58373ff8 FC |
686 | debugs(50, DBG_IMPORTANT, "ERROR opening swap log " << logPath << ": " << xstrerror()); |
687 | fatal("UFSSwapDir::openLog: Failed to open swap log."); | |
d3b3ab85 | 688 | } |
62e76326 | 689 | |
58373ff8 | 690 | debugs(50, 3, HERE << "Cache Dir #" << index << " log opened on FD " << swaplog_fd); |
62e76326 | 691 | |
d3b3ab85 | 692 | if (0 == NumberOfUFSDirs) |
62e76326 | 693 | assert(NULL == UFSDirToGlobalDirMapping); |
694 | ||
d3b3ab85 | 695 | ++NumberOfUFSDirs; |
62e76326 | 696 | |
d3b3ab85 | 697 | assert(NumberOfUFSDirs <= Config.cacheSwap.n_configured); |
698 | } | |
699 | ||
700 | void | |
58373ff8 | 701 | Fs::Ufs::UFSSwapDir::closeLog() |
d3b3ab85 | 702 | { |
58373ff8 | 703 | if (swaplog_fd < 0) /* not open */ |
62e76326 | 704 | return; |
705 | ||
d3b3ab85 | 706 | file_close(swaplog_fd); |
62e76326 | 707 | |
bf8fe701 | 708 | debugs(47, 3, "Cache Dir #" << index << " log closed on FD " << swaplog_fd); |
62e76326 | 709 | |
d3b3ab85 | 710 | swaplog_fd = -1; |
62e76326 | 711 | |
59b2d47f | 712 | --NumberOfUFSDirs; |
62e76326 | 713 | |
d3b3ab85 | 714 | assert(NumberOfUFSDirs >= 0); |
62e76326 | 715 | |
eb5ac42b | 716 | if (0 == NumberOfUFSDirs) |
62e76326 | 717 | safe_free(UFSDirToGlobalDirMapping); |
d3b3ab85 | 718 | } |
719 | ||
720 | bool | |
58373ff8 | 721 | Fs::Ufs::UFSSwapDir::validL1(int anInt) const |
d3b3ab85 | 722 | { |
723 | return anInt < l1; | |
724 | } | |
725 | ||
726 | bool | |
58373ff8 | 727 | Fs::Ufs::UFSSwapDir::validL2(int anInt) const |
d3b3ab85 | 728 | { |
729 | return anInt < l2; | |
730 | } | |
731 | ||
d3b3ab85 | 732 | StoreEntry * |
58373ff8 | 733 | Fs::Ufs::UFSSwapDir::addDiskRestore(const cache_key * key, |
62e76326 | 734 | sfileno file_number, |
47f6e231 | 735 | uint64_t swap_file_sz, |
62e76326 | 736 | time_t expires, |
737 | time_t timestamp, | |
738 | time_t lastref, | |
739 | time_t lastmod, | |
09aabd84 FC |
740 | uint32_t refcount, |
741 | uint16_t newFlags, | |
62e76326 | 742 | int clean) |
d3b3ab85 | 743 | { |
744 | StoreEntry *e = NULL; | |
58373ff8 | 745 | debugs(47, 5, HERE << storeKeyText(key) << |
bf8fe701 | 746 | ", fileno="<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << file_number); |
62e76326 | 747 | /* if you call this you'd better be sure file_number is not |
d3b3ab85 | 748 | * already in use! */ |
c8f4eac4 | 749 | e = new StoreEntry(); |
d3b3ab85 | 750 | e->store_status = STORE_OK; |
3900307b | 751 | e->setMemStatus(NOT_IN_MEMORY); |
d3b3ab85 | 752 | e->swap_status = SWAPOUT_DONE; |
753 | e->swap_filen = file_number; | |
754 | e->swap_dirn = index; | |
755 | e->swap_file_sz = swap_file_sz; | |
756 | e->lock_count = 0; | |
757 | e->lastref = lastref; | |
758 | e->timestamp = timestamp; | |
759 | e->expires = expires; | |
760 | e->lastmod = lastmod; | |
761 | e->refcount = refcount; | |
e4ae841b | 762 | e->flags = newFlags; |
d3b3ab85 | 763 | EBIT_SET(e->flags, ENTRY_CACHABLE); |
764 | EBIT_CLR(e->flags, RELEASE_REQUEST); | |
765 | EBIT_CLR(e->flags, KEY_PRIVATE); | |
766 | e->ping_status = PING_NONE; | |
767 | EBIT_CLR(e->flags, ENTRY_VALIDATED); | |
768 | mapBitSet(e->swap_filen); | |
a86ff82a DK |
769 | cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); |
770 | ++n_disk_objects; | |
58373ff8 | 771 | e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ |
d3b3ab85 | 772 | replacementAdd (e); |
773 | return e; | |
774 | } | |
775 | ||
c6e51aff | 776 | void |
58373ff8 | 777 | Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e) |
c6e51aff AR |
778 | { |
779 | debugs(47, 5, HERE << *e); | |
780 | replacementRemove(e); // checks swap_dirn so do it before we invalidate it | |
781 | // Do not unlink the file as it might be used by a subsequent entry. | |
782 | mapBitReset(e->swap_filen); | |
783 | e->swap_filen = -1; | |
784 | e->swap_dirn = -1; | |
785 | cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz); | |
786 | --n_disk_objects; | |
787 | } | |
788 | ||
d3b3ab85 | 789 | void |
58373ff8 | 790 | Fs::Ufs::UFSSwapDir::rebuild() |
d3b3ab85 | 791 | { |
bef81ea5 | 792 | ++StoreController::store_dirs_rebuilding; |
58373ff8 | 793 | eventAdd("storeRebuild", Fs::Ufs::RebuildState::RebuildStep, new Fs::Ufs::RebuildState(this), 0.0, 1); |
d3b3ab85 | 794 | } |
795 | ||
796 | void | |
58373ff8 | 797 | Fs::Ufs::UFSSwapDir::closeTmpSwapLog() |
d3b3ab85 | 798 | { |
799 | char *swaplog_path = xstrdup(logFile(NULL)); | |
800 | char *new_path = xstrdup(logFile(".new")); | |
801 | int fd; | |
802 | file_close(swaplog_fd); | |
62e76326 | 803 | |
d3b3ab85 | 804 | if (xrename(new_path, swaplog_path) < 0) { |
58373ff8 | 805 | debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); |
c031952f | 806 | fatalf("Failed to rename log file %s to %s.new", swaplog_path, swaplog_path); |
d3b3ab85 | 807 | } |
62e76326 | 808 | |
d3b3ab85 | 809 | fd = file_open(swaplog_path, O_WRONLY | O_CREAT | O_BINARY); |
62e76326 | 810 | |
d3b3ab85 | 811 | if (fd < 0) { |
58373ff8 | 812 | debugs(50, DBG_IMPORTANT, HERE << "ERROR: " << swaplog_path << ": " << xstrerror()); |
c031952f | 813 | fatalf("Failed to open swap log %s", swaplog_path); |
d3b3ab85 | 814 | } |
62e76326 | 815 | |
d3b3ab85 | 816 | safe_free(swaplog_path); |
817 | safe_free(new_path); | |
818 | swaplog_fd = fd; | |
58373ff8 | 819 | debugs(47, 3, HERE << "Cache Dir #" << index << " log opened on FD " << fd); |
d3b3ab85 | 820 | } |
821 | ||
822 | FILE * | |
58373ff8 | 823 | Fs::Ufs::UFSSwapDir::openTmpSwapLog(int *clean_flag, int *zero_flag) |
d3b3ab85 | 824 | { |
825 | char *swaplog_path = xstrdup(logFile(NULL)); | |
826 | char *clean_path = xstrdup(logFile(".last-clean")); | |
827 | char *new_path = xstrdup(logFile(".new")); | |
62e76326 | 828 | |
d3b3ab85 | 829 | struct stat log_sb; |
62e76326 | 830 | |
d3b3ab85 | 831 | struct stat clean_sb; |
832 | FILE *fp; | |
833 | int fd; | |
62e76326 | 834 | |
c8f4eac4 | 835 | if (::stat(swaplog_path, &log_sb) < 0) { |
58373ff8 | 836 | debugs(47, DBG_IMPORTANT, HERE << "Cache Dir #" << index << ": No log file"); |
62e76326 | 837 | safe_free(swaplog_path); |
838 | safe_free(clean_path); | |
839 | safe_free(new_path); | |
840 | return NULL; | |
d3b3ab85 | 841 | } |
62e76326 | 842 | |
d3b3ab85 | 843 | *zero_flag = log_sb.st_size == 0 ? 1 : 0; |
844 | /* close the existing write-only FD */ | |
62e76326 | 845 | |
d3b3ab85 | 846 | if (swaplog_fd >= 0) |
62e76326 | 847 | file_close(swaplog_fd); |
848 | ||
d3b3ab85 | 849 | /* open a write-only FD for the new log */ |
850 | fd = file_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); | |
26ac0430 | 851 | |
d3b3ab85 | 852 | if (fd < 0) { |
58373ff8 FC |
853 | debugs(50, DBG_IMPORTANT, "ERROR: while opening swap log" << new_path << ": " << xstrerror()); |
854 | fatal("UFSSwapDir::openTmpSwapLog: Failed to open swap log."); | |
d3b3ab85 | 855 | } |
26ac0430 | 856 | |
d3b3ab85 | 857 | swaplog_fd = fd; |
47f6e231 | 858 | |
95ec9c9d | 859 | { |
155d0ed6 AR |
860 | const StoreSwapLogHeader header; |
861 | MemBuf buf; | |
862 | buf.init(header.record_size, header.record_size); | |
863 | buf.append(reinterpret_cast<const char*>(&header), sizeof(header)); | |
864 | // Pad to keep in sync with UFSSwapDir::writeCleanStart(). | |
786f6516 AR |
865 | memset(buf.space(), 0, header.gapSize()); |
866 | buf.appended(header.gapSize()); | |
155d0ed6 AR |
867 | file_write(swaplog_fd, -1, buf.content(), buf.contentSize(), |
868 | NULL, NULL, buf.freeFunc()); | |
95ec9c9d | 869 | } |
47f6e231 | 870 | |
d3b3ab85 | 871 | /* open a read-only stream of the old log */ |
872 | fp = fopen(swaplog_path, "rb"); | |
62e76326 | 873 | |
d3b3ab85 | 874 | if (fp == NULL) { |
58373ff8 | 875 | debugs(50, DBG_CRITICAL, "ERROR: while opening " << swaplog_path << ": " << xstrerror()); |
62e76326 | 876 | fatal("Failed to open swap log for reading"); |
d3b3ab85 | 877 | } |
62e76326 | 878 | |
d3b3ab85 | 879 | memset(&clean_sb, '\0', sizeof(struct stat)); |
62e76326 | 880 | |
c8f4eac4 | 881 | if (::stat(clean_path, &clean_sb) < 0) |
62e76326 | 882 | *clean_flag = 0; |
d3b3ab85 | 883 | else if (clean_sb.st_mtime < log_sb.st_mtime) |
62e76326 | 884 | *clean_flag = 0; |
d3b3ab85 | 885 | else |
62e76326 | 886 | *clean_flag = 1; |
887 | ||
d3b3ab85 | 888 | safeunlink(clean_path, 1); |
62e76326 | 889 | |
d3b3ab85 | 890 | safe_free(swaplog_path); |
62e76326 | 891 | |
d3b3ab85 | 892 | safe_free(clean_path); |
62e76326 | 893 | |
d3b3ab85 | 894 | safe_free(new_path); |
62e76326 | 895 | |
d3b3ab85 | 896 | return fp; |
897 | } | |
898 | ||
59b2d47f | 899 | /* |
900 | * Begin the process to write clean cache state. For AUFS this means | |
901 | * opening some log files and allocating write buffers. Return 0 if | |
902 | * we succeed, and assign the 'func' and 'data' return pointers. | |
903 | */ | |
d3b3ab85 | 904 | int |
58373ff8 | 905 | Fs::Ufs::UFSSwapDir::writeCleanStart() |
d3b3ab85 | 906 | { |
907 | UFSCleanLog *state = new UFSCleanLog(this); | |
47f6e231 | 908 | StoreSwapLogHeader header; |
d3b3ab85 | 909 | #if HAVE_FCHMOD |
62e76326 | 910 | |
d3b3ab85 | 911 | struct stat sb; |
912 | #endif | |
62e76326 | 913 | |
d3b3ab85 | 914 | cleanLog = NULL; |
915 | state->newLog = xstrdup(logFile(".clean")); | |
916 | state->fd = file_open(state->newLog, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY); | |
62e76326 | 917 | |
d3b3ab85 | 918 | if (state->fd < 0) { |
62e76326 | 919 | xfree(state->newLog); |
920 | delete state; | |
921 | return -1; | |
d3b3ab85 | 922 | } |
62e76326 | 923 | |
d3b3ab85 | 924 | state->cur = xstrdup(logFile(NULL)); |
925 | state->cln = xstrdup(logFile(".last-clean")); | |
926 | state->outbuf = (char *)xcalloc(CLEAN_BUF_SZ, 1); | |
927 | state->outbuf_offset = 0; | |
47f6e231 | 928 | /*copy the header */ |
41d00cd3 | 929 | memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader)); |
155d0ed6 | 930 | // Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog(). |
786f6516 | 931 | memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize()); |
47f6e231 | 932 | state->outbuf_offset += header.record_size; |
933 | ||
d3b3ab85 | 934 | state->walker = repl->WalkInit(repl); |
935 | ::unlink(state->cln); | |
58373ff8 | 936 | debugs(47, 3, HERE << "opened " << state->newLog << ", FD " << state->fd); |
d3b3ab85 | 937 | #if HAVE_FCHMOD |
62e76326 | 938 | |
c8f4eac4 | 939 | if (::stat(state->cur, &sb) == 0) |
62e76326 | 940 | fchmod(state->fd, sb.st_mode); |
941 | ||
d3b3ab85 | 942 | #endif |
26ac0430 | 943 | |
62e76326 | 944 | |
d3b3ab85 | 945 | cleanLog = state; |
946 | return 0; | |
947 | } | |
948 | ||
d3b3ab85 | 949 | void |
58373ff8 | 950 | Fs::Ufs::UFSSwapDir::writeCleanDone() |
d3b3ab85 | 951 | { |
952 | UFSCleanLog *state = (UFSCleanLog *)cleanLog; | |
953 | int fd; | |
62e76326 | 954 | |
d3b3ab85 | 955 | if (NULL == state) |
62e76326 | 956 | return; |
957 | ||
d3b3ab85 | 958 | if (state->fd < 0) |
62e76326 | 959 | return; |
960 | ||
d3b3ab85 | 961 | state->walker->Done(state->walker); |
62e76326 | 962 | |
d3b3ab85 | 963 | if (FD_WRITE_METHOD(state->fd, state->outbuf, state->outbuf_offset) < 0) { |
58373ff8 FC |
964 | debugs(50, DBG_CRITICAL, HERE << state->newLog << ": write: " << xstrerror()); |
965 | debugs(50, DBG_CRITICAL, HERE << "Current swap logfile not replaced."); | |
62e76326 | 966 | file_close(state->fd); |
967 | state->fd = -1; | |
968 | ::unlink(state->newLog); | |
d3b3ab85 | 969 | } |
62e76326 | 970 | |
d3b3ab85 | 971 | safe_free(state->outbuf); |
972 | /* | |
973 | * You can't rename open files on Microsoft "operating systems" | |
974 | * so we have to close before renaming. | |
975 | */ | |
976 | closeLog(); | |
977 | /* save the fd value for a later test */ | |
978 | fd = state->fd; | |
979 | /* rename */ | |
62e76326 | 980 | |
d3b3ab85 | 981 | if (state->fd >= 0) { |
be266cb2 | 982 | #if _SQUID_OS2_ || _SQUID_WINDOWS_ |
62e76326 | 983 | file_close(state->fd); |
984 | state->fd = -1; | |
d3b3ab85 | 985 | #endif |
62e76326 | 986 | |
987 | xrename(state->newLog, state->cur); | |
d3b3ab85 | 988 | } |
62e76326 | 989 | |
d3b3ab85 | 990 | /* touch a timestamp file if we're not still validating */ |
bef81ea5 | 991 | if (StoreController::store_dirs_rebuilding) |
62e76326 | 992 | (void) 0; |
d3b3ab85 | 993 | else if (fd < 0) |
62e76326 | 994 | (void) 0; |
d3b3ab85 | 995 | else |
62e76326 | 996 | file_close(file_open(state->cln, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY)); |
997 | ||
d3b3ab85 | 998 | /* close */ |
999 | safe_free(state->cur); | |
62e76326 | 1000 | |
d3b3ab85 | 1001 | safe_free(state->newLog); |
62e76326 | 1002 | |
d3b3ab85 | 1003 | safe_free(state->cln); |
62e76326 | 1004 | |
d3b3ab85 | 1005 | if (state->fd >= 0) |
62e76326 | 1006 | file_close(state->fd); |
1007 | ||
d3b3ab85 | 1008 | state->fd = -1; |
62e76326 | 1009 | |
d3b3ab85 | 1010 | delete state; |
62e76326 | 1011 | |
d3b3ab85 | 1012 | cleanLog = NULL; |
1013 | } | |
1014 | ||
d3b3ab85 | 1015 | void |
58373ff8 | 1016 | Fs::Ufs::UFSSwapDir::CleanEvent(void *unused) |
d3b3ab85 | 1017 | { |
1018 | static int swap_index = 0; | |
1019 | int i; | |
1020 | int j = 0; | |
1021 | int n = 0; | |
1022 | /* | |
1023 | * Assert that there are UFS cache_dirs configured, otherwise | |
1024 | * we should never be called. | |
1025 | */ | |
1026 | assert(NumberOfUFSDirs); | |
62e76326 | 1027 | |
d3b3ab85 | 1028 | if (NULL == UFSDirToGlobalDirMapping) { |
62e76326 | 1029 | SwapDir *sd; |
1030 | /* | |
1031 | * Initialize the little array that translates UFS cache_dir | |
1032 | * number into the Config.cacheSwap.swapDirs array index. | |
1033 | */ | |
1034 | UFSDirToGlobalDirMapping = (int *)xcalloc(NumberOfUFSDirs, sizeof(*UFSDirToGlobalDirMapping)); | |
1035 | ||
cb4185f1 | 1036 | for (i = 0, n = 0; i < Config.cacheSwap.n_configured; ++i) { |
c8f4eac4 | 1037 | /* This is bogus, the controller should just clean each instance once */ |
1038 | sd = dynamic_cast <SwapDir *>(INDEXSD(i)); | |
62e76326 | 1039 | |
1040 | if (!UFSSwapDir::IsUFSDir(sd)) | |
1041 | continue; | |
1042 | ||
1043 | UFSSwapDir *usd = dynamic_cast<UFSSwapDir *>(sd); | |
1044 | ||
1045 | assert (usd); | |
1046 | ||
a38ec4b1 FC |
1047 | UFSDirToGlobalDirMapping[n] = i; |
1048 | ++n; | |
62e76326 | 1049 | |
1050 | j += (usd->l1 * usd->l2); | |
1051 | } | |
1052 | ||
1053 | assert(n == NumberOfUFSDirs); | |
1054 | /* | |
1055 | * Start the commonUfsDirClean() swap_index with a random | |
1056 | * value. j equals the total number of UFS level 2 | |
1057 | * swap directories | |
1058 | */ | |
1059 | swap_index = (int) (squid_random() % j); | |
d3b3ab85 | 1060 | } |
62e76326 | 1061 | |
bef81ea5 | 1062 | /* if the rebuild is finished, start cleaning directories. */ |
1063 | if (0 == StoreController::store_dirs_rebuilding) { | |
62e76326 | 1064 | n = DirClean(swap_index); |
cb4185f1 | 1065 | ++swap_index; |
d3b3ab85 | 1066 | } |
62e76326 | 1067 | |
d3b3ab85 | 1068 | eventAdd("storeDirClean", CleanEvent, NULL, |
62e76326 | 1069 | 15.0 * exp(-0.25 * n), 1); |
d3b3ab85 | 1070 | } |
1071 | ||
58373ff8 FC |
1072 | bool |
1073 | Fs::Ufs::UFSSwapDir::IsUFSDir(SwapDir * sd) | |
d3b3ab85 | 1074 | { |
1075 | UFSSwapDir *mySD = dynamic_cast<UFSSwapDir *>(sd); | |
58373ff8 | 1076 | return (mySD != 0) ; |
d3b3ab85 | 1077 | } |
1078 | ||
1079 | /* | |
d3b3ab85 | 1080 | * XXX: this is broken - it assumes all cache dirs use the same |
26ac0430 | 1081 | * l1 and l2 scheme. -RBC 20021215. Partial fix is in place - |
d3b3ab85 | 1082 | * if not UFSSwapDir return 0; |
1083 | */ | |
58373ff8 FC |
1084 | bool |
1085 | Fs::Ufs::UFSSwapDir::FilenoBelongsHere(int fn, int F0, int F1, int F2) | |
d3b3ab85 | 1086 | { |
1087 | int D1, D2; | |
1088 | int L1, L2; | |
1089 | int filn = fn; | |
1090 | assert(F0 < Config.cacheSwap.n_configured); | |
c8f4eac4 | 1091 | assert (UFSSwapDir::IsUFSDir (dynamic_cast<SwapDir *>(INDEXSD(F0)))); |
d3b3ab85 | 1092 | UFSSwapDir *sd = dynamic_cast<UFSSwapDir *>(INDEXSD(F0)); |
62e76326 | 1093 | |
d3b3ab85 | 1094 | if (!sd) |
62e76326 | 1095 | return 0; |
1096 | ||
d3b3ab85 | 1097 | L1 = sd->l1; |
62e76326 | 1098 | |
d3b3ab85 | 1099 | L2 = sd->l2; |
62e76326 | 1100 | |
d3b3ab85 | 1101 | D1 = ((filn / L2) / L2) % L1; |
62e76326 | 1102 | |
d3b3ab85 | 1103 | if (F1 != D1) |
62e76326 | 1104 | return 0; |
1105 | ||
d3b3ab85 | 1106 | D2 = (filn / L2) % L2; |
62e76326 | 1107 | |
d3b3ab85 | 1108 | if (F2 != D2) |
62e76326 | 1109 | return 0; |
1110 | ||
d3b3ab85 | 1111 | return 1; |
1112 | } | |
1113 | ||
1114 | int | |
58373ff8 | 1115 | Fs::Ufs::UFSSwapDir::validFileno(sfileno filn, int flag) const |
d3b3ab85 | 1116 | { |
1117 | if (filn < 0) | |
62e76326 | 1118 | return 0; |
1119 | ||
d3b3ab85 | 1120 | /* |
1121 | * If flag is set it means out-of-range file number should | |
1122 | * be considered invalid. | |
1123 | */ | |
1124 | if (flag) | |
75f8f9a2 | 1125 | if (filn > map->capacity()) |
62e76326 | 1126 | return 0; |
1127 | ||
d3b3ab85 | 1128 | return 1; |
1129 | } | |
1130 | ||
d3b3ab85 | 1131 | void |
58373ff8 | 1132 | Fs::Ufs::UFSSwapDir::unlinkFile(sfileno f) |
d3b3ab85 | 1133 | { |
58373ff8 | 1134 | debugs(79, 3, HERE << "unlinking fileno " << std::setfill('0') << |
26ac0430 AJ |
1135 | std::hex << std::uppercase << std::setw(8) << f << " '" << |
1136 | fullPath(f,NULL) << "'"); | |
d3b3ab85 | 1137 | /* commonUfsDirMapBitReset(this, f); */ |
59b2d47f | 1138 | IO->unlinkFile(fullPath(f,NULL)); |
1139 | } | |
1140 | ||
c521ad17 | 1141 | bool |
58373ff8 | 1142 | Fs::Ufs::UFSSwapDir::unlinkdUseful() const |
c521ad17 DK |
1143 | { |
1144 | // unlinkd may be useful only in workers | |
1145 | return IamWorkerProcess() && IO->io->unlinkdUseful(); | |
1146 | } | |
1147 | ||
59b2d47f | 1148 | void |
58373ff8 | 1149 | Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e) |
59b2d47f | 1150 | { |
58373ff8 | 1151 | debugs(79, 3, HERE << "dirno " << index << ", fileno "<< |
bf8fe701 | 1152 | std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); |
c6e51aff | 1153 | if (e.swap_status == SWAPOUT_DONE) { |
a86ff82a DK |
1154 | cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); |
1155 | --n_disk_objects; | |
1156 | } | |
59b2d47f | 1157 | replacementRemove(&e); |
1158 | mapBitReset(e.swap_filen); | |
1159 | UFSSwapDir::unlinkFile(e.swap_filen); | |
d3b3ab85 | 1160 | } |
1161 | ||
d3b3ab85 | 1162 | void |
58373ff8 | 1163 | Fs::Ufs::UFSSwapDir::replacementAdd(StoreEntry * e) |
d3b3ab85 | 1164 | { |
58373ff8 | 1165 | debugs(47, 4, HERE << "added node " << e << " to dir " << index); |
d3b3ab85 | 1166 | repl->Add(repl, e, &e->repl); |
1167 | } | |
1168 | ||
1169 | ||
1170 | void | |
58373ff8 | 1171 | Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e) |
d3b3ab85 | 1172 | { |
c8f4eac4 | 1173 | StorePointer SD; |
62e76326 | 1174 | |
d3b3ab85 | 1175 | if (e->swap_dirn < 0) |
62e76326 | 1176 | return; |
1177 | ||
d3b3ab85 | 1178 | SD = INDEXSD(e->swap_dirn); |
62e76326 | 1179 | |
c8f4eac4 | 1180 | assert (dynamic_cast<UFSSwapDir *>(SD.getRaw()) == this); |
62e76326 | 1181 | |
58373ff8 | 1182 | debugs(47, 4, HERE << "remove node " << e << " from dir " << index); |
62e76326 | 1183 | |
d3b3ab85 | 1184 | repl->Remove(repl, e, &e->repl); |
1185 | } | |
1186 | ||
1187 | void | |
58373ff8 | 1188 | Fs::Ufs::UFSSwapDir::dump(StoreEntry & entry) const |
d3b3ab85 | 1189 | { |
c91ca3ce | 1190 | storeAppendPrintf(&entry, " %" PRIu64 " %d %d", maxSize() >> 20, l1, l2); |
59b2d47f | 1191 | dumpOptions(&entry); |
d3b3ab85 | 1192 | } |
1193 | ||
1194 | char * | |
58373ff8 | 1195 | Fs::Ufs::UFSSwapDir::fullPath(sfileno filn, char *fullpath) const |
d3b3ab85 | 1196 | { |
ef364f64 | 1197 | LOCAL_ARRAY(char, fullfilename, MAXPATHLEN); |
d3b3ab85 | 1198 | int L1 = l1; |
1199 | int L2 = l2; | |
62e76326 | 1200 | |
d3b3ab85 | 1201 | if (!fullpath) |
62e76326 | 1202 | fullpath = fullfilename; |
1203 | ||
d3b3ab85 | 1204 | fullpath[0] = '\0'; |
62e76326 | 1205 | |
ef364f64 | 1206 | snprintf(fullpath, MAXPATHLEN, "%s/%02X/%02X/%08X", |
62e76326 | 1207 | path, |
1208 | ((filn / L2) / L2) % L1, | |
1209 | (filn / L2) % L2, | |
1210 | filn); | |
1211 | ||
d3b3ab85 | 1212 | return fullpath; |
1213 | } | |
59b2d47f | 1214 | |
1215 | int | |
58373ff8 | 1216 | Fs::Ufs::UFSSwapDir::callback() |
59b2d47f | 1217 | { |
1218 | return IO->callback(); | |
1219 | } | |
1220 | ||
1221 | void | |
58373ff8 | 1222 | Fs::Ufs::UFSSwapDir::sync() |
59b2d47f | 1223 | { |
1224 | IO->sync(); | |
1225 | } | |
c8f4eac4 | 1226 | |
da9d3191 | 1227 | void |
58373ff8 | 1228 | Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e) |
da9d3191 | 1229 | { |
a86ff82a DK |
1230 | cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); |
1231 | ++n_disk_objects; | |
da9d3191 DK |
1232 | } |
1233 | ||
c8f4eac4 | 1234 | StoreSearch * |
58373ff8 | 1235 | Fs::Ufs::UFSSwapDir::search(String const url, HttpRequest *request) |
c8f4eac4 | 1236 | { |
1237 | if (url.size()) | |
1238 | fatal ("Cannot search by url yet\n"); | |
1239 | ||
58373ff8 | 1240 | return new Fs::Ufs::StoreSearchUFS (this); |
c8f4eac4 | 1241 | } |
1242 | ||
1243 | void | |
58373ff8 | 1244 | Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const |
c8f4eac4 | 1245 | { |
58373ff8 FC |
1246 | StoreSwapLogData *s = new StoreSwapLogData; |
1247 | s->op = (char) op; | |
1248 | s->swap_filen = e.swap_filen; | |
1249 | s->timestamp = e.timestamp; | |
1250 | s->lastref = e.lastref; | |
1251 | s->expires = e.expires; | |
1252 | s->lastmod = e.lastmod; | |
1253 | s->swap_file_sz = e.swap_file_sz; | |
1254 | s->refcount = e.refcount; | |
1255 | s->flags = e.flags; | |
1256 | memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH); | |
1257 | s->finalize(); | |
1258 | file_write(swaplog_fd, | |
1259 | -1, | |
1260 | s, | |
1261 | sizeof(StoreSwapLogData), | |
1262 | NULL, | |
1263 | NULL, | |
1264 | FreeObject); | |
c8f4eac4 | 1265 | } |
1266 | ||
58373ff8 FC |
1267 | int |
1268 | Fs::Ufs::UFSSwapDir::DirClean(int swap_index) | |
c8f4eac4 | 1269 | { |
58373ff8 | 1270 | DIR *dir_pointer = NULL; |
c8f4eac4 | 1271 | |
58373ff8 FC |
1272 | LOCAL_ARRAY(char, p1, MAXPATHLEN + 1); |
1273 | LOCAL_ARRAY(char, p2, MAXPATHLEN + 1); | |
c8f4eac4 | 1274 | |
58373ff8 FC |
1275 | int files[20]; |
1276 | int swapfileno; | |
1277 | int fn; /* same as swapfileno, but with dirn bits set */ | |
1278 | int n = 0; | |
1279 | int k = 0; | |
1280 | int N0, N1, N2; | |
1281 | int D0, D1, D2; | |
1282 | UFSSwapDir *SD; | |
1283 | N0 = NumberOfUFSDirs; | |
1284 | D0 = UFSDirToGlobalDirMapping[swap_index % N0]; | |
1285 | SD = dynamic_cast<UFSSwapDir *>(INDEXSD(D0)); | |
1286 | assert (SD); | |
1287 | N1 = SD->l1; | |
1288 | D1 = (swap_index / N0) % N1; | |
1289 | N2 = SD->l2; | |
1290 | D2 = ((swap_index / N0) / N1) % N2; | |
1291 | snprintf(p1, MAXPATHLEN, "%s/%02X/%02X", | |
1292 | SD->path, D1, D2); | |
1293 | debugs(36, 3, HERE << "Cleaning directory " << p1); | |
1294 | dir_pointer = opendir(p1); | |
c8f4eac4 | 1295 | |
58373ff8 FC |
1296 | if (dir_pointer == NULL) { |
1297 | if (errno == ENOENT) { | |
1298 | debugs(36, DBG_CRITICAL, HERE << "WARNING: Creating " << p1); | |
1299 | #if _SQUID_MSWIN_ | |
c8f4eac4 | 1300 | |
58373ff8 FC |
1301 | if (mkdir(p1) == 0) |
1302 | #else | |
c8f4eac4 | 1303 | |
58373ff8 FC |
1304 | if (mkdir(p1, 0777) == 0) |
1305 | #endif | |
c8f4eac4 | 1306 | |
58373ff8 FC |
1307 | return 0; |
1308 | } | |
1309 | ||
1310 | debugs(50, DBG_CRITICAL, HERE << p1 << ": " << xstrerror()); | |
1311 | safeunlink(p1, 1); | |
1312 | return 0; | |
1313 | } | |
1314 | ||
1315 | dirent_t *de; | |
1316 | while ((de = readdir(dir_pointer)) != NULL && k < 20) { | |
1317 | if (sscanf(de->d_name, "%X", &swapfileno) != 1) | |
1318 | continue; | |
1319 | ||
1320 | fn = swapfileno; /* XXX should remove this cruft ! */ | |
1321 | ||
1322 | if (SD->validFileno(fn, 1)) | |
1323 | if (SD->mapBitTest(fn)) | |
1324 | if (UFSSwapDir::FilenoBelongsHere(fn, D0, D1, D2)) | |
1325 | continue; | |
1326 | ||
1327 | files[k] = swapfileno; | |
1328 | ++k; | |
1329 | } | |
1330 | ||
1331 | closedir(dir_pointer); | |
1332 | ||
1333 | if (k == 0) | |
1334 | return 0; | |
1335 | ||
1336 | qsort(files, k, sizeof(int), rev_int_sort); | |
1337 | ||
1338 | if (k > 10) | |
1339 | k = 10; | |
1340 | ||
1341 | for (n = 0; n < k; ++n) { | |
1342 | debugs(36, 3, HERE << "Cleaning file "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << files[n]); | |
1343 | snprintf(p2, MAXPATHLEN + 1, "%s/%08X", p1, files[n]); | |
1344 | safeunlink(p2, 0); | |
1345 | ++statCounter.swap.files_cleaned; | |
1346 | } | |
1347 | ||
1348 | debugs(36, 3, HERE << "Cleaned " << k << " unused files from " << p1); | |
1349 | return k; | |
c8f4eac4 | 1350 | } |