]>
Commit | Line | Data |
---|---|---|
4d6d905e | 1 | /* |
c3031d67 | 2 | * $Id: ufscommon.cc,v 1.15 2007/11/15 16:47:38 wessels Exp $ |
c8f4eac4 | 3 | * vim: set et : |
4d6d905e | 4 | * |
5 | * DEBUG: section 47 Store Directory Routines | |
51ee7c82 | 6 | * AUTHOR: Robert Collins |
4d6d905e | 7 | * |
8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ | |
9 | * ---------------------------------------------------------- | |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from | |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
19 | * | |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
24 | * | |
25 | * This program is distributed in the hope that it will be useful, | |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
29 | * | |
30 | * You should have received a copy of the GNU General Public License | |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
51ee7c82 | 34 | * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org> |
4d6d905e | 35 | */ |
36 | ||
37 | #include "ufscommon.h" | |
e6ccf245 | 38 | #include "Store.h" |
528b2c61 | 39 | #include "fde.h" |
40 | #include "StoreMeta.h" | |
41 | #include "Generic.h" | |
42 | #include "StoreMetaUnpacker.h" | |
d3b3ab85 | 43 | #include "RefCount.h" |
51ee7c82 | 44 | #include "StoreSwapLogData.h" |
4d6d905e | 45 | |
d3b3ab85 | 46 | CBDATA_CLASS_INIT(RebuildState); |
4d6d905e | 47 | |
47f6e231 | 48 | |
49 | class UFSSwapLogParser_old:public UFSSwapLogParser{ | |
50 | public: | |
51 | struct StoreSwapLogDataOld{ | |
52 | char op; | |
53 | sfileno swap_filen; | |
54 | time_t timestamp; | |
55 | time_t lastref; | |
56 | time_t expires; | |
57 | time_t lastmod; | |
58 | size_t swap_file_sz; | |
59 | u_short refcount; | |
60 | u_short flags; | |
c3031d67 | 61 | unsigned char key[SQUID_MD5_DIGEST_LENGTH]; |
47f6e231 | 62 | }; |
63 | UFSSwapLogParser_old(FILE *fp):UFSSwapLogParser(fp) | |
64 | { | |
65 | record_size = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); | |
66 | } | |
67 | bool ReadRecord(StoreSwapLogData &swapData); | |
68 | }; | |
69 | ||
70 | ||
71 | bool UFSSwapLogParser_old::ReadRecord(StoreSwapLogData &swapData){ | |
72 | UFSSwapLogParser_old::StoreSwapLogDataOld readData; | |
73 | int bytes = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); | |
74 | ||
75 | assert(log); | |
76 | ||
77 | if (fread(&readData, bytes, 1, log) != 1){ | |
78 | return false; | |
79 | } | |
80 | swapData.op = readData.op; | |
81 | swapData.swap_filen = readData.swap_filen; | |
82 | swapData.timestamp = readData.timestamp; | |
83 | swapData.lastref = readData.lastref; | |
84 | swapData.expires = readData.expires; | |
85 | swapData.lastmod = readData.lastmod; | |
86 | swapData.swap_file_sz = readData.swap_file_sz; | |
87 | swapData.refcount = readData.refcount; | |
88 | swapData.flags = readData.flags; | |
c3031d67 | 89 | xmemcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); |
47f6e231 | 90 | return true; |
91 | } | |
92 | ||
93 | ||
94 | class UFSSwapLogParser_v1:public UFSSwapLogParser{ | |
95 | public: | |
96 | UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) | |
97 | { | |
98 | record_size = sizeof(StoreSwapLogData); | |
99 | } | |
100 | bool ReadRecord(StoreSwapLogData &swapData); | |
101 | }; | |
102 | ||
103 | ||
104 | bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData) | |
105 | { | |
106 | int bytes = sizeof(StoreSwapLogData); | |
107 | ||
108 | assert(log); | |
109 | ||
110 | if (fread(&swapData, bytes, 1, log) != 1){ | |
111 | return false; | |
112 | } | |
113 | return true; | |
114 | } | |
115 | ||
116 | ||
117 | UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) | |
118 | { | |
119 | StoreSwapLogHeader header; | |
120 | ||
121 | assert(fp); | |
122 | ||
123 | if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) | |
124 | return NULL; | |
125 | ||
126 | if (header.op != SWAP_LOG_VERSION){ | |
127 | debugs(47, 1, "Old swap file detected... "); | |
128 | fseek(fp, 0, SEEK_SET); | |
129 | return new UFSSwapLogParser_old(fp); | |
130 | } | |
131 | ||
132 | if (header.version == 1){ | |
133 | if (fseek(fp, header.record_size, SEEK_SET) != 0) | |
134 | return NULL; | |
135 | ||
136 | if (header.record_size == sizeof(struct UFSSwapLogParser_old::StoreSwapLogDataOld)){ | |
137 | debugs(47, 1, "Version 1 of swap file without LFS support detected... "); | |
138 | return new UFSSwapLogParser_old(fp); | |
139 | } | |
140 | ||
141 | if (header.record_size == sizeof(StoreSwapLogData)){ | |
142 | debugs(47, 1, "Version 1 of swap file with LFS support detected... "); | |
143 | return new UFSSwapLogParser_v1(fp); | |
144 | } | |
145 | ||
146 | debugs(47, 1, "The swap file has wrong format!... "); | |
147 | return NULL; | |
148 | } | |
149 | ||
150 | return NULL; | |
151 | } | |
152 | ||
153 | int UFSSwapLogParser::SwapLogEntries(){ | |
154 | struct stat sb; | |
155 | ||
156 | if (log_entries >= 0) | |
157 | return log_entries; | |
158 | ||
159 | if (log && record_size && 0 == fstat(fileno(log), &sb)){ | |
160 | log_entries = sb.st_size/record_size; | |
161 | return log_entries; | |
162 | } | |
163 | ||
164 | return 0; | |
165 | } | |
166 | ||
167 | ||
168 | ||
169 | ||
170 | RebuildState::RebuildState (RefCount<UFSSwapDir> aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false) | |
4d6d905e | 171 | { |
c8f4eac4 | 172 | speed = opt_foreground_rebuild ? 1 << 30 : 50; |
173 | /* | |
174 | * If the swap.state file exists in the cache_dir, then | |
175 | * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll | |
176 | * use commonUfsDirRebuildFromDirectory() to open up each file | |
177 | * and suck in the meta data. | |
178 | */ | |
179 | int clean = 0; | |
180 | int zeroLengthLog = 0; | |
181 | FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); | |
182 | ||
47f6e231 | 183 | if (fp && !zeroLengthLog) |
184 | LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp); | |
185 | ||
186 | if (LogParser == NULL ) { | |
c8f4eac4 | 187 | fromLog = false; |
188 | ||
189 | if (fp != NULL) | |
190 | fclose(fp); | |
191 | ||
192 | } else { | |
47f6e231 | 193 | fromLog = true; |
194 | flags.clean = (unsigned int) clean; | |
c8f4eac4 | 195 | } |
62e76326 | 196 | |
c8f4eac4 | 197 | if (!clean) |
198 | flags.need_to_validate = 1; | |
199 | ||
bf8fe701 | 200 | debugs(47, 1, "Rebuilding storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")"); |
4d6d905e | 201 | } |
202 | ||
d3b3ab85 | 203 | RebuildState::~RebuildState() |
4d6d905e | 204 | { |
d3b3ab85 | 205 | sd->closeTmpSwapLog(); |
47f6e231 | 206 | |
207 | if (LogParser) | |
208 | delete LogParser; | |
4d6d905e | 209 | } |
210 | ||
211 | void | |
c8f4eac4 | 212 | RebuildState::RebuildStep(void *data) |
4d6d905e | 213 | { |
d3b3ab85 | 214 | RebuildState *rb = (RebuildState *)data; |
c8f4eac4 | 215 | rb->rebuildStep(); |
216 | ||
217 | if (!rb->isDone()) | |
218 | eventAdd("storeRebuild", RebuildStep, rb, 0.0, 1); | |
219 | else { | |
bef81ea5 | 220 | StoreController::store_dirs_rebuilding--; |
c8f4eac4 | 221 | storeRebuildComplete(&rb->counts); |
222 | delete rb; | |
223 | } | |
224 | } | |
225 | ||
226 | void | |
227 | RebuildState::rebuildStep() | |
228 | { | |
229 | if (fromLog) | |
230 | rebuildFromSwapLog(); | |
231 | else | |
232 | rebuildFromDirectory(); | |
4d6d905e | 233 | } |
234 | ||
528b2c61 | 235 | struct InitStoreEntry : public unary_function<StoreMeta, void> |
236 | { | |
237 | InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey){} | |
62e76326 | 238 | |
239 | void operator()(StoreMeta const &x) | |
240 | { | |
241 | switch (x.getType()) { | |
242 | ||
243 | case STORE_META_KEY: | |
c3031d67 | 244 | assert(x.length == SQUID_MD5_DIGEST_LENGTH); |
245 | xmemcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH); | |
62e76326 | 246 | break; |
247 | ||
248 | case STORE_META_STD: | |
47f6e231 | 249 | struct old_metahdr{ |
250 | time_t timestamp; | |
251 | time_t lastref; | |
252 | time_t expires; | |
253 | time_t lastmod; | |
254 | size_t swap_file_sz; | |
255 | u_short refcount; | |
256 | u_short flags; | |
257 | } *tmp; | |
258 | tmp = (struct old_metahdr *)x.value; | |
259 | assert(x.length == STORE_HDR_METASIZE_OLD); | |
260 | what->timestamp = tmp->timestamp; | |
261 | what->lastref = tmp->lastref; | |
262 | what->expires = tmp->expires; | |
263 | what->lastmod = tmp->lastmod; | |
264 | what->swap_file_sz = tmp->swap_file_sz; | |
265 | what->refcount = tmp->refcount; | |
266 | what->flags = tmp->flags; | |
267 | break; | |
268 | ||
269 | case STORE_META_STD_LFS: | |
270 | assert(x.length == STORE_HDR_METASIZE); | |
62e76326 | 271 | xmemcpy(&what->timestamp, x.value, STORE_HDR_METASIZE); |
272 | break; | |
273 | ||
274 | default: | |
275 | break; | |
276 | } | |
528b2c61 | 277 | } |
62e76326 | 278 | |
528b2c61 | 279 | StoreEntry *what; |
280 | cache_key *index; | |
281 | }; | |
282 | ||
4d6d905e | 283 | void |
d3b3ab85 | 284 | RebuildState::rebuildFromDirectory() |
4d6d905e | 285 | { |
4d6d905e | 286 | LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); |
822b78b5 | 287 | currentEntry(NULL); |
c3031d67 | 288 | cache_key key[SQUID_MD5_DIGEST_LENGTH]; |
62e76326 | 289 | |
4d6d905e | 290 | struct stat sb; |
291 | int swap_hdr_len; | |
292 | int fd = -1; | |
528b2c61 | 293 | StoreMeta *tlv_list; |
d3b3ab85 | 294 | assert(this != NULL); |
bf8fe701 | 295 | debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index); |
62e76326 | 296 | |
d3b3ab85 | 297 | for (int count = 0; count < speed; count++) { |
62e76326 | 298 | assert(fd == -1); |
299 | sfileno filn = 0; | |
300 | int size; | |
301 | fd = getNextFile(&filn, &size); | |
302 | ||
303 | if (fd == -2) { | |
bf8fe701 | 304 | debugs(47, 1, "Done scanning " << sd->path << " swaplog (" << n_read << " entries)"); |
c8f4eac4 | 305 | _done = true; |
62e76326 | 306 | return; |
307 | } else if (fd < 0) { | |
308 | continue; | |
309 | } | |
310 | ||
311 | assert(fd > -1); | |
312 | /* lets get file stats here */ | |
313 | ||
314 | if (fstat(fd, &sb) < 0) { | |
bf8fe701 | 315 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror()); |
62e76326 | 316 | file_close(fd); |
317 | store_open_disk_fd--; | |
318 | fd = -1; | |
319 | continue; | |
320 | } | |
321 | ||
322 | if ((++counts.scancount & 0xFFFF) == 0) | |
bf8fe701 | 323 | debugs(47, 3, " " << sd->path << " " << std::setw(7) << counts.scancount << " files opened so far."); |
324 | debugs(47, 9, "file_in: fd=" << fd << " "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn); | |
62e76326 | 325 | |
62e76326 | 326 | |
327 | statCounter.syscalls.disk.reads++; | |
328 | ||
329 | int len; | |
330 | ||
331 | if ((len = FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE)) < 0) { | |
bf8fe701 | 332 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: read(FD " << fd << "): " << xstrerror()); |
62e76326 | 333 | file_close(fd); |
334 | store_open_disk_fd--; | |
335 | fd = -1; | |
336 | continue; | |
337 | } | |
338 | ||
339 | file_close(fd); | |
340 | store_open_disk_fd--; | |
341 | fd = -1; | |
342 | swap_hdr_len = 0; | |
62e76326 | 343 | |
528b2c61 | 344 | StoreMetaUnpacker aBuilder(hdr_buf, len, &swap_hdr_len); |
62e76326 | 345 | |
346 | if (!aBuilder.isBufferSane()) { | |
bf8fe701 | 347 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: Swap data buffer length is not sane."); |
62e76326 | 348 | /* XXX shouldn't this be a call to commonUfsUnlink ? */ |
349 | sd->unlinkFile ( filn); | |
350 | continue; | |
351 | } | |
352 | ||
528b2c61 | 353 | tlv_list = aBuilder.createStoreMeta (); |
62e76326 | 354 | |
355 | if (tlv_list == NULL) { | |
bf8fe701 | 356 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: failed to get meta data"); |
62e76326 | 357 | /* XXX shouldn't this be a call to commonUfsUnlink ? */ |
358 | sd->unlinkFile (filn); | |
359 | continue; | |
360 | } | |
361 | ||
bf8fe701 | 362 | debugs(47, 3, "commonUfsDirRebuildFromDirectory: successful swap meta unpacking"); |
c3031d67 | 363 | memset(key, '\0', SQUID_MD5_DIGEST_LENGTH); |
f894387a | 364 | |
365 | StoreEntry tmpe; | |
62e76326 | 366 | InitStoreEntry visitor(&tmpe, key); |
367 | for_each(*tlv_list, visitor); | |
368 | storeSwapTLVFree(tlv_list); | |
369 | tlv_list = NULL; | |
370 | ||
371 | if (storeKeyNull(key)) { | |
bf8fe701 | 372 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: NULL key"); |
62e76326 | 373 | sd->unlinkFile(filn); |
374 | continue; | |
375 | } | |
376 | ||
377 | tmpe.key = key; | |
378 | /* check sizes */ | |
379 | ||
380 | if (tmpe.swap_file_sz == 0) { | |
47f6e231 | 381 | tmpe.swap_file_sz = (uint64_t) sb.st_size; |
382 | } else if (tmpe.swap_file_sz == (uint64_t)(sb.st_size - swap_hdr_len)) { | |
383 | tmpe.swap_file_sz = (uint64_t) sb.st_size; | |
384 | } else if (tmpe.swap_file_sz != (uint64_t)sb.st_size) { | |
bf8fe701 | 385 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: SIZE MISMATCH " << |
47f6e231 | 386 | tmpe.swap_file_sz << "!=" << |
387 | sb.st_size); | |
bf8fe701 | 388 | |
62e76326 | 389 | sd->unlinkFile(filn); |
390 | continue; | |
391 | } | |
392 | ||
393 | if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { | |
394 | sd->unlinkFile(filn); | |
395 | counts.badflags++; | |
396 | continue; | |
397 | } | |
398 | ||
c8f4eac4 | 399 | /* this needs to become |
400 | * 1) unpack url | |
401 | * 2) make synthetic request with headers ?? or otherwise search | |
402 | * for a matching object in the store | |
403 | * TODO FIXME change to new async api | |
404 | * TODO FIXME I think there is a race condition here with the | |
405 | * async api : | |
406 | * store A reads in object foo, searchs for it, and finds nothing. | |
407 | * store B reads in object foo, searchs for it, finds nothing. | |
408 | * store A gets called back with nothing, so registers the object | |
409 | * store B gets called back with nothing, so registers the object, | |
410 | * which will conflict when the in core index gets around to scanning | |
411 | * store B. | |
412 | * | |
413 | * this suggests that rather than searching for duplicates, the | |
414 | * index rebuild should just assume its the most recent accurate | |
415 | * store entry and whoever indexes the stores handles duplicates. | |
416 | */ | |
417 | e = Store::Root().get(key); | |
62e76326 | 418 | |
419 | if (e && e->lastref >= tmpe.lastref) { | |
420 | /* key already exists, current entry is newer */ | |
421 | /* keep old, ignore new */ | |
422 | counts.dupcount++; | |
423 | continue; | |
424 | } else if (NULL != e) { | |
425 | /* URL already exists, this swapfile not being used */ | |
426 | /* junk old, load new */ | |
5f33b71d | 427 | e->release(); /* release old entry */ |
62e76326 | 428 | counts.dupcount++; |
429 | } | |
430 | ||
431 | counts.objcount++; | |
47f6e231 | 432 | // tmpe.dump(5); |
c8f4eac4 | 433 | currentEntry(sd->addDiskRestore(key, |
434 | filn, | |
435 | tmpe.swap_file_sz, | |
436 | tmpe.expires, | |
437 | tmpe.timestamp, | |
438 | tmpe.lastref, | |
439 | tmpe.lastmod, | |
440 | tmpe.refcount, /* refcount */ | |
441 | tmpe.flags, /* flags */ | |
442 | (int) flags.clean)); | |
443 | storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); | |
4d6d905e | 444 | } |
62e76326 | 445 | |
d3b3ab85 | 446 | } |
447 | ||
822b78b5 | 448 | StoreEntry * |
449 | RebuildState::currentEntry() const | |
450 | { | |
451 | return e; | |
452 | } | |
453 | ||
454 | void | |
455 | RebuildState::currentEntry(StoreEntry *newValue) | |
456 | { | |
457 | e = newValue; | |
458 | } | |
459 | ||
d3b3ab85 | 460 | void |
461 | RebuildState::rebuildFromSwapLog() | |
462 | { | |
822b78b5 | 463 | currentEntry (NULL); |
4d6d905e | 464 | double x; |
4d6d905e | 465 | /* load a number of objects per invocation */ |
62e76326 | 466 | |
d3b3ab85 | 467 | for (int count = 0; count < speed; count++) { |
c8f4eac4 | 468 | StoreSwapLogData swapData; |
62e76326 | 469 | |
47f6e231 | 470 | if (LogParser->ReadRecord(swapData) != 1) { |
bf8fe701 | 471 | debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); |
47f6e231 | 472 | LogParser->Close(); |
473 | delete LogParser; | |
474 | LogParser = NULL; | |
c8f4eac4 | 475 | _done = true; |
62e76326 | 476 | return; |
477 | } | |
478 | ||
479 | n_read++; | |
480 | ||
c8f4eac4 | 481 | if (swapData.op <= SWAP_LOG_NOP) |
62e76326 | 482 | continue; |
483 | ||
c8f4eac4 | 484 | if (swapData.op >= SWAP_LOG_MAX) |
62e76326 | 485 | continue; |
486 | ||
487 | /* | |
488 | * BC: during 2.4 development, we changed the way swap file | |
489 | * numbers are assigned and stored. The high 16 bits used | |
490 | * to encode the SD index number. There used to be a call | |
491 | * to storeDirProperFileno here that re-assigned the index | |
492 | * bits. Now, for backwards compatibility, we just need | |
493 | * to mask it off. | |
494 | */ | |
c8f4eac4 | 495 | swapData.swap_filen &= 0x00FFFFFF; |
62e76326 | 496 | |
bf8fe701 | 497 | debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " << |
498 | swap_log_op_str[(int) swapData.op] << " " << | |
499 | storeKeyText(swapData.key) << " "<< std::setfill('0') << | |
500 | std::hex << std::uppercase << std::setw(8) << | |
501 | swapData.swap_filen); | |
62e76326 | 502 | |
c8f4eac4 | 503 | if (swapData.op == SWAP_LOG_ADD) { |
62e76326 | 504 | (void) 0; |
c8f4eac4 | 505 | } else if (swapData.op == SWAP_LOG_DEL) { |
506 | /* Delete unless we already have a newer copy anywhere in any store */ | |
507 | /* this needs to become | |
508 | * 1) unpack url | |
509 | * 2) make synthetic request with headers ?? or otherwise search | |
510 | * for a matching object in the store | |
511 | * TODO FIXME change to new async api | |
512 | */ | |
513 | currentEntry (Store::Root().get(swapData.key)); | |
514 | ||
77fde53f | 515 | if (currentEntry() != NULL && swapData.lastref >= e->lastref) { |
62e76326 | 516 | /* |
517 | * Make sure we don't unlink the file, it might be | |
518 | * in use by a subsequent entry. Also note that | |
519 | * we don't have to subtract from store_swap_size | |
520 | * because adding to store_swap_size happens in | |
521 | * the cleanup procedure. | |
522 | */ | |
d88e3c49 | 523 | currentEntry()->expireNow(); |
524 | currentEntry()->releaseRequest(); | |
62e76326 | 525 | |
526 | if (currentEntry()->swap_filen > -1) { | |
527 | UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn)); | |
528 | assert (sdForThisEntry); | |
529 | sdForThisEntry->replacementRemove(currentEntry()); | |
530 | sdForThisEntry->mapBitReset(currentEntry()->swap_filen); | |
531 | currentEntry()->swap_filen = -1; | |
532 | currentEntry()->swap_dirn = -1; | |
533 | } | |
534 | ||
5f33b71d | 535 | currentEntry()->release(); |
62e76326 | 536 | counts.objcount--; |
537 | counts.cancelcount++; | |
2fadd50d | 538 | } continue; |
62e76326 | 539 | } else { |
411c6ea3 | 540 | x = ::log(static_cast<double>(++counts.bad_log_op)) / ::log(10.0); |
62e76326 | 541 | |
542 | if (0.0 == x - (double) (int) x) | |
bf8fe701 | 543 | debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); |
62e76326 | 544 | |
545 | counts.invalid++; | |
546 | ||
547 | continue; | |
548 | } | |
549 | ||
550 | if ((++counts.scancount & 0xFFF) == 0) { | |
551 | ||
47f6e231 | 552 | int swap_entries = LogParser->SwapLogEntries(); |
62e76326 | 553 | |
47f6e231 | 554 | if (0 != swap_entries ) |
62e76326 | 555 | storeRebuildProgress(sd->index, |
47f6e231 | 556 | swap_entries, n_read); |
62e76326 | 557 | } |
558 | ||
c8f4eac4 | 559 | if (!sd->validFileno(swapData.swap_filen, 0)) { |
62e76326 | 560 | counts.invalid++; |
561 | continue; | |
562 | } | |
563 | ||
c8f4eac4 | 564 | if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) { |
62e76326 | 565 | counts.badflags++; |
566 | continue; | |
567 | } | |
568 | ||
c8f4eac4 | 569 | /* this needs to become |
570 | * 1) unpack url | |
571 | * 2) make synthetic request with headers ?? or otherwise search | |
572 | * for a matching object in the store | |
573 | * TODO FIXME change to new async api | |
574 | */ | |
575 | currentEntry (Store::Root().get(swapData.key)); | |
576 | ||
d3b3ab85 | 577 | int used; /* is swapfile already in use? */ |
c8f4eac4 | 578 | |
579 | used = sd->mapBitTest(swapData.swap_filen); | |
580 | ||
62e76326 | 581 | /* If this URL already exists in the cache, does the swap log |
582 | * appear to have a newer entry? Compare 'lastref' from the | |
583 | * swap log to e->lastref. */ | |
584 | /* is the log entry newer than current entry? */ | |
c8f4eac4 | 585 | int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; |
62e76326 | 586 | |
587 | if (used && !disk_entry_newer) { | |
588 | /* log entry is old, ignore it */ | |
589 | counts.clashcount++; | |
590 | continue; | |
c8f4eac4 | 591 | } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { |
62e76326 | 592 | /* swapfile taken, same URL, newer, update meta */ |
593 | ||
594 | if (currentEntry()->store_status == STORE_OK) { | |
c8f4eac4 | 595 | currentEntry()->lastref = swapData.timestamp; |
596 | currentEntry()->timestamp = swapData.timestamp; | |
597 | currentEntry()->expires = swapData.expires; | |
598 | currentEntry()->lastmod = swapData.lastmod; | |
599 | currentEntry()->flags = swapData.flags; | |
600 | currentEntry()->refcount += swapData.refcount; | |
62e76326 | 601 | sd->dereference(*currentEntry()); |
602 | } else { | |
603 | debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); | |
bf8fe701 | 604 | debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__); |
2fadd50d | 605 | } continue; |
62e76326 | 606 | } else if (used) { |
607 | /* swapfile in use, not by this URL, log entry is newer */ | |
608 | /* This is sorta bad: the log entry should NOT be newer at this | |
609 | * point. If the log is dirty, the filesize check should have | |
610 | * caught this. If the log is clean, there should never be a | |
611 | * newer entry. */ | |
bf8fe701 | 612 | debugs(47, 1, "WARNING: newer swaplog entry for dirno " << |
613 | sd->index << ", fileno "<< std::setfill('0') << std::hex << | |
614 | std::uppercase << std::setw(8) << swapData.swap_filen); | |
615 | ||
62e76326 | 616 | /* I'm tempted to remove the swapfile here just to be safe, |
617 | * but there is a bad race condition in the NOVM version if | |
618 | * the swapfile has recently been opened for writing, but | |
619 | * not yet opened for reading. Because we can't map | |
620 | * swapfiles back to StoreEntrys, we don't know the state | |
621 | * of the entry using that file. */ | |
622 | /* We'll assume the existing entry is valid, probably because | |
623 | * were in a slow rebuild and the the swap file number got taken | |
624 | * and the validation procedure hasn't run. */ | |
625 | assert(flags.need_to_validate); | |
626 | counts.clashcount++; | |
627 | continue; | |
628 | } else if (currentEntry() && !disk_entry_newer) { | |
629 | /* key already exists, current entry is newer */ | |
630 | /* keep old, ignore new */ | |
631 | counts.dupcount++; | |
632 | continue; | |
633 | } else if (currentEntry()) { | |
634 | /* key already exists, this swapfile not being used */ | |
635 | /* junk old, load new */ | |
d88e3c49 | 636 | currentEntry()->expireNow(); |
637 | currentEntry()->releaseRequest(); | |
62e76326 | 638 | |
639 | if (currentEntry()->swap_filen > -1) { | |
640 | UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn)); | |
641 | sdForThisEntry->replacementRemove(currentEntry()); | |
642 | /* Make sure we don't actually unlink the file */ | |
643 | sdForThisEntry->mapBitReset(currentEntry()->swap_filen); | |
644 | currentEntry()->swap_filen = -1; | |
645 | currentEntry()->swap_dirn = -1; | |
646 | } | |
647 | ||
5f33b71d | 648 | currentEntry()->release(); |
62e76326 | 649 | counts.dupcount++; |
650 | } else { | |
651 | /* URL doesnt exist, swapfile not in use */ | |
652 | /* load new */ | |
653 | (void) 0; | |
654 | } | |
655 | ||
656 | /* update store_swap_size */ | |
657 | counts.objcount++; | |
658 | ||
c8f4eac4 | 659 | currentEntry(sd->addDiskRestore(swapData.key, |
660 | swapData.swap_filen, | |
661 | swapData.swap_file_sz, | |
662 | swapData.expires, | |
663 | swapData.timestamp, | |
664 | swapData.lastref, | |
665 | swapData.lastmod, | |
666 | swapData.refcount, | |
667 | swapData.flags, | |
62e76326 | 668 | (int) flags.clean)); |
669 | ||
670 | storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); | |
4d6d905e | 671 | } |
62e76326 | 672 | |
4d6d905e | 673 | } |
674 | ||
675 | int | |
d3b3ab85 | 676 | RebuildState::getNextFile(sfileno * filn_p, int *size) |
4d6d905e | 677 | { |
4d6d905e | 678 | int fd = -1; |
4d6d905e | 679 | int dirs_opened = 0; |
bf8fe701 | 680 | debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init << ", " << |
681 | sd->index << ": /"<< std::setfill('0') << std::hex << | |
682 | std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) << | |
683 | curlvl2); | |
62e76326 | 684 | |
d3b3ab85 | 685 | if (done) |
62e76326 | 686 | return -2; |
687 | ||
d3b3ab85 | 688 | while (fd < 0 && done == 0) { |
62e76326 | 689 | fd = -1; |
690 | ||
691 | if (0 == flags.init) { /* initialize, open first file */ | |
692 | done = 0; | |
693 | curlvl1 = 0; | |
694 | curlvl2 = 0; | |
695 | in_dir = 0; | |
696 | flags.init = 1; | |
697 | assert(Config.cacheSwap.n_configured > 0); | |
698 | } | |
699 | ||
700 | if (0 == in_dir) { /* we need to read in a new directory */ | |
701 | snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", | |
702 | sd->path, | |
703 | curlvl1, curlvl2); | |
704 | ||
705 | if (dirs_opened) | |
706 | return -1; | |
707 | ||
708 | td = opendir(fullpath); | |
709 | ||
710 | dirs_opened++; | |
711 | ||
712 | if (td == NULL) { | |
bf8fe701 | 713 | debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror()); |
62e76326 | 714 | } else { |
715 | entry = readdir(td); /* skip . and .. */ | |
716 | entry = readdir(td); | |
717 | ||
718 | if (entry == NULL && errno == ENOENT) | |
bf8fe701 | 719 | debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!."); |
720 | debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath); | |
62e76326 | 721 | } |
722 | } | |
723 | ||
724 | if (td != NULL && (entry = readdir(td)) != NULL) { | |
725 | in_dir++; | |
726 | ||
727 | if (sscanf(entry->d_name, "%x", &fn) != 1) { | |
bf8fe701 | 728 | debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name); |
62e76326 | 729 | continue; |
730 | } | |
731 | ||
732 | if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) { | |
bf8fe701 | 733 | debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') << |
734 | std::hex << std::uppercase << std::setw(8) << fn << | |
735 | " does not belong in " << std::dec << sd->index << "/" << | |
736 | curlvl1 << "/" << curlvl2); | |
737 | ||
62e76326 | 738 | continue; |
739 | } | |
740 | ||
741 | if (sd->mapBitTest(fn)) { | |
bf8fe701 | 742 | debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next."); |
62e76326 | 743 | continue; |
744 | } | |
745 | ||
746 | snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", | |
747 | fullpath, entry->d_name); | |
bf8fe701 | 748 | debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename); |
62e76326 | 749 | fd = file_open(fullfilename, O_RDONLY | O_BINARY); |
750 | ||
751 | if (fd < 0) | |
bf8fe701 | 752 | debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror()); |
62e76326 | 753 | else |
754 | store_open_disk_fd++; | |
755 | ||
756 | continue; | |
757 | } | |
758 | ||
759 | if (td != NULL) | |
760 | closedir(td); | |
761 | ||
762 | td = NULL; | |
763 | ||
764 | in_dir = 0; | |
765 | ||
766 | if (sd->validL2(++curlvl2)) | |
767 | continue; | |
768 | ||
769 | curlvl2 = 0; | |
770 | ||
771 | if (sd->validL1(++curlvl1)) | |
772 | continue; | |
773 | ||
774 | curlvl1 = 0; | |
775 | ||
776 | done = 1; | |
4d6d905e | 777 | } |
62e76326 | 778 | |
d3b3ab85 | 779 | *filn_p = fn; |
4d6d905e | 780 | return fd; |
781 | } | |
528b2c61 | 782 | |
c8f4eac4 | 783 | void |
784 | RebuildState::next(void (callback)(void *cbdata), void *cbdata) | |
785 | { | |
786 | /* for now, we don't cache at all */ | |
787 | speed = 1; | |
788 | currentEntry(NULL); | |
789 | ||
790 | while (!isDone() && currentEntry() == NULL) | |
791 | rebuildStep(); | |
792 | ||
793 | callback(cbdata); | |
794 | } | |
795 | ||
796 | bool | |
797 | RebuildState::next() | |
798 | { | |
799 | return false; | |
800 | } | |
801 | ||
802 | bool | |
803 | RebuildState::error() const | |
804 | { | |
805 | return false; | |
806 | } | |
807 | ||
808 | bool | |
809 | RebuildState::isDone() const | |
810 | { | |
811 | return _done; | |
812 | } | |
813 | ||
814 | StoreEntry * | |
815 | RebuildState::currentItem() | |
816 | { | |
817 | return currentEntry(); | |
818 | } | |
819 | ||
528b2c61 | 820 | #ifndef _USE_INLINE_ |
821 | #include "ufscommon.cci" | |
822 | #endif |