]>
Commit | Line | Data |
---|---|---|
4d6d905e | 1 | /* |
262a0e14 | 2 | * $Id$ |
26ac0430 | 3 | * vim: set et : |
4d6d905e | 4 | * |
5 | * DEBUG: section 47 Store Directory Routines | |
51ee7c82 | 6 | * AUTHOR: Robert Collins |
4d6d905e | 7 | * |
8 | * SQUID Web Proxy Cache http://www.squid-cache.org/ | |
9 | * ---------------------------------------------------------- | |
10 | * | |
11 | * Squid is the result of efforts by numerous individuals from | |
12 | * the Internet community; see the CONTRIBUTORS file for full | |
13 | * details. Many organizations have provided support for Squid's | |
14 | * development; see the SPONSORS file for full details. Squid is | |
15 | * Copyrighted (C) 2001 by the Regents of the University of | |
16 | * California; see the COPYRIGHT file for full details. Squid | |
17 | * incorporates software developed and/or copyrighted by other | |
18 | * sources; see the CREDITS file for full details. | |
19 | * | |
20 | * This program is free software; you can redistribute it and/or modify | |
21 | * it under the terms of the GNU General Public License as published by | |
22 | * the Free Software Foundation; either version 2 of the License, or | |
23 | * (at your option) any later version. | |
26ac0430 | 24 | * |
4d6d905e | 25 | * This program is distributed in the hope that it will be useful, |
26 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
27 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
28 | * GNU General Public License for more details. | |
26ac0430 | 29 | * |
4d6d905e | 30 | * You should have received a copy of the GNU General Public License |
31 | * along with this program; if not, write to the Free Software | |
32 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. | |
33 | * | |
51ee7c82 | 34 | * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org> |
4d6d905e | 35 | */ |
36 | ||
37 | #include "ufscommon.h" | |
e6ccf245 | 38 | #include "Store.h" |
528b2c61 | 39 | #include "fde.h" |
40 | #include "StoreMeta.h" | |
41 | #include "Generic.h" | |
42 | #include "StoreMetaUnpacker.h" | |
d3b3ab85 | 43 | #include "RefCount.h" |
51ee7c82 | 44 | #include "StoreSwapLogData.h" |
4d6d905e | 45 | |
d3b3ab85 | 46 | CBDATA_CLASS_INIT(RebuildState); |
4d6d905e | 47 | |
47f6e231 | 48 | |
26ac0430 AJ |
49 | class UFSSwapLogParser_old:public UFSSwapLogParser |
50 | { | |
47f6e231 | 51 | public: |
26ac0430 AJ |
52 | struct StoreSwapLogDataOld { |
53 | char op; | |
54 | sfileno swap_filen; | |
55 | time_t timestamp; | |
56 | time_t lastref; | |
57 | time_t expires; | |
58 | time_t lastmod; | |
59 | size_t swap_file_sz; | |
60 | u_short refcount; | |
61 | u_short flags; | |
62 | unsigned char key[SQUID_MD5_DIGEST_LENGTH]; | |
47f6e231 | 63 | }; |
26ac0430 AJ |
64 | UFSSwapLogParser_old(FILE *fp):UFSSwapLogParser(fp) { |
65 | record_size = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); | |
47f6e231 | 66 | } |
67 | bool ReadRecord(StoreSwapLogData &swapData); | |
68 | }; | |
69 | ||
70 | ||
26ac0430 AJ |
71 | bool UFSSwapLogParser_old::ReadRecord(StoreSwapLogData &swapData) |
72 | { | |
47f6e231 | 73 | UFSSwapLogParser_old::StoreSwapLogDataOld readData; |
74 | int bytes = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld); | |
75 | ||
76 | assert(log); | |
77 | ||
26ac0430 AJ |
78 | if (fread(&readData, bytes, 1, log) != 1) { |
79 | return false; | |
47f6e231 | 80 | } |
81 | swapData.op = readData.op; | |
82 | swapData.swap_filen = readData.swap_filen; | |
83 | swapData.timestamp = readData.timestamp; | |
84 | swapData.lastref = readData.lastref; | |
85 | swapData.expires = readData.expires; | |
86 | swapData.lastmod = readData.lastmod; | |
87 | swapData.swap_file_sz = readData.swap_file_sz; | |
88 | swapData.refcount = readData.refcount; | |
89 | swapData.flags = readData.flags; | |
c3031d67 | 90 | xmemcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH); |
47f6e231 | 91 | return true; |
92 | } | |
93 | ||
94 | ||
26ac0430 AJ |
95 | class UFSSwapLogParser_v1:public UFSSwapLogParser |
96 | { | |
47f6e231 | 97 | public: |
26ac0430 AJ |
98 | UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) { |
99 | record_size = sizeof(StoreSwapLogData); | |
47f6e231 | 100 | } |
101 | bool ReadRecord(StoreSwapLogData &swapData); | |
102 | }; | |
103 | ||
104 | ||
105 | bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData) | |
106 | { | |
107 | int bytes = sizeof(StoreSwapLogData); | |
108 | ||
109 | assert(log); | |
26ac0430 AJ |
110 | |
111 | if (fread(&swapData, bytes, 1, log) != 1) { | |
47f6e231 | 112 | return false; |
113 | } | |
114 | return true; | |
115 | } | |
116 | ||
117 | ||
118 | UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp) | |
119 | { | |
120 | StoreSwapLogHeader header; | |
121 | ||
122 | assert(fp); | |
123 | ||
124 | if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1) | |
26ac0430 | 125 | return NULL; |
47f6e231 | 126 | |
26ac0430 AJ |
127 | if (header.op != SWAP_LOG_VERSION) { |
128 | debugs(47, 1, "Old swap file detected... "); | |
129 | fseek(fp, 0, SEEK_SET); | |
130 | return new UFSSwapLogParser_old(fp); | |
47f6e231 | 131 | } |
132 | ||
26ac0430 AJ |
133 | if (header.version == 1) { |
134 | if (fseek(fp, header.record_size, SEEK_SET) != 0) | |
135 | return NULL; | |
136 | ||
137 | if (header.record_size == sizeof(struct UFSSwapLogParser_old::StoreSwapLogDataOld)) { | |
138 | debugs(47, 1, "Version 1 of swap file without LFS support detected... "); | |
139 | return new UFSSwapLogParser_old(fp); | |
140 | } | |
141 | ||
142 | if (header.record_size == sizeof(StoreSwapLogData)) { | |
143 | debugs(47, 1, "Version 1 of swap file with LFS support detected... "); | |
144 | return new UFSSwapLogParser_v1(fp); | |
145 | } | |
146 | ||
147 | debugs(47, 1, "The swap file has wrong format!... "); | |
148 | return NULL; | |
149 | } | |
47f6e231 | 150 | |
151 | return NULL; | |
152 | } | |
153 | ||
26ac0430 AJ |
154 | int UFSSwapLogParser::SwapLogEntries() |
155 | { | |
47f6e231 | 156 | struct stat sb; |
26ac0430 | 157 | |
47f6e231 | 158 | if (log_entries >= 0) |
26ac0430 | 159 | return log_entries; |
47f6e231 | 160 | |
26ac0430 AJ |
161 | if (log && record_size && 0 == fstat(fileno(log), &sb)) { |
162 | log_entries = sb.st_size/record_size; | |
163 | return log_entries; | |
47f6e231 | 164 | } |
165 | ||
166 | return 0; | |
167 | } | |
168 | ||
169 | ||
170 | ||
171 | ||
172 | RebuildState::RebuildState (RefCount<UFSSwapDir> aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false) | |
4d6d905e | 173 | { |
c8f4eac4 | 174 | speed = opt_foreground_rebuild ? 1 << 30 : 50; |
175 | /* | |
176 | * If the swap.state file exists in the cache_dir, then | |
177 | * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll | |
178 | * use commonUfsDirRebuildFromDirectory() to open up each file | |
179 | * and suck in the meta data. | |
180 | */ | |
181 | int clean = 0; | |
182 | int zeroLengthLog = 0; | |
183 | FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog); | |
184 | ||
47f6e231 | 185 | if (fp && !zeroLengthLog) |
26ac0430 | 186 | LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp); |
47f6e231 | 187 | |
188 | if (LogParser == NULL ) { | |
c8f4eac4 | 189 | fromLog = false; |
190 | ||
191 | if (fp != NULL) | |
192 | fclose(fp); | |
193 | ||
194 | } else { | |
26ac0430 AJ |
195 | fromLog = true; |
196 | flags.clean = (unsigned int) clean; | |
c8f4eac4 | 197 | } |
62e76326 | 198 | |
c8f4eac4 | 199 | if (!clean) |
200 | flags.need_to_validate = 1; | |
201 | ||
bf8fe701 | 202 | debugs(47, 1, "Rebuilding storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")"); |
4d6d905e | 203 | } |
204 | ||
d3b3ab85 | 205 | RebuildState::~RebuildState() |
4d6d905e | 206 | { |
d3b3ab85 | 207 | sd->closeTmpSwapLog(); |
47f6e231 | 208 | |
209 | if (LogParser) | |
26ac0430 | 210 | delete LogParser; |
4d6d905e | 211 | } |
212 | ||
213 | void | |
c8f4eac4 | 214 | RebuildState::RebuildStep(void *data) |
4d6d905e | 215 | { |
d3b3ab85 | 216 | RebuildState *rb = (RebuildState *)data; |
c8f4eac4 | 217 | rb->rebuildStep(); |
218 | ||
219 | if (!rb->isDone()) | |
220 | eventAdd("storeRebuild", RebuildStep, rb, 0.0, 1); | |
221 | else { | |
bef81ea5 | 222 | StoreController::store_dirs_rebuilding--; |
c8f4eac4 | 223 | storeRebuildComplete(&rb->counts); |
224 | delete rb; | |
225 | } | |
226 | } | |
227 | ||
228 | void | |
229 | RebuildState::rebuildStep() | |
230 | { | |
231 | if (fromLog) | |
232 | rebuildFromSwapLog(); | |
233 | else | |
234 | rebuildFromDirectory(); | |
4d6d905e | 235 | } |
236 | ||
26ac0430 AJ |
237 | struct InitStoreEntry : public unary_function<StoreMeta, void> { |
238 | InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {} | |
62e76326 | 239 | |
26ac0430 | 240 | void operator()(StoreMeta const &x) { |
62e76326 | 241 | switch (x.getType()) { |
242 | ||
243 | case STORE_META_KEY: | |
c3031d67 | 244 | assert(x.length == SQUID_MD5_DIGEST_LENGTH); |
245 | xmemcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH); | |
62e76326 | 246 | break; |
247 | ||
248 | case STORE_META_STD: | |
26ac0430 | 249 | struct old_metahdr { |
47f6e231 | 250 | time_t timestamp; |
251 | time_t lastref; | |
252 | time_t expires; | |
253 | time_t lastmod; | |
254 | size_t swap_file_sz; | |
255 | u_short refcount; | |
256 | u_short flags; | |
257 | } *tmp; | |
26ac0430 | 258 | tmp = (struct old_metahdr *)x.value; |
47f6e231 | 259 | assert(x.length == STORE_HDR_METASIZE_OLD); |
26ac0430 AJ |
260 | what->timestamp = tmp->timestamp; |
261 | what->lastref = tmp->lastref; | |
262 | what->expires = tmp->expires; | |
263 | what->lastmod = tmp->lastmod; | |
264 | what->swap_file_sz = tmp->swap_file_sz; | |
265 | what->refcount = tmp->refcount; | |
266 | what->flags = tmp->flags; | |
47f6e231 | 267 | break; |
268 | ||
26ac0430 AJ |
269 | case STORE_META_STD_LFS: |
270 | assert(x.length == STORE_HDR_METASIZE); | |
62e76326 | 271 | xmemcpy(&what->timestamp, x.value, STORE_HDR_METASIZE); |
272 | break; | |
273 | ||
274 | default: | |
275 | break; | |
276 | } | |
528b2c61 | 277 | } |
62e76326 | 278 | |
528b2c61 | 279 | StoreEntry *what; |
280 | cache_key *index; | |
281 | }; | |
282 | ||
4d6d905e | 283 | void |
d3b3ab85 | 284 | RebuildState::rebuildFromDirectory() |
4d6d905e | 285 | { |
4d6d905e | 286 | LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE); |
822b78b5 | 287 | currentEntry(NULL); |
c3031d67 | 288 | cache_key key[SQUID_MD5_DIGEST_LENGTH]; |
62e76326 | 289 | |
4d6d905e | 290 | struct stat sb; |
291 | int swap_hdr_len; | |
292 | int fd = -1; | |
528b2c61 | 293 | StoreMeta *tlv_list; |
d3b3ab85 | 294 | assert(this != NULL); |
bf8fe701 | 295 | debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index); |
62e76326 | 296 | |
d3b3ab85 | 297 | for (int count = 0; count < speed; count++) { |
62e76326 | 298 | assert(fd == -1); |
299 | sfileno filn = 0; | |
300 | int size; | |
301 | fd = getNextFile(&filn, &size); | |
302 | ||
303 | if (fd == -2) { | |
bf8fe701 | 304 | debugs(47, 1, "Done scanning " << sd->path << " swaplog (" << n_read << " entries)"); |
c8f4eac4 | 305 | _done = true; |
62e76326 | 306 | return; |
307 | } else if (fd < 0) { | |
308 | continue; | |
309 | } | |
310 | ||
311 | assert(fd > -1); | |
312 | /* lets get file stats here */ | |
313 | ||
314 | if (fstat(fd, &sb) < 0) { | |
bf8fe701 | 315 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror()); |
62e76326 | 316 | file_close(fd); |
317 | store_open_disk_fd--; | |
318 | fd = -1; | |
319 | continue; | |
320 | } | |
321 | ||
322 | if ((++counts.scancount & 0xFFFF) == 0) | |
bf8fe701 | 323 | debugs(47, 3, " " << sd->path << " " << std::setw(7) << counts.scancount << " files opened so far."); |
26ac0430 | 324 | debugs(47, 9, "file_in: fd=" << fd << " "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn); |
62e76326 | 325 | |
62e76326 | 326 | |
327 | statCounter.syscalls.disk.reads++; | |
328 | ||
329 | int len; | |
330 | ||
331 | if ((len = FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE)) < 0) { | |
bf8fe701 | 332 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: read(FD " << fd << "): " << xstrerror()); |
62e76326 | 333 | file_close(fd); |
334 | store_open_disk_fd--; | |
335 | fd = -1; | |
336 | continue; | |
337 | } | |
338 | ||
339 | file_close(fd); | |
340 | store_open_disk_fd--; | |
341 | fd = -1; | |
342 | swap_hdr_len = 0; | |
62e76326 | 343 | |
528b2c61 | 344 | StoreMetaUnpacker aBuilder(hdr_buf, len, &swap_hdr_len); |
62e76326 | 345 | |
346 | if (!aBuilder.isBufferSane()) { | |
bf8fe701 | 347 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: Swap data buffer length is not sane."); |
62e76326 | 348 | /* XXX shouldn't this be a call to commonUfsUnlink ? */ |
349 | sd->unlinkFile ( filn); | |
350 | continue; | |
351 | } | |
352 | ||
528b2c61 | 353 | tlv_list = aBuilder.createStoreMeta (); |
62e76326 | 354 | |
355 | if (tlv_list == NULL) { | |
bf8fe701 | 356 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: failed to get meta data"); |
62e76326 | 357 | /* XXX shouldn't this be a call to commonUfsUnlink ? */ |
358 | sd->unlinkFile (filn); | |
359 | continue; | |
360 | } | |
361 | ||
bf8fe701 | 362 | debugs(47, 3, "commonUfsDirRebuildFromDirectory: successful swap meta unpacking"); |
c3031d67 | 363 | memset(key, '\0', SQUID_MD5_DIGEST_LENGTH); |
f894387a | 364 | |
365 | StoreEntry tmpe; | |
62e76326 | 366 | InitStoreEntry visitor(&tmpe, key); |
367 | for_each(*tlv_list, visitor); | |
368 | storeSwapTLVFree(tlv_list); | |
369 | tlv_list = NULL; | |
370 | ||
371 | if (storeKeyNull(key)) { | |
bf8fe701 | 372 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: NULL key"); |
62e76326 | 373 | sd->unlinkFile(filn); |
374 | continue; | |
375 | } | |
376 | ||
377 | tmpe.key = key; | |
378 | /* check sizes */ | |
379 | ||
380 | if (tmpe.swap_file_sz == 0) { | |
47f6e231 | 381 | tmpe.swap_file_sz = (uint64_t) sb.st_size; |
382 | } else if (tmpe.swap_file_sz == (uint64_t)(sb.st_size - swap_hdr_len)) { | |
383 | tmpe.swap_file_sz = (uint64_t) sb.st_size; | |
384 | } else if (tmpe.swap_file_sz != (uint64_t)sb.st_size) { | |
bf8fe701 | 385 | debugs(47, 1, "commonUfsDirRebuildFromDirectory: SIZE MISMATCH " << |
47f6e231 | 386 | tmpe.swap_file_sz << "!=" << |
387 | sb.st_size); | |
bf8fe701 | 388 | |
62e76326 | 389 | sd->unlinkFile(filn); |
390 | continue; | |
391 | } | |
392 | ||
393 | if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) { | |
394 | sd->unlinkFile(filn); | |
395 | counts.badflags++; | |
396 | continue; | |
397 | } | |
398 | ||
c8f4eac4 | 399 | /* this needs to become |
400 | * 1) unpack url | |
401 | * 2) make synthetic request with headers ?? or otherwise search | |
402 | * for a matching object in the store | |
403 | * TODO FIXME change to new async api | |
404 | * TODO FIXME I think there is a race condition here with the | |
405 | * async api : | |
406 | * store A reads in object foo, searchs for it, and finds nothing. | |
407 | * store B reads in object foo, searchs for it, finds nothing. | |
408 | * store A gets called back with nothing, so registers the object | |
409 | * store B gets called back with nothing, so registers the object, | |
410 | * which will conflict when the in core index gets around to scanning | |
411 | * store B. | |
412 | * | |
26ac0430 | 413 | * this suggests that rather than searching for duplicates, the |
c8f4eac4 | 414 | * index rebuild should just assume its the most recent accurate |
415 | * store entry and whoever indexes the stores handles duplicates. | |
416 | */ | |
417 | e = Store::Root().get(key); | |
62e76326 | 418 | |
419 | if (e && e->lastref >= tmpe.lastref) { | |
420 | /* key already exists, current entry is newer */ | |
421 | /* keep old, ignore new */ | |
422 | counts.dupcount++; | |
423 | continue; | |
424 | } else if (NULL != e) { | |
425 | /* URL already exists, this swapfile not being used */ | |
426 | /* junk old, load new */ | |
5f33b71d | 427 | e->release(); /* release old entry */ |
62e76326 | 428 | counts.dupcount++; |
429 | } | |
430 | ||
431 | counts.objcount++; | |
26ac0430 | 432 | // tmpe.dump(5); |
c8f4eac4 | 433 | currentEntry(sd->addDiskRestore(key, |
434 | filn, | |
435 | tmpe.swap_file_sz, | |
436 | tmpe.expires, | |
437 | tmpe.timestamp, | |
438 | tmpe.lastref, | |
439 | tmpe.lastmod, | |
440 | tmpe.refcount, /* refcount */ | |
441 | tmpe.flags, /* flags */ | |
442 | (int) flags.clean)); | |
443 | storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); | |
4d6d905e | 444 | } |
62e76326 | 445 | |
d3b3ab85 | 446 | } |
447 | ||
822b78b5 | 448 | StoreEntry * |
449 | RebuildState::currentEntry() const | |
450 | { | |
451 | return e; | |
452 | } | |
453 | ||
454 | void | |
455 | RebuildState::currentEntry(StoreEntry *newValue) | |
456 | { | |
457 | e = newValue; | |
458 | } | |
459 | ||
d3b3ab85 | 460 | void |
461 | RebuildState::rebuildFromSwapLog() | |
462 | { | |
822b78b5 | 463 | currentEntry (NULL); |
4d6d905e | 464 | double x; |
4d6d905e | 465 | /* load a number of objects per invocation */ |
62e76326 | 466 | |
d3b3ab85 | 467 | for (int count = 0; count < speed; count++) { |
c8f4eac4 | 468 | StoreSwapLogData swapData; |
62e76326 | 469 | |
47f6e231 | 470 | if (LogParser->ReadRecord(swapData) != 1) { |
bf8fe701 | 471 | debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)"); |
47f6e231 | 472 | LogParser->Close(); |
26ac0430 AJ |
473 | delete LogParser; |
474 | LogParser = NULL; | |
c8f4eac4 | 475 | _done = true; |
62e76326 | 476 | return; |
477 | } | |
478 | ||
479 | n_read++; | |
480 | ||
c8f4eac4 | 481 | if (swapData.op <= SWAP_LOG_NOP) |
62e76326 | 482 | continue; |
483 | ||
c8f4eac4 | 484 | if (swapData.op >= SWAP_LOG_MAX) |
62e76326 | 485 | continue; |
486 | ||
487 | /* | |
488 | * BC: during 2.4 development, we changed the way swap file | |
489 | * numbers are assigned and stored. The high 16 bits used | |
490 | * to encode the SD index number. There used to be a call | |
26ac0430 | 491 | * to storeDirProperFileno here that re-assigned the index |
62e76326 | 492 | * bits. Now, for backwards compatibility, we just need |
493 | * to mask it off. | |
494 | */ | |
c8f4eac4 | 495 | swapData.swap_filen &= 0x00FFFFFF; |
62e76326 | 496 | |
bf8fe701 | 497 | debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " << |
498 | swap_log_op_str[(int) swapData.op] << " " << | |
499 | storeKeyText(swapData.key) << " "<< std::setfill('0') << | |
500 | std::hex << std::uppercase << std::setw(8) << | |
501 | swapData.swap_filen); | |
62e76326 | 502 | |
c8f4eac4 | 503 | if (swapData.op == SWAP_LOG_ADD) { |
62e76326 | 504 | (void) 0; |
c8f4eac4 | 505 | } else if (swapData.op == SWAP_LOG_DEL) { |
506 | /* Delete unless we already have a newer copy anywhere in any store */ | |
507 | /* this needs to become | |
508 | * 1) unpack url | |
509 | * 2) make synthetic request with headers ?? or otherwise search | |
510 | * for a matching object in the store | |
511 | * TODO FIXME change to new async api | |
512 | */ | |
513 | currentEntry (Store::Root().get(swapData.key)); | |
514 | ||
77fde53f | 515 | if (currentEntry() != NULL && swapData.lastref >= e->lastref) { |
62e76326 | 516 | /* |
517 | * Make sure we don't unlink the file, it might be | |
518 | * in use by a subsequent entry. Also note that | |
519 | * we don't have to subtract from store_swap_size | |
520 | * because adding to store_swap_size happens in | |
521 | * the cleanup procedure. | |
522 | */ | |
d88e3c49 | 523 | currentEntry()->expireNow(); |
524 | currentEntry()->releaseRequest(); | |
62e76326 | 525 | |
526 | if (currentEntry()->swap_filen > -1) { | |
527 | UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn)); | |
528 | assert (sdForThisEntry); | |
529 | sdForThisEntry->replacementRemove(currentEntry()); | |
530 | sdForThisEntry->mapBitReset(currentEntry()->swap_filen); | |
531 | currentEntry()->swap_filen = -1; | |
532 | currentEntry()->swap_dirn = -1; | |
533 | } | |
534 | ||
5f33b71d | 535 | currentEntry()->release(); |
62e76326 | 536 | counts.objcount--; |
537 | counts.cancelcount++; | |
26ac0430 AJ |
538 | } |
539 | continue; | |
62e76326 | 540 | } else { |
411c6ea3 | 541 | x = ::log(static_cast<double>(++counts.bad_log_op)) / ::log(10.0); |
62e76326 | 542 | |
543 | if (0.0 == x - (double) (int) x) | |
bf8fe701 | 544 | debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found"); |
62e76326 | 545 | |
546 | counts.invalid++; | |
547 | ||
548 | continue; | |
549 | } | |
550 | ||
551 | if ((++counts.scancount & 0xFFF) == 0) { | |
552 | ||
47f6e231 | 553 | int swap_entries = LogParser->SwapLogEntries(); |
62e76326 | 554 | |
47f6e231 | 555 | if (0 != swap_entries ) |
62e76326 | 556 | storeRebuildProgress(sd->index, |
47f6e231 | 557 | swap_entries, n_read); |
62e76326 | 558 | } |
559 | ||
c8f4eac4 | 560 | if (!sd->validFileno(swapData.swap_filen, 0)) { |
62e76326 | 561 | counts.invalid++; |
562 | continue; | |
563 | } | |
564 | ||
c8f4eac4 | 565 | if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) { |
62e76326 | 566 | counts.badflags++; |
567 | continue; | |
568 | } | |
569 | ||
c8f4eac4 | 570 | /* this needs to become |
571 | * 1) unpack url | |
572 | * 2) make synthetic request with headers ?? or otherwise search | |
573 | * for a matching object in the store | |
574 | * TODO FIXME change to new async api | |
575 | */ | |
576 | currentEntry (Store::Root().get(swapData.key)); | |
577 | ||
d3b3ab85 | 578 | int used; /* is swapfile already in use? */ |
c8f4eac4 | 579 | |
580 | used = sd->mapBitTest(swapData.swap_filen); | |
581 | ||
62e76326 | 582 | /* If this URL already exists in the cache, does the swap log |
583 | * appear to have a newer entry? Compare 'lastref' from the | |
584 | * swap log to e->lastref. */ | |
585 | /* is the log entry newer than current entry? */ | |
c8f4eac4 | 586 | int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; |
62e76326 | 587 | |
588 | if (used && !disk_entry_newer) { | |
589 | /* log entry is old, ignore it */ | |
590 | counts.clashcount++; | |
591 | continue; | |
c8f4eac4 | 592 | } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { |
62e76326 | 593 | /* swapfile taken, same URL, newer, update meta */ |
594 | ||
595 | if (currentEntry()->store_status == STORE_OK) { | |
c8f4eac4 | 596 | currentEntry()->lastref = swapData.timestamp; |
597 | currentEntry()->timestamp = swapData.timestamp; | |
598 | currentEntry()->expires = swapData.expires; | |
599 | currentEntry()->lastmod = swapData.lastmod; | |
600 | currentEntry()->flags = swapData.flags; | |
601 | currentEntry()->refcount += swapData.refcount; | |
62e76326 | 602 | sd->dereference(*currentEntry()); |
603 | } else { | |
604 | debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); | |
bf8fe701 | 605 | debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__); |
26ac0430 AJ |
606 | } |
607 | continue; | |
62e76326 | 608 | } else if (used) { |
609 | /* swapfile in use, not by this URL, log entry is newer */ | |
610 | /* This is sorta bad: the log entry should NOT be newer at this | |
611 | * point. If the log is dirty, the filesize check should have | |
612 | * caught this. If the log is clean, there should never be a | |
613 | * newer entry. */ | |
26ac0430 AJ |
614 | debugs(47, 1, "WARNING: newer swaplog entry for dirno " << |
615 | sd->index << ", fileno "<< std::setfill('0') << std::hex << | |
616 | std::uppercase << std::setw(8) << swapData.swap_filen); | |
bf8fe701 | 617 | |
62e76326 | 618 | /* I'm tempted to remove the swapfile here just to be safe, |
619 | * but there is a bad race condition in the NOVM version if | |
620 | * the swapfile has recently been opened for writing, but | |
621 | * not yet opened for reading. Because we can't map | |
622 | * swapfiles back to StoreEntrys, we don't know the state | |
623 | * of the entry using that file. */ | |
624 | /* We'll assume the existing entry is valid, probably because | |
625 | * were in a slow rebuild and the the swap file number got taken | |
626 | * and the validation procedure hasn't run. */ | |
627 | assert(flags.need_to_validate); | |
628 | counts.clashcount++; | |
629 | continue; | |
630 | } else if (currentEntry() && !disk_entry_newer) { | |
631 | /* key already exists, current entry is newer */ | |
632 | /* keep old, ignore new */ | |
633 | counts.dupcount++; | |
634 | continue; | |
635 | } else if (currentEntry()) { | |
636 | /* key already exists, this swapfile not being used */ | |
637 | /* junk old, load new */ | |
d88e3c49 | 638 | currentEntry()->expireNow(); |
639 | currentEntry()->releaseRequest(); | |
62e76326 | 640 | |
641 | if (currentEntry()->swap_filen > -1) { | |
642 | UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn)); | |
643 | sdForThisEntry->replacementRemove(currentEntry()); | |
644 | /* Make sure we don't actually unlink the file */ | |
645 | sdForThisEntry->mapBitReset(currentEntry()->swap_filen); | |
646 | currentEntry()->swap_filen = -1; | |
647 | currentEntry()->swap_dirn = -1; | |
648 | } | |
649 | ||
5f33b71d | 650 | currentEntry()->release(); |
62e76326 | 651 | counts.dupcount++; |
652 | } else { | |
653 | /* URL doesnt exist, swapfile not in use */ | |
654 | /* load new */ | |
655 | (void) 0; | |
656 | } | |
657 | ||
658 | /* update store_swap_size */ | |
659 | counts.objcount++; | |
660 | ||
c8f4eac4 | 661 | currentEntry(sd->addDiskRestore(swapData.key, |
662 | swapData.swap_filen, | |
663 | swapData.swap_file_sz, | |
664 | swapData.expires, | |
665 | swapData.timestamp, | |
666 | swapData.lastref, | |
667 | swapData.lastmod, | |
668 | swapData.refcount, | |
669 | swapData.flags, | |
62e76326 | 670 | (int) flags.clean)); |
671 | ||
672 | storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); | |
4d6d905e | 673 | } |
62e76326 | 674 | |
4d6d905e | 675 | } |
676 | ||
677 | int | |
d3b3ab85 | 678 | RebuildState::getNextFile(sfileno * filn_p, int *size) |
4d6d905e | 679 | { |
4d6d905e | 680 | int fd = -1; |
4d6d905e | 681 | int dirs_opened = 0; |
bf8fe701 | 682 | debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init << ", " << |
683 | sd->index << ": /"<< std::setfill('0') << std::hex << | |
684 | std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) << | |
685 | curlvl2); | |
62e76326 | 686 | |
d3b3ab85 | 687 | if (done) |
62e76326 | 688 | return -2; |
689 | ||
d3b3ab85 | 690 | while (fd < 0 && done == 0) { |
62e76326 | 691 | fd = -1; |
692 | ||
693 | if (0 == flags.init) { /* initialize, open first file */ | |
694 | done = 0; | |
695 | curlvl1 = 0; | |
696 | curlvl2 = 0; | |
697 | in_dir = 0; | |
698 | flags.init = 1; | |
699 | assert(Config.cacheSwap.n_configured > 0); | |
700 | } | |
701 | ||
702 | if (0 == in_dir) { /* we need to read in a new directory */ | |
703 | snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X", | |
704 | sd->path, | |
705 | curlvl1, curlvl2); | |
706 | ||
707 | if (dirs_opened) | |
708 | return -1; | |
709 | ||
710 | td = opendir(fullpath); | |
711 | ||
712 | dirs_opened++; | |
713 | ||
714 | if (td == NULL) { | |
bf8fe701 | 715 | debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror()); |
62e76326 | 716 | } else { |
717 | entry = readdir(td); /* skip . and .. */ | |
718 | entry = readdir(td); | |
719 | ||
720 | if (entry == NULL && errno == ENOENT) | |
26ac0430 | 721 | debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!."); |
bf8fe701 | 722 | debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath); |
62e76326 | 723 | } |
724 | } | |
725 | ||
726 | if (td != NULL && (entry = readdir(td)) != NULL) { | |
727 | in_dir++; | |
728 | ||
729 | if (sscanf(entry->d_name, "%x", &fn) != 1) { | |
bf8fe701 | 730 | debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name); |
62e76326 | 731 | continue; |
732 | } | |
733 | ||
734 | if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) { | |
bf8fe701 | 735 | debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') << |
736 | std::hex << std::uppercase << std::setw(8) << fn << | |
737 | " does not belong in " << std::dec << sd->index << "/" << | |
738 | curlvl1 << "/" << curlvl2); | |
739 | ||
62e76326 | 740 | continue; |
741 | } | |
742 | ||
743 | if (sd->mapBitTest(fn)) { | |
bf8fe701 | 744 | debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next."); |
62e76326 | 745 | continue; |
746 | } | |
747 | ||
748 | snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s", | |
749 | fullpath, entry->d_name); | |
bf8fe701 | 750 | debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename); |
62e76326 | 751 | fd = file_open(fullfilename, O_RDONLY | O_BINARY); |
752 | ||
753 | if (fd < 0) | |
bf8fe701 | 754 | debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror()); |
62e76326 | 755 | else |
756 | store_open_disk_fd++; | |
757 | ||
758 | continue; | |
759 | } | |
760 | ||
761 | if (td != NULL) | |
762 | closedir(td); | |
763 | ||
764 | td = NULL; | |
765 | ||
766 | in_dir = 0; | |
767 | ||
768 | if (sd->validL2(++curlvl2)) | |
769 | continue; | |
770 | ||
771 | curlvl2 = 0; | |
772 | ||
773 | if (sd->validL1(++curlvl1)) | |
774 | continue; | |
775 | ||
776 | curlvl1 = 0; | |
777 | ||
778 | done = 1; | |
4d6d905e | 779 | } |
62e76326 | 780 | |
d3b3ab85 | 781 | *filn_p = fn; |
4d6d905e | 782 | return fd; |
783 | } | |
528b2c61 | 784 | |
c8f4eac4 | 785 | void |
786 | RebuildState::next(void (callback)(void *cbdata), void *cbdata) | |
787 | { | |
788 | /* for now, we don't cache at all */ | |
789 | speed = 1; | |
790 | currentEntry(NULL); | |
791 | ||
792 | while (!isDone() && currentEntry() == NULL) | |
793 | rebuildStep(); | |
794 | ||
795 | callback(cbdata); | |
796 | } | |
797 | ||
798 | bool | |
799 | RebuildState::next() | |
800 | { | |
801 | return false; | |
802 | } | |
803 | ||
804 | bool | |
805 | RebuildState::error() const | |
806 | { | |
807 | return false; | |
808 | } | |
809 | ||
810 | bool | |
811 | RebuildState::isDone() const | |
812 | { | |
813 | return _done; | |
814 | } | |
815 | ||
816 | StoreEntry * | |
817 | RebuildState::currentItem() | |
818 | { | |
819 | return currentEntry(); | |
820 | } | |
821 | ||
528b2c61 | 822 | #ifndef _USE_INLINE_ |
823 | #include "ufscommon.cci" | |
824 | #endif |