]> git.ipfire.org Git - thirdparty/squid.git/blame - src/fs/ufs/ufscommon.cc
Cleanup: zap CVS Id tags
[thirdparty/squid.git] / src / fs / ufs / ufscommon.cc
CommitLineData
4d6d905e 1/*
262a0e14 2 * $Id$
26ac0430 3 * vim: set et :
4d6d905e 4 *
5 * DEBUG: section 47 Store Directory Routines
51ee7c82 6 * AUTHOR: Robert Collins
4d6d905e 7 *
8 * SQUID Web Proxy Cache http://www.squid-cache.org/
9 * ----------------------------------------------------------
10 *
11 * Squid is the result of efforts by numerous individuals from
12 * the Internet community; see the CONTRIBUTORS file for full
13 * details. Many organizations have provided support for Squid's
14 * development; see the SPONSORS file for full details. Squid is
15 * Copyrighted (C) 2001 by the Regents of the University of
16 * California; see the COPYRIGHT file for full details. Squid
17 * incorporates software developed and/or copyrighted by other
18 * sources; see the CREDITS file for full details.
19 *
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
26ac0430 24 *
4d6d905e 25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
26ac0430 29 *
4d6d905e 30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
33 *
51ee7c82 34 * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org>
4d6d905e 35 */
36
37#include "ufscommon.h"
e6ccf245 38#include "Store.h"
528b2c61 39#include "fde.h"
40#include "StoreMeta.h"
41#include "Generic.h"
42#include "StoreMetaUnpacker.h"
d3b3ab85 43#include "RefCount.h"
51ee7c82 44#include "StoreSwapLogData.h"
4d6d905e 45
d3b3ab85 46CBDATA_CLASS_INIT(RebuildState);
4d6d905e 47
47f6e231 48
26ac0430
AJ
49class UFSSwapLogParser_old:public UFSSwapLogParser
50{
47f6e231 51public:
26ac0430
AJ
52 struct StoreSwapLogDataOld {
53 char op;
54 sfileno swap_filen;
55 time_t timestamp;
56 time_t lastref;
57 time_t expires;
58 time_t lastmod;
59 size_t swap_file_sz;
60 u_short refcount;
61 u_short flags;
62 unsigned char key[SQUID_MD5_DIGEST_LENGTH];
47f6e231 63 };
26ac0430
AJ
64 UFSSwapLogParser_old(FILE *fp):UFSSwapLogParser(fp) {
65 record_size = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld);
47f6e231 66 }
67 bool ReadRecord(StoreSwapLogData &swapData);
68};
69
70
26ac0430
AJ
71bool UFSSwapLogParser_old::ReadRecord(StoreSwapLogData &swapData)
72{
47f6e231 73 UFSSwapLogParser_old::StoreSwapLogDataOld readData;
74 int bytes = sizeof(UFSSwapLogParser_old::StoreSwapLogDataOld);
75
76 assert(log);
77
26ac0430
AJ
78 if (fread(&readData, bytes, 1, log) != 1) {
79 return false;
47f6e231 80 }
81 swapData.op = readData.op;
82 swapData.swap_filen = readData.swap_filen;
83 swapData.timestamp = readData.timestamp;
84 swapData.lastref = readData.lastref;
85 swapData.expires = readData.expires;
86 swapData.lastmod = readData.lastmod;
87 swapData.swap_file_sz = readData.swap_file_sz;
88 swapData.refcount = readData.refcount;
89 swapData.flags = readData.flags;
c3031d67 90 xmemcpy(swapData.key, readData.key, SQUID_MD5_DIGEST_LENGTH);
47f6e231 91 return true;
92}
93
94
26ac0430
AJ
95class UFSSwapLogParser_v1:public UFSSwapLogParser
96{
47f6e231 97public:
26ac0430
AJ
98 UFSSwapLogParser_v1(FILE *fp):UFSSwapLogParser(fp) {
99 record_size = sizeof(StoreSwapLogData);
47f6e231 100 }
101 bool ReadRecord(StoreSwapLogData &swapData);
102};
103
104
105bool UFSSwapLogParser_v1::ReadRecord(StoreSwapLogData &swapData)
106{
107 int bytes = sizeof(StoreSwapLogData);
108
109 assert(log);
26ac0430
AJ
110
111 if (fread(&swapData, bytes, 1, log) != 1) {
47f6e231 112 return false;
113 }
114 return true;
115}
116
117
118UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp)
119{
120 StoreSwapLogHeader header;
121
122 assert(fp);
123
124 if (fread(&header, sizeof(StoreSwapLogHeader), 1, fp) != 1)
26ac0430 125 return NULL;
47f6e231 126
26ac0430
AJ
127 if (header.op != SWAP_LOG_VERSION) {
128 debugs(47, 1, "Old swap file detected... ");
129 fseek(fp, 0, SEEK_SET);
130 return new UFSSwapLogParser_old(fp);
47f6e231 131 }
132
26ac0430
AJ
133 if (header.version == 1) {
134 if (fseek(fp, header.record_size, SEEK_SET) != 0)
135 return NULL;
136
137 if (header.record_size == sizeof(struct UFSSwapLogParser_old::StoreSwapLogDataOld)) {
138 debugs(47, 1, "Version 1 of swap file without LFS support detected... ");
139 return new UFSSwapLogParser_old(fp);
140 }
141
142 if (header.record_size == sizeof(StoreSwapLogData)) {
143 debugs(47, 1, "Version 1 of swap file with LFS support detected... ");
144 return new UFSSwapLogParser_v1(fp);
145 }
146
147 debugs(47, 1, "The swap file has wrong format!... ");
148 return NULL;
149 }
47f6e231 150
151 return NULL;
152}
153
26ac0430
AJ
154int UFSSwapLogParser::SwapLogEntries()
155{
47f6e231 156 struct stat sb;
26ac0430 157
47f6e231 158 if (log_entries >= 0)
26ac0430 159 return log_entries;
47f6e231 160
26ac0430
AJ
161 if (log && record_size && 0 == fstat(fileno(log), &sb)) {
162 log_entries = sb.st_size/record_size;
163 return log_entries;
47f6e231 164 }
165
166 return 0;
167}
168
169
170
171
172RebuildState::RebuildState (RefCount<UFSSwapDir> aSwapDir) : sd (aSwapDir),LogParser(NULL), e(NULL), fromLog(true), _done (false)
4d6d905e 173{
c8f4eac4 174 speed = opt_foreground_rebuild ? 1 << 30 : 50;
175 /*
176 * If the swap.state file exists in the cache_dir, then
177 * we'll use commonUfsDirRebuildFromSwapLog(), otherwise we'll
178 * use commonUfsDirRebuildFromDirectory() to open up each file
179 * and suck in the meta data.
180 */
181 int clean = 0;
182 int zeroLengthLog = 0;
183 FILE *fp = sd->openTmpSwapLog(&clean, &zeroLengthLog);
184
47f6e231 185 if (fp && !zeroLengthLog)
26ac0430 186 LogParser = UFSSwapLogParser::GetUFSSwapLogParser(fp);
47f6e231 187
188 if (LogParser == NULL ) {
c8f4eac4 189 fromLog = false;
190
191 if (fp != NULL)
192 fclose(fp);
193
194 } else {
26ac0430
AJ
195 fromLog = true;
196 flags.clean = (unsigned int) clean;
c8f4eac4 197 }
62e76326 198
c8f4eac4 199 if (!clean)
200 flags.need_to_validate = 1;
201
bf8fe701 202 debugs(47, 1, "Rebuilding storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")");
4d6d905e 203}
204
d3b3ab85 205RebuildState::~RebuildState()
4d6d905e 206{
d3b3ab85 207 sd->closeTmpSwapLog();
47f6e231 208
209 if (LogParser)
26ac0430 210 delete LogParser;
4d6d905e 211}
212
213void
c8f4eac4 214RebuildState::RebuildStep(void *data)
4d6d905e 215{
d3b3ab85 216 RebuildState *rb = (RebuildState *)data;
c8f4eac4 217 rb->rebuildStep();
218
219 if (!rb->isDone())
220 eventAdd("storeRebuild", RebuildStep, rb, 0.0, 1);
221 else {
bef81ea5 222 StoreController::store_dirs_rebuilding--;
c8f4eac4 223 storeRebuildComplete(&rb->counts);
224 delete rb;
225 }
226}
227
228void
229RebuildState::rebuildStep()
230{
231 if (fromLog)
232 rebuildFromSwapLog();
233 else
234 rebuildFromDirectory();
4d6d905e 235}
236
26ac0430
AJ
237struct InitStoreEntry : public unary_function<StoreMeta, void> {
238 InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
62e76326 239
26ac0430 240 void operator()(StoreMeta const &x) {
62e76326 241 switch (x.getType()) {
242
243 case STORE_META_KEY:
c3031d67 244 assert(x.length == SQUID_MD5_DIGEST_LENGTH);
245 xmemcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH);
62e76326 246 break;
247
248 case STORE_META_STD:
26ac0430 249 struct old_metahdr {
47f6e231 250 time_t timestamp;
251 time_t lastref;
252 time_t expires;
253 time_t lastmod;
254 size_t swap_file_sz;
255 u_short refcount;
256 u_short flags;
257 } *tmp;
26ac0430 258 tmp = (struct old_metahdr *)x.value;
47f6e231 259 assert(x.length == STORE_HDR_METASIZE_OLD);
26ac0430
AJ
260 what->timestamp = tmp->timestamp;
261 what->lastref = tmp->lastref;
262 what->expires = tmp->expires;
263 what->lastmod = tmp->lastmod;
264 what->swap_file_sz = tmp->swap_file_sz;
265 what->refcount = tmp->refcount;
266 what->flags = tmp->flags;
47f6e231 267 break;
268
26ac0430
AJ
269 case STORE_META_STD_LFS:
270 assert(x.length == STORE_HDR_METASIZE);
62e76326 271 xmemcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
272 break;
273
274 default:
275 break;
276 }
528b2c61 277 }
62e76326 278
528b2c61 279 StoreEntry *what;
280 cache_key *index;
281};
282
4d6d905e 283void
d3b3ab85 284RebuildState::rebuildFromDirectory()
4d6d905e 285{
4d6d905e 286 LOCAL_ARRAY(char, hdr_buf, SM_PAGE_SIZE);
822b78b5 287 currentEntry(NULL);
c3031d67 288 cache_key key[SQUID_MD5_DIGEST_LENGTH];
62e76326 289
4d6d905e 290 struct stat sb;
291 int swap_hdr_len;
292 int fd = -1;
528b2c61 293 StoreMeta *tlv_list;
d3b3ab85 294 assert(this != NULL);
bf8fe701 295 debugs(47, 3, "commonUfsDirRebuildFromDirectory: DIR #" << sd->index);
62e76326 296
d3b3ab85 297 for (int count = 0; count < speed; count++) {
62e76326 298 assert(fd == -1);
299 sfileno filn = 0;
300 int size;
301 fd = getNextFile(&filn, &size);
302
303 if (fd == -2) {
bf8fe701 304 debugs(47, 1, "Done scanning " << sd->path << " swaplog (" << n_read << " entries)");
c8f4eac4 305 _done = true;
62e76326 306 return;
307 } else if (fd < 0) {
308 continue;
309 }
310
311 assert(fd > -1);
312 /* lets get file stats here */
313
314 if (fstat(fd, &sb) < 0) {
bf8fe701 315 debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror());
62e76326 316 file_close(fd);
317 store_open_disk_fd--;
318 fd = -1;
319 continue;
320 }
321
322 if ((++counts.scancount & 0xFFFF) == 0)
bf8fe701 323 debugs(47, 3, " " << sd->path << " " << std::setw(7) << counts.scancount << " files opened so far.");
26ac0430 324 debugs(47, 9, "file_in: fd=" << fd << " "<< std::setfill('0') << std::hex << std::uppercase << std::setw(8) << filn);
62e76326 325
62e76326 326
327 statCounter.syscalls.disk.reads++;
328
329 int len;
330
331 if ((len = FD_READ_METHOD(fd, hdr_buf, SM_PAGE_SIZE)) < 0) {
bf8fe701 332 debugs(47, 1, "commonUfsDirRebuildFromDirectory: read(FD " << fd << "): " << xstrerror());
62e76326 333 file_close(fd);
334 store_open_disk_fd--;
335 fd = -1;
336 continue;
337 }
338
339 file_close(fd);
340 store_open_disk_fd--;
341 fd = -1;
342 swap_hdr_len = 0;
62e76326 343
528b2c61 344 StoreMetaUnpacker aBuilder(hdr_buf, len, &swap_hdr_len);
62e76326 345
346 if (!aBuilder.isBufferSane()) {
bf8fe701 347 debugs(47, 1, "commonUfsDirRebuildFromDirectory: Swap data buffer length is not sane.");
62e76326 348 /* XXX shouldn't this be a call to commonUfsUnlink ? */
349 sd->unlinkFile ( filn);
350 continue;
351 }
352
528b2c61 353 tlv_list = aBuilder.createStoreMeta ();
62e76326 354
355 if (tlv_list == NULL) {
bf8fe701 356 debugs(47, 1, "commonUfsDirRebuildFromDirectory: failed to get meta data");
62e76326 357 /* XXX shouldn't this be a call to commonUfsUnlink ? */
358 sd->unlinkFile (filn);
359 continue;
360 }
361
bf8fe701 362 debugs(47, 3, "commonUfsDirRebuildFromDirectory: successful swap meta unpacking");
c3031d67 363 memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
f894387a 364
365 StoreEntry tmpe;
62e76326 366 InitStoreEntry visitor(&tmpe, key);
367 for_each(*tlv_list, visitor);
368 storeSwapTLVFree(tlv_list);
369 tlv_list = NULL;
370
371 if (storeKeyNull(key)) {
bf8fe701 372 debugs(47, 1, "commonUfsDirRebuildFromDirectory: NULL key");
62e76326 373 sd->unlinkFile(filn);
374 continue;
375 }
376
377 tmpe.key = key;
378 /* check sizes */
379
380 if (tmpe.swap_file_sz == 0) {
47f6e231 381 tmpe.swap_file_sz = (uint64_t) sb.st_size;
382 } else if (tmpe.swap_file_sz == (uint64_t)(sb.st_size - swap_hdr_len)) {
383 tmpe.swap_file_sz = (uint64_t) sb.st_size;
384 } else if (tmpe.swap_file_sz != (uint64_t)sb.st_size) {
bf8fe701 385 debugs(47, 1, "commonUfsDirRebuildFromDirectory: SIZE MISMATCH " <<
47f6e231 386 tmpe.swap_file_sz << "!=" <<
387 sb.st_size);
bf8fe701 388
62e76326 389 sd->unlinkFile(filn);
390 continue;
391 }
392
393 if (EBIT_TEST(tmpe.flags, KEY_PRIVATE)) {
394 sd->unlinkFile(filn);
395 counts.badflags++;
396 continue;
397 }
398
c8f4eac4 399 /* this needs to become
400 * 1) unpack url
401 * 2) make synthetic request with headers ?? or otherwise search
402 * for a matching object in the store
403 * TODO FIXME change to new async api
404 * TODO FIXME I think there is a race condition here with the
405 * async api :
406 * store A reads in object foo, searchs for it, and finds nothing.
407 * store B reads in object foo, searchs for it, finds nothing.
408 * store A gets called back with nothing, so registers the object
409 * store B gets called back with nothing, so registers the object,
410 * which will conflict when the in core index gets around to scanning
411 * store B.
412 *
26ac0430 413 * this suggests that rather than searching for duplicates, the
c8f4eac4 414 * index rebuild should just assume its the most recent accurate
415 * store entry and whoever indexes the stores handles duplicates.
416 */
417 e = Store::Root().get(key);
62e76326 418
419 if (e && e->lastref >= tmpe.lastref) {
420 /* key already exists, current entry is newer */
421 /* keep old, ignore new */
422 counts.dupcount++;
423 continue;
424 } else if (NULL != e) {
425 /* URL already exists, this swapfile not being used */
426 /* junk old, load new */
5f33b71d 427 e->release(); /* release old entry */
62e76326 428 counts.dupcount++;
429 }
430
431 counts.objcount++;
26ac0430 432 // tmpe.dump(5);
c8f4eac4 433 currentEntry(sd->addDiskRestore(key,
434 filn,
435 tmpe.swap_file_sz,
436 tmpe.expires,
437 tmpe.timestamp,
438 tmpe.lastref,
439 tmpe.lastmod,
440 tmpe.refcount, /* refcount */
441 tmpe.flags, /* flags */
442 (int) flags.clean));
443 storeDirSwapLog(currentEntry(), SWAP_LOG_ADD);
4d6d905e 444 }
62e76326 445
d3b3ab85 446}
447
822b78b5 448StoreEntry *
449RebuildState::currentEntry() const
450{
451 return e;
452}
453
454void
455RebuildState::currentEntry(StoreEntry *newValue)
456{
457 e = newValue;
458}
459
d3b3ab85 460void
461RebuildState::rebuildFromSwapLog()
462{
822b78b5 463 currentEntry (NULL);
4d6d905e 464 double x;
4d6d905e 465 /* load a number of objects per invocation */
62e76326 466
d3b3ab85 467 for (int count = 0; count < speed; count++) {
c8f4eac4 468 StoreSwapLogData swapData;
62e76326 469
47f6e231 470 if (LogParser->ReadRecord(swapData) != 1) {
bf8fe701 471 debugs(47, 1, "Done reading " << sd->path << " swaplog (" << n_read << " entries)");
47f6e231 472 LogParser->Close();
26ac0430
AJ
473 delete LogParser;
474 LogParser = NULL;
c8f4eac4 475 _done = true;
62e76326 476 return;
477 }
478
479 n_read++;
480
c8f4eac4 481 if (swapData.op <= SWAP_LOG_NOP)
62e76326 482 continue;
483
c8f4eac4 484 if (swapData.op >= SWAP_LOG_MAX)
62e76326 485 continue;
486
487 /*
488 * BC: during 2.4 development, we changed the way swap file
489 * numbers are assigned and stored. The high 16 bits used
490 * to encode the SD index number. There used to be a call
26ac0430 491 * to storeDirProperFileno here that re-assigned the index
62e76326 492 * bits. Now, for backwards compatibility, we just need
493 * to mask it off.
494 */
c8f4eac4 495 swapData.swap_filen &= 0x00FFFFFF;
62e76326 496
bf8fe701 497 debugs(47, 3, "commonUfsDirRebuildFromSwapLog: " <<
498 swap_log_op_str[(int) swapData.op] << " " <<
499 storeKeyText(swapData.key) << " "<< std::setfill('0') <<
500 std::hex << std::uppercase << std::setw(8) <<
501 swapData.swap_filen);
62e76326 502
c8f4eac4 503 if (swapData.op == SWAP_LOG_ADD) {
62e76326 504 (void) 0;
c8f4eac4 505 } else if (swapData.op == SWAP_LOG_DEL) {
506 /* Delete unless we already have a newer copy anywhere in any store */
507 /* this needs to become
508 * 1) unpack url
509 * 2) make synthetic request with headers ?? or otherwise search
510 * for a matching object in the store
511 * TODO FIXME change to new async api
512 */
513 currentEntry (Store::Root().get(swapData.key));
514
77fde53f 515 if (currentEntry() != NULL && swapData.lastref >= e->lastref) {
62e76326 516 /*
517 * Make sure we don't unlink the file, it might be
518 * in use by a subsequent entry. Also note that
519 * we don't have to subtract from store_swap_size
520 * because adding to store_swap_size happens in
521 * the cleanup procedure.
522 */
d88e3c49 523 currentEntry()->expireNow();
524 currentEntry()->releaseRequest();
62e76326 525
526 if (currentEntry()->swap_filen > -1) {
527 UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn));
528 assert (sdForThisEntry);
529 sdForThisEntry->replacementRemove(currentEntry());
530 sdForThisEntry->mapBitReset(currentEntry()->swap_filen);
531 currentEntry()->swap_filen = -1;
532 currentEntry()->swap_dirn = -1;
533 }
534
5f33b71d 535 currentEntry()->release();
62e76326 536 counts.objcount--;
537 counts.cancelcount++;
26ac0430
AJ
538 }
539 continue;
62e76326 540 } else {
411c6ea3 541 x = ::log(static_cast<double>(++counts.bad_log_op)) / ::log(10.0);
62e76326 542
543 if (0.0 == x - (double) (int) x)
bf8fe701 544 debugs(47, 1, "WARNING: " << counts.bad_log_op << " invalid swap log entries found");
62e76326 545
546 counts.invalid++;
547
548 continue;
549 }
550
551 if ((++counts.scancount & 0xFFF) == 0) {
552
47f6e231 553 int swap_entries = LogParser->SwapLogEntries();
62e76326 554
47f6e231 555 if (0 != swap_entries )
62e76326 556 storeRebuildProgress(sd->index,
47f6e231 557 swap_entries, n_read);
62e76326 558 }
559
c8f4eac4 560 if (!sd->validFileno(swapData.swap_filen, 0)) {
62e76326 561 counts.invalid++;
562 continue;
563 }
564
c8f4eac4 565 if (EBIT_TEST(swapData.flags, KEY_PRIVATE)) {
62e76326 566 counts.badflags++;
567 continue;
568 }
569
c8f4eac4 570 /* this needs to become
571 * 1) unpack url
572 * 2) make synthetic request with headers ?? or otherwise search
573 * for a matching object in the store
574 * TODO FIXME change to new async api
575 */
576 currentEntry (Store::Root().get(swapData.key));
577
d3b3ab85 578 int used; /* is swapfile already in use? */
c8f4eac4 579
580 used = sd->mapBitTest(swapData.swap_filen);
581
62e76326 582 /* If this URL already exists in the cache, does the swap log
583 * appear to have a newer entry? Compare 'lastref' from the
584 * swap log to e->lastref. */
585 /* is the log entry newer than current entry? */
c8f4eac4 586 int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0;
62e76326 587
588 if (used && !disk_entry_newer) {
589 /* log entry is old, ignore it */
590 counts.clashcount++;
591 continue;
c8f4eac4 592 } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) {
62e76326 593 /* swapfile taken, same URL, newer, update meta */
594
595 if (currentEntry()->store_status == STORE_OK) {
c8f4eac4 596 currentEntry()->lastref = swapData.timestamp;
597 currentEntry()->timestamp = swapData.timestamp;
598 currentEntry()->expires = swapData.expires;
599 currentEntry()->lastmod = swapData.lastmod;
600 currentEntry()->flags = swapData.flags;
601 currentEntry()->refcount += swapData.refcount;
62e76326 602 sd->dereference(*currentEntry());
603 } else {
604 debug_trap("commonUfsDirRebuildFromSwapLog: bad condition");
bf8fe701 605 debugs(47, 1, "\tSee " << __FILE__ << ":" << __LINE__);
26ac0430
AJ
606 }
607 continue;
62e76326 608 } else if (used) {
609 /* swapfile in use, not by this URL, log entry is newer */
610 /* This is sorta bad: the log entry should NOT be newer at this
611 * point. If the log is dirty, the filesize check should have
612 * caught this. If the log is clean, there should never be a
613 * newer entry. */
26ac0430
AJ
614 debugs(47, 1, "WARNING: newer swaplog entry for dirno " <<
615 sd->index << ", fileno "<< std::setfill('0') << std::hex <<
616 std::uppercase << std::setw(8) << swapData.swap_filen);
bf8fe701 617
62e76326 618 /* I'm tempted to remove the swapfile here just to be safe,
619 * but there is a bad race condition in the NOVM version if
620 * the swapfile has recently been opened for writing, but
621 * not yet opened for reading. Because we can't map
622 * swapfiles back to StoreEntrys, we don't know the state
623 * of the entry using that file. */
624 /* We'll assume the existing entry is valid, probably because
625 * were in a slow rebuild and the the swap file number got taken
626 * and the validation procedure hasn't run. */
627 assert(flags.need_to_validate);
628 counts.clashcount++;
629 continue;
630 } else if (currentEntry() && !disk_entry_newer) {
631 /* key already exists, current entry is newer */
632 /* keep old, ignore new */
633 counts.dupcount++;
634 continue;
635 } else if (currentEntry()) {
636 /* key already exists, this swapfile not being used */
637 /* junk old, load new */
d88e3c49 638 currentEntry()->expireNow();
639 currentEntry()->releaseRequest();
62e76326 640
641 if (currentEntry()->swap_filen > -1) {
642 UFSSwapDir *sdForThisEntry = dynamic_cast<UFSSwapDir *>(INDEXSD(currentEntry()->swap_dirn));
643 sdForThisEntry->replacementRemove(currentEntry());
644 /* Make sure we don't actually unlink the file */
645 sdForThisEntry->mapBitReset(currentEntry()->swap_filen);
646 currentEntry()->swap_filen = -1;
647 currentEntry()->swap_dirn = -1;
648 }
649
5f33b71d 650 currentEntry()->release();
62e76326 651 counts.dupcount++;
652 } else {
653 /* URL doesnt exist, swapfile not in use */
654 /* load new */
655 (void) 0;
656 }
657
658 /* update store_swap_size */
659 counts.objcount++;
660
c8f4eac4 661 currentEntry(sd->addDiskRestore(swapData.key,
662 swapData.swap_filen,
663 swapData.swap_file_sz,
664 swapData.expires,
665 swapData.timestamp,
666 swapData.lastref,
667 swapData.lastmod,
668 swapData.refcount,
669 swapData.flags,
62e76326 670 (int) flags.clean));
671
672 storeDirSwapLog(currentEntry(), SWAP_LOG_ADD);
4d6d905e 673 }
62e76326 674
4d6d905e 675}
676
677int
d3b3ab85 678RebuildState::getNextFile(sfileno * filn_p, int *size)
4d6d905e 679{
4d6d905e 680 int fd = -1;
4d6d905e 681 int dirs_opened = 0;
bf8fe701 682 debugs(47, 3, "commonUfsDirGetNextFile: flag=" << flags.init << ", " <<
683 sd->index << ": /"<< std::setfill('0') << std::hex <<
684 std::uppercase << std::setw(2) << curlvl1 << "/" << std::setw(2) <<
685 curlvl2);
62e76326 686
d3b3ab85 687 if (done)
62e76326 688 return -2;
689
d3b3ab85 690 while (fd < 0 && done == 0) {
62e76326 691 fd = -1;
692
693 if (0 == flags.init) { /* initialize, open first file */
694 done = 0;
695 curlvl1 = 0;
696 curlvl2 = 0;
697 in_dir = 0;
698 flags.init = 1;
699 assert(Config.cacheSwap.n_configured > 0);
700 }
701
702 if (0 == in_dir) { /* we need to read in a new directory */
703 snprintf(fullpath, SQUID_MAXPATHLEN, "%s/%02X/%02X",
704 sd->path,
705 curlvl1, curlvl2);
706
707 if (dirs_opened)
708 return -1;
709
710 td = opendir(fullpath);
711
712 dirs_opened++;
713
714 if (td == NULL) {
bf8fe701 715 debugs(47, 1, "commonUfsDirGetNextFile: opendir: " << fullpath << ": " << xstrerror());
62e76326 716 } else {
717 entry = readdir(td); /* skip . and .. */
718 entry = readdir(td);
719
720 if (entry == NULL && errno == ENOENT)
26ac0430 721 debugs(47, 1, "commonUfsDirGetNextFile: directory does not exist!.");
bf8fe701 722 debugs(47, 3, "commonUfsDirGetNextFile: Directory " << fullpath);
62e76326 723 }
724 }
725
726 if (td != NULL && (entry = readdir(td)) != NULL) {
727 in_dir++;
728
729 if (sscanf(entry->d_name, "%x", &fn) != 1) {
bf8fe701 730 debugs(47, 3, "commonUfsDirGetNextFile: invalid " << entry->d_name);
62e76326 731 continue;
732 }
733
734 if (!UFSSwapDir::FilenoBelongsHere(fn, sd->index, curlvl1, curlvl2)) {
bf8fe701 735 debugs(47, 3, "commonUfsDirGetNextFile: "<< std::setfill('0') <<
736 std::hex << std::uppercase << std::setw(8) << fn <<
737 " does not belong in " << std::dec << sd->index << "/" <<
738 curlvl1 << "/" << curlvl2);
739
62e76326 740 continue;
741 }
742
743 if (sd->mapBitTest(fn)) {
bf8fe701 744 debugs(47, 3, "commonUfsDirGetNextFile: Locked, continuing with next.");
62e76326 745 continue;
746 }
747
748 snprintf(fullfilename, SQUID_MAXPATHLEN, "%s/%s",
749 fullpath, entry->d_name);
bf8fe701 750 debugs(47, 3, "commonUfsDirGetNextFile: Opening " << fullfilename);
62e76326 751 fd = file_open(fullfilename, O_RDONLY | O_BINARY);
752
753 if (fd < 0)
bf8fe701 754 debugs(47, 1, "commonUfsDirGetNextFile: " << fullfilename << ": " << xstrerror());
62e76326 755 else
756 store_open_disk_fd++;
757
758 continue;
759 }
760
761 if (td != NULL)
762 closedir(td);
763
764 td = NULL;
765
766 in_dir = 0;
767
768 if (sd->validL2(++curlvl2))
769 continue;
770
771 curlvl2 = 0;
772
773 if (sd->validL1(++curlvl1))
774 continue;
775
776 curlvl1 = 0;
777
778 done = 1;
4d6d905e 779 }
62e76326 780
d3b3ab85 781 *filn_p = fn;
4d6d905e 782 return fd;
783}
528b2c61 784
c8f4eac4 785void
786RebuildState::next(void (callback)(void *cbdata), void *cbdata)
787{
788 /* for now, we don't cache at all */
789 speed = 1;
790 currentEntry(NULL);
791
792 while (!isDone() && currentEntry() == NULL)
793 rebuildStep();
794
795 callback(cbdata);
796}
797
798bool
799RebuildState::next()
800{
801 return false;
802}
803
804bool
805RebuildState::error() const
806{
807 return false;
808}
809
810bool
811RebuildState::isDone() const
812{
813 return _done;
814}
815
816StoreEntry *
817RebuildState::currentItem()
818{
819 return currentEntry();
820}
821
528b2c61 822#ifndef _USE_INLINE_
823#include "ufscommon.cci"
824#endif