#include "StoreSwapLogData.h"
#include "swap_log_op.h"
-StoreSwapLogData::StoreSwapLogData(): op(0), swap_filen (0), timestamp (0), lastref (0), expires (0), lastmod(0), swap_file_sz (0), refcount (0), flags (0)
+// Based on Internet Checksum (RFC 1071) algorithm but takes three 32bit ints.
+// TODO: Consider Fletcher's checksum algorithm as a higher quality alternative
+void
+SwapChecksum24::set(uint32_t f1, uint32_t f2, uint32_t f3)
{
- memset (key, '\0', sizeof(key));
+ uint64_t sum = f1;
+ sum += f2;
+ sum += f3;
+
+ while (const uint64_t higherBits = sum >> 24)
+ sum = (sum & 0xFFFFFF) + higherBits;
+
+ sum = ~sum;
+
+ raw[0] = static_cast<uint8_t>(sum);
+ raw[1] = static_cast<uint8_t>(sum >> 8);
+ raw[2] = static_cast<uint8_t>(sum >> 16);
+}
+
+/// Same as 3-argument SwapChecksum24::set() but for int32_t and uint64_t
+void
+SwapChecksum24::set(int32_t f1, uint64_t f2)
+{
+ // split the second 64bit word into two 32bit words
+ set(static_cast<uint32_t>(f1),
+ static_cast<uint32_t>(f2 >> 32),
+ static_cast<uint32_t>(f2 & 0xFFFFFFFF));
+}
+
+std::ostream &
+SwapChecksum24::print(std::ostream &os) const
+{
+ return os << raw[0] << '-' << raw[1] << '-' << raw[2];
+}
+
+StoreSwapLogData::StoreSwapLogData()
+{
+ memset(this, 0, sizeof(*this));
}
bool
StoreSwapLogData::sane() const
{
- // TODO: These checks are rather weak. A corrupted swap.state may still
- // cause havoc (e.g., cur_size may become astronomical). Add checksums?
+ SwapChecksum24 actualSum;
+ actualSum.set(swap_filen, swap_file_sz);
+ if (checksum != actualSum)
+ return false;
const time_t minTime = -2; // -1 is common; expires sometimes uses -2
swap_file_sz > 0; // because swap headers ought to consume space
}
-StoreSwapLogHeader::StoreSwapLogHeader():op(SWAP_LOG_VERSION), version(1)
+void
+StoreSwapLogData::finalize()
+{
+ checksum.set(swap_filen, swap_file_sz);
+}
+
+StoreSwapLogHeader::StoreSwapLogHeader(): op(SWAP_LOG_VERSION), version(2),
+ record_size(sizeof(StoreSwapLogData))
+{
+ checksum.set(version, record_size, 0);
+}
+
+bool
+StoreSwapLogHeader::sane() const
+{
+ SwapChecksum24 actualSum;
+ actualSum.set(version, record_size, 0);
+ if (checksum != actualSum)
+ return false;
+
+ return op == SWAP_LOG_VERSION && version >= 2 && record_size > 0;
+}
+
+size_t
+StoreSwapLogHeader::gapSize() const
{
- record_size = sizeof(StoreSwapLogData);
+ assert(record_size > 0);
+ assert(static_cast<size_t>(record_size) > sizeof(*this));
+ return static_cast<size_t>(record_size) - sizeof(*this);
}
#include "squid-old.h"
-/*
- * Do we need to have the dirn in here? I don't think so, since we already
- * know the dirn ..
- */
+/// maintains a 24-bit checksum over integer fields
+class SwapChecksum24
+{
+public:
+ SwapChecksum24() { raw[0] = raw[1] = raw[2] = 0; }
+
+ bool operator ==(const SwapChecksum24 &o) const {
+ return raw[0] == o.raw[0] && raw[1] == o.raw[1] && raw[2] == o.raw[2];
+ }
+
+ bool operator !=(const SwapChecksum24 &o) const {
+ return !(*this == o);
+ }
+
+ /// compute and store checksum based on three 32bit integers
+ void set(uint32_t f1, uint32_t f2, uint32_t f3);
+
+ /// compute and store checksum based on int32_t and uint64_t integers
+ void set(int32_t f1, uint64_t f2);
+
+ // printing for debugging
+ std::ostream &print(std::ostream &os) const;
+
+private:
+ uint8_t raw[3]; // designed to follow "op" members, in pading space
+};
+
+inline std::ostream &
+operator <<(std::ostream &os, const SwapChecksum24 &sum)
+{
+ return sum.print(os);
+}
+
/**
\ingroup FielFormatSwapStateAPI
- \note This information is current as of version 2.2.STABLE4
- *
- \li Binary format on disk.
- \li DO NOT randomly alter.
- \li DO NOT add ANY virtual's.
*
\par
- * Defines the structure of a binary swap.state file entry.
+ * Defines the structure of a binary swap.state file entry for UFS stores.
+ * TODO: Move to fs/ufs (and remove from COSS).
*
\note StoreSwapLogData entries are written in native machine byte order
* They are not necessarily portable across architectures.
public:
MEMPROXY_CLASS(StoreSwapLogData);
+
+ /// type to use for storing time-related members; must be signed
+ typedef int64_t SwappedTime;
+
StoreSwapLogData();
/// consistency self-check: whether the data appears to make sense
bool sane() const;
+ /// call this before storing the log entry
+ void finalize();
+
/**
* Either SWAP_LOG_ADD when an object is added to the disk storage,
* or SWAP_LOG_DEL when an object is deleted.
*/
- char op;
+ uint8_t op;
+
+ /**
+ * Fingerprint to weed out bogus/corrupted swap.state entries.
+ */
+ SwapChecksum24 checksum; // follows "op" because compiler will pad anyway
/**
* The 32-bit file number which maps to a pathname.
sfileno swap_filen;
/**
- * A 32-bit Unix time value that represents the time when
+ * A Unix time value that represents the time when
* the origin server generated this response. If the response
* has a valid Date: header, this timestamp corresponds
* to that time. Otherwise, it is set to the Squid process time
* when the response is read (as soon as the end of headers are found).
*/
- time_t timestamp;
+ SwappedTime timestamp;
/**
* The last time that a client requested this object.
* Strictly speaking, this time is set whenever the StoreEntry
* is locked (via storeLockObject()).
*/
- time_t lastref;
+ SwappedTime lastref;
/**
* The value of the response's Expires: header, if any.
* where Squid sets expires to -2. This happens for the
* internal "netdb" object and for FTP URL responses.
*/
- time_t expires;
+ SwappedTime expires;
/**
* The value of the response's Last-modified: header, if any.
* This is set to -1 if there is no Last-modified: header,
* or if it is unparseable.
*/
- time_t lastmod;
+ SwappedTime lastmod;
/**
* This is the number of bytes that the object occupies on
MEMPROXY_CLASS_INLINE(StoreSwapLogData);
/// \ingroup FileFormatSwapStateAPI
+/// Swap log starts with this binary structure.
class StoreSwapLogHeader
{
public:
+ // sets default values for this Squid version; loaded values may differ
StoreSwapLogHeader();
- char op;
- int version;
- int record_size;
+
+ /// consistency self-check: whether the data appears to make sense
+ bool sane() const;
+
+ /// number of bytes after the log header before the first log entry
+ size_t gapSize() const;
+
+ uint8_t op;
+ SwapChecksum24 checksum; // follows "op" because compiler will pad anyway
+ int32_t version;
+ int32_t record_size;
};
buf.init(header.record_size, header.record_size);
buf.append(reinterpret_cast<const char*>(&header), sizeof(header));
// Pad to keep in sync with UFSSwapDir::writeCleanStart().
- // TODO: When MemBuf::spaceSize() is fixed not to subtract one,
- // memset() space() with zeroes and use spaceSize() below.
- buf.appended(static_cast<size_t>(header.record_size) - sizeof(header));
+ memset(buf.space(), 0, header.gapSize());
+ buf.appended(header.gapSize());
file_write(swaplog_fd, -1, buf.content(), buf.contentSize(),
NULL, NULL, buf.freeFunc());
}
/*copy the header */
memcpy(state->outbuf, &header, sizeof(StoreSwapLogHeader));
// Leave a gap to keep in sync with UFSSwapDir::openTmpSwapLog().
+ memset(state->outbuf + sizeof(StoreSwapLogHeader), 0, header.gapSize());
state->outbuf_offset += header.record_size;
state->walker = repl->WalkInit(repl);
s.refcount = e.refcount;
s.flags = e.flags;
memcpy(&s.key, e.key, SQUID_MD5_DIGEST_LENGTH);
+ s.finalize();
memcpy(outbuf + outbuf_offset, &s, ss);
outbuf_offset += ss;
/* buffered write */
s->refcount = e.refcount;
s->flags = e.flags;
memcpy(s->key, e.key, SQUID_MD5_DIGEST_LENGTH);
+ s->finalize();
file_write(swaplog_fd,
-1,
s,
}
};
+#if UNUSED_CODE
/// Parse a swap header entry created on a system with 32-bit size_t, time_t and sfileno
/// this is typical of 32-bit systems without large file support and with old kernels
/// NP: SQUID_MD5_DIGEST_LENGTH is very risky still.
}
return true;
}
+#endif /* UNUSED_CODE */
+
+/// swap.state v2 log parser
+class UFSSwapLogParser_v2: public UFSSwapLogParser
+{
+public:
+ UFSSwapLogParser_v2(FILE *fp): UFSSwapLogParser(fp) {
+ record_size = sizeof(StoreSwapLogData);
+ }
+ bool ReadRecord(StoreSwapLogData &swapData) {
+ assert(log);
+ return fread(&swapData, sizeof(StoreSwapLogData), 1, log) == 1;
+ }
+};
UFSSwapLogParser *UFSSwapLogParser::GetUFSSwapLogParser(FILE *fp)
return new UFSSwapLogParser_v1_32bs(fp); // Um. 32-bits except time_t, and can't determine that.
}
+ debugs(47, 2, "Swap file version: " << header.version);
+
if (header.version == 1) {
if (fseek(fp, header.record_size, SEEK_SET) != 0)
return NULL;
+ debugs(47, DBG_IMPORTANT, "Rejecting swap file v1 to avoid cache " <<
+ "index corruption. Forcing a full cache index rebuild. " <<
+ "See Squid bug #3441.");
+ return NULL;
+
+#if UNUSED_CODE
// baseline
// 32-bit sfileno
// native time_t (hopefully 64-bit)
debugs(47, 1, "WARNING: The swap file has wrong format!... ");
debugs(47, 1, "NOTE: Cannot safely downgrade caches to short (32-bit) timestamps.");
return NULL;
+#endif
}
- // XXX: version 2 of swapfile. This time use fixed-bit sizes for everything!!
- // and preferrably write to disk in network-order bytes for the larger fields.
+ if (header.version >= 2) {
+ if (!header.sane()) {
+ debugs(47, DBG_IMPORTANT, "ERROR: Corrupted v" << header.version <<
+ " swap file header.");
+ return NULL;
+ }
+
+ if (fseek(fp, header.record_size, SEEK_SET) != 0)
+ return NULL;
+ if (header.version == 2)
+ return new UFSSwapLogParser_v2(fp);
+ }
+
+ // TODO: v3: write to disk in network-order bytes for the larger fields?
+
+ debugs(47, DBG_IMPORTANT, "Unknown swap file version: " << header.version);
return NULL;
}
if (!clean)
flags.need_to_validate = 1;
- debugs(47, 1, "Rebuilding storage in " << sd->path << " (" << (clean ? "CLEAN" : "DIRTY") << ")");
+ debugs(47, DBG_IMPORTANT, "Rebuilding storage in " << sd->path << " (" <<
+ (clean ? "clean log" : (LogParser ? "dirty log" : "no log")) << ")");
}
RebuildState::~RebuildState()
fd = getNextFile(&filn, &size);
if (fd == -2) {
- debugs(47, 1, "Done scanning " << sd->path << " swaplog (" << n_read << " entries)");
+ debugs(47, DBG_IMPORTANT, "Done scanning " << sd->path << " dir (" <<
+ n_read << " entries)");
_done = true;
return;
} else if (fd < 0) {
assert(fd > -1);
/* lets get file stats here */
+ n_read++;
+
if (fstat(fd, &sb) < 0) {
debugs(47, 1, "commonUfsDirRebuildFromDirectory: fstat(FD " << fd << "): " << xstrerror());
file_close(fd);
projects / thirdparty / squid.git / commitdiff
