62 ERROR: ... while accepting a TLS connection on ...: ...
63 Resuming indexing cache_dir # ... from ...
64 DNS IPv4 socket created at ..., FD ...
+65 WARNING: Indexer ignores a cache_dir entry: ...
\endverbatim
*/
section 20 Storage Manager Logging Functions
section 20 Storage Manager MD5 Cache Keys
section 20 Storage Manager Statistics
-section 20 Storage Manager Swapfile Metadata
-section 20 Storage Manager Swapfile Unpacker
section 20 Storage Manager Swapin Functions
section 20 Storage Manager Swapout Functions
section 20 Store Controller
DNSSOURCE = \
dns_internal.cc
-STOREMETA_SOURCE = \
- StoreMeta.cc \
- StoreMeta.h \
- StoreMetaMD5.cc \
- StoreMetaMD5.h \
- StoreMetaSTD.cc \
- StoreMetaSTD.h \
- StoreMetaSTDLFS.cc \
- StoreMetaSTDLFS.h \
- StoreMetaObjSize.h \
- StoreMetaURL.cc \
- StoreMetaURL.h \
- StoreMetaVary.cc \
- StoreMetaVary.h
-
LOADABLE_MODULES_SOURCES = \
LoadableModule.cc \
LoadableModule.h \
$(HTCPSOURCE) \
$(IPC_SOURCE) \
$(SNMP_SOURCE) \
- $(STOREMETA_SOURCE) \
$(UNLINKDSOURCE) \
$(WIN32_SOURCE) \
$(WINSVC_SOURCE) \
StoreIOBuffer.h \
StoreIOState.cc \
StoreIOState.h \
- StoreMetaUnpacker.cc \
- StoreMetaUnpacker.h \
StoreSearch.h \
StoreStats.cc \
StoreStats.h \
store_rebuild.h \
store_swapin.cc \
store_swapin.h \
- store_swapmeta.cc \
store_swapout.cc \
swap_log_op.h \
tools.cc \
check_PROGRAMS += tests/testRock
tests_testRock_SOURCES = \
$(DELAY_POOL_SOURCE) \
- $(STOREMETA_SOURCE) \
$(UNLINKDSOURCE) \
AccessLogEntry.cc \
AccessLogEntry.h \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- StoreMetaUnpacker.cc \
tests/testStoreSupport.cc \
tests/testStoreSupport.h \
StoreSwapLogData.cc \
tests/stub_store_rebuild.cc \
store_rebuild.h \
tests/stub_store_stats.cc \
- store_swapmeta.cc \
store_swapout.cc \
tests/stub_tools.cc \
tools.h \
check_PROGRAMS += tests/testUfs
tests_testUfs_SOURCES = \
$(DELAY_POOL_SOURCE) \
- $(STOREMETA_SOURCE) \
$(UNLINKDSOURCE) \
$(WIN32_SOURCE) \
AccessLogEntry.cc \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- StoreMetaUnpacker.cc \
tests/testStoreSupport.cc \
tests/testStoreSupport.h \
StoreSwapLogData.cc \
tests/stub_store_rebuild.cc \
store_rebuild.h \
tests/stub_store_stats.cc \
- store_swapmeta.cc \
store_swapout.cc \
tests/stub_tools.cc \
tools.h \
tests/testStoreHashIndex.cc \
tests/testStoreHashIndex.h \
StoreIOState.cc \
- tests/stub_StoreMeta.cc \
- StoreMetaUnpacker.cc \
tests/testStoreSupport.cc \
tests/testStoreSupport.h \
StoreSwapLogData.cc \
store_rebuild.h \
tests/stub_store_stats.cc \
store_swapout.cc \
- tests/stub_store_swapout.cc \
tests/CapturingStoreEntry.h \
tests/TestSwapDir.cc \
tests/TestSwapDir.h \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- tests/stub_StoreMeta.cc \
- StoreMetaUnpacker.cc \
tests/testStoreSupport.cc \
tests/testStoreSupport.h \
StoreSwapLogData.cc \
tests/stub_store_rebuild.cc \
store_rebuild.h \
tests/stub_store_stats.cc \
- store_swapmeta.cc \
store_swapout.cc \
tests/stub_tools.cc \
tools.h \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- tests/stub_StoreMeta.cc \
- StoreMetaUnpacker.cc \
StoreSwapLogData.cc \
StrList.cc \
StrList.h \
tests/stub_store_stats.cc \
store_swapin.cc \
store_swapin.h \
- store_swapmeta.cc \
store_swapout.cc \
tools.cc \
tools.h \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- tests/stub_StoreMeta.cc \
- StoreMetaUnpacker.cc \
StoreSwapLogData.cc \
StrList.cc \
StrList.h \
tests/stub_store_stats.cc \
store_swapin.cc \
store_swapin.h \
- store_swapmeta.cc \
store_swapout.cc \
tools.cc \
tools.h \
StatHist.h \
StoreFileSystem.cc \
StoreIOState.cc \
- tests/stub_StoreMeta.cc \
- StoreMetaUnpacker.cc \
StoreSwapLogData.cc \
StrList.cc \
StrList.h \
tests/stub_store_stats.cc \
store_swapin.cc \
store_swapin.h \
- store_swapmeta.cc \
store_swapout.cc \
tools.cc \
tools.h \
return var;
}
+/// converts a given non-negative integer into an integer of a given type
+/// without loss of information or undefined behavior
+template <typename Result, typename Source>
+Result
+NaturalCast(const Source s)
+{
+ return NaturalSum<Result>(s).value();
+}
+
#endif /* _SQUID_SRC_SQUIDMATH_H */
void scheduleMemRead();
void scheduleRead();
bool startSwapin();
- bool unpackHeader(char const *buf, ssize_t len);
void fail();
void callback(ssize_t);
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "base/Range.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMeta.h"
-#include "StoreMetaMD5.h"
-#include "StoreMetaObjSize.h"
-#include "StoreMetaSTD.h"
-#include "StoreMetaSTDLFS.h"
-#include "StoreMetaURL.h"
-#include "StoreMetaVary.h"
-
-bool
-StoreMeta::validType(char type)
-{
- /* VOID is reserved, and new types have to be added as classes */
- if (type <= STORE_META_VOID || type >= STORE_META_END + 10) {
- debugs(20, DBG_CRITICAL, "ERROR: storeSwapMetaUnpack: bad type (" << type << ")!");
- return false;
- }
-
- /* Not yet implemented */
- if (type >= STORE_META_END ||
- type == STORE_META_STOREURL ||
- type == STORE_META_VARY_ID) {
- debugs(20, 3, "storeSwapMetaUnpack: Not yet implemented (" << type << ") in disk metadata");
- return false;
- }
-
- /* Unused in any current squid code */
- if (type == STORE_META_KEY_URL ||
- type == STORE_META_KEY_SHA ||
- type == STORE_META_HITMETERING ||
- type == STORE_META_VALID) {
- debugs(20, DBG_CRITICAL, "ERROR: Obsolete and unused type (" << type << ") in disk metadata");
- return false;
- }
-
- return true;
-}
-
-const int StoreMeta::MinimumTLVLength = 0;
-const int StoreMeta::MaximumTLVLength = 1 << 16;
-
-bool
-StoreMeta::validLength(int aLength) const
-{
- static const Range<int> TlvValidLengths = Range<int>(StoreMeta::MinimumTLVLength, StoreMeta::MaximumTLVLength);
- if (!TlvValidLengths.contains(aLength)) {
- debugs(20, DBG_CRITICAL, MYNAME << ": insane length (" << aLength << ")!");
- return false;
- }
-
- return true;
-}
-
-StoreMeta *
-StoreMeta::Factory (char type, size_t len, void const *value)
-{
- if (!validType(type))
- return nullptr;
-
- StoreMeta *result;
-
- switch (type) {
-
- case STORE_META_KEY:
- result = new StoreMetaMD5;
- break;
-
- case STORE_META_URL:
- result = new StoreMetaURL;
- break;
-
- case STORE_META_STD:
- result = new StoreMetaSTD;
- break;
-
- case STORE_META_STD_LFS:
- result = new StoreMetaSTDLFS;
- break;
-
- case STORE_META_OBJSIZE:
- result = new StoreMetaObjSize;
- break;
-
- case STORE_META_VARY_HEADERS:
- result = new StoreMetaVary;
- break;
-
- default:
- debugs(20, DBG_CRITICAL, "ERROR: Attempt to create unknown concrete StoreMeta");
- return nullptr;
- }
-
- if (!result->validLength(len)) {
- delete result;
- return nullptr;
- }
-
- result->length = len;
- result->value = xmalloc(len);
- memcpy(result->value, value, len);
- return result;
-}
-
-void
-StoreMeta::FreeList(StoreMeta **head)
-{
- StoreMeta *node;
-
- while ((node = *head) != nullptr) {
- *head = node->next;
- xfree(node->value);
- delete node;
- }
-}
-
-StoreMeta **
-StoreMeta::Add(StoreMeta **tail, StoreMeta *aNode)
-{
- assert (*tail == nullptr);
- *tail = aNode;
- return &aNode->next; /* return new tail pointer */
-}
-
-bool
-StoreMeta::checkConsistency(StoreEntry *) const
-{
- switch (getType()) {
-
- case STORE_META_KEY:
-
- case STORE_META_URL:
-
- case STORE_META_VARY_HEADERS:
- assert(0);
- break;
-
- case STORE_META_STD:
- break;
-
- case STORE_META_STD_LFS:
- break;
-
- case STORE_META_OBJSIZE:
- break;
-
- default:
- debugs(20, DBG_IMPORTANT, "WARNING: got unused STORE_META type " << getType());
- break;
- }
-
- return true;
-}
-
-StoreMeta::StoreMeta(const StoreMeta &s) :
- length(s.length),
- value(s.value),
- next(s.next)
-{}
-
-StoreMeta& StoreMeta::operator=(const StoreMeta &s)
-{
- length=s.length;
- value=s.value;
- next=s.next;
- return *this;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_TYPELENGTHVALUE_H
-#define SQUID_TYPELENGTHVALUE_H
-
-class StoreEntry;
-
-// WTF?
-typedef class StoreMeta tlv;
-
-/**
- \ingroup SwapStoreAPI
- * XXX: for critical lists like this we should use A=64,B=65 etc to enforce and reserve values.
- \note NOTE! We must preserve the order of this list!
- *
- \section StoreSwapMeta Store "swap meta" Description
- \par
- * "swap meta" refers to a section of meta data stored at the beginning
- * of an object that is stored on disk. This meta data includes information
- * such as the object's cache key (MD5), URL, and part of the StoreEntry
- * structure.
- *
- \par
- * The meta data is stored using a TYPE-LENGTH-VALUE format. That is,
- * each chunk of meta information consists of a TYPE identifier, a
- * LENGTH field, and then the VALUE (which is LENGTH octets long).
- */
-enum {
- /**
- * Just a placeholder for the zeroth value. It is never used on disk.
- */
- STORE_META_VOID,
-
- /**
- \deprecated
- * This represents the case when we use the URL as the cache
- * key, as Squid-1.1 does. Currently we don't support using
- * a URL as a cache key, so this is not used.
- */
- STORE_META_KEY_URL,
-
- /**
- \deprecated
- * For a brief time we considered supporting SHA (secure
- * hash algorithm) as a cache key. Nobody liked it, and
- * this type is not currently used.
- */
- STORE_META_KEY_SHA,
-
- /**
- * This represents the MD5 cache key that Squid currently uses.
- * When Squid opens a disk file for reading, it can check that
- * this MD5 matches the MD5 of the user's request. If not, then
- * something went wrong and this is probably the wrong object.
- */
- STORE_META_KEY_MD5,
-
- /**
- * The object's URL. This also may be matched against a user's
- * request for cache hits to make sure we got the right object.
- */
- STORE_META_URL,
-
- /**
- * This is the "standard metadata" for an object.
- * Really its just this middle chunk of the StoreEntry structure:
- \code
- time_t timestamp;
- time_t lastref;
- time_t expires;
- time_t lastmod;
- uint64_t swap_file_sz;
- uint16_t refcount;
- uint16_t flags;
- \endcode
- */
- STORE_META_STD,
-
- /**
- * Reserved for future hit-metering (RFC 2227) stuff
- */
- STORE_META_HITMETERING,
-
- // TODO: document this TLV type code
- STORE_META_VALID,
-
- /**
- * Stores Vary request headers
- */
- STORE_META_VARY_HEADERS,
-
- /**
- * Updated version of STORE_META_STD, with support for >2GB objects.
- * As STORE_META_STD except that the swap_file_sz is a 64-bit integer instead of 32-bit.
- */
- STORE_META_STD_LFS,
-
- // TODO: document this TLV type code
- STORE_META_OBJSIZE,
-
- STORE_META_STOREURL, /**< the Store-ID url, if different to the normal URL */
- STORE_META_VARY_ID, /**< Unique ID linking variants */
- STORE_META_END
-};
-
-/// \ingroup SwapStoreAPI
-class StoreMeta
-{
-protected:
- StoreMeta() : length(-1), value(nullptr), next(nullptr) { }
- StoreMeta(const StoreMeta &);
- StoreMeta& operator=(const StoreMeta &);
-
-public:
- static bool validType(char);
- static int const MaximumTLVLength;
- static int const MinimumTLVLength;
- static StoreMeta *Factory(char type, size_t len, void const *value);
- static StoreMeta **Add(StoreMeta **tail, StoreMeta *aNode);
- static void FreeList(StoreMeta **head);
-
- virtual char getType() const = 0;
- virtual bool validLength(int) const;
- virtual bool checkConsistency(StoreEntry *) const;
- virtual ~StoreMeta() {}
-
- int length;
- void *value;
- tlv *next;
-};
-
-/// \ingroup SwapStoreAPI
-char *storeSwapMetaPack(tlv * tlv_list, int *length);
-/// \ingroup SwapStoreAPI
-tlv *storeSwapMetaBuild(const StoreEntry *);
-/// \ingroup SwapStoreAPI
-void storeSwapTLVFree(tlv * n);
-
-#endif /* SQUID_TYPELENGTHVALUE_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "int.h"
-#include "md5.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMetaMD5.h"
-
-bool
-StoreMetaMD5::validLength(int len) const
-{
- return len == SQUID_MD5_DIGEST_LENGTH;
-}
-
-int StoreMetaMD5::md5_mismatches = 0;
-
-bool
-StoreMetaMD5::checkConsistency(StoreEntry *e) const
-{
- assert (getType() == STORE_META_KEY_MD5);
- assert(length == SQUID_MD5_DIGEST_LENGTH);
-
- if (!EBIT_TEST(e->flags, KEY_PRIVATE) &&
- memcmp(value, e->key, SQUID_MD5_DIGEST_LENGTH)) {
- debugs(20, 2, "storeClientReadHeader: swapin MD5 mismatch");
- // debugs(20, 2, "\t" << storeKeyText((const cache_key *)value));
- debugs(20, 2, "\t" << e->getMD5Text());
-
- if (isPowTen(++md5_mismatches))
- debugs(20, DBG_IMPORTANT, "WARNING: " << md5_mismatches << " swapin MD5 mismatches");
-
- return false;
- }
-
- return true;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETAMD5_H
-#define SQUID_STOREMETAMD5_H
-
-#include "StoreMeta.h"
-/* for STORE_META_KEY_MD5 */
-#include "enums.h"
-
-class StoreMetaMD5 : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaMD5);
-
-public:
- char getType() const {return STORE_META_KEY_MD5;}
-
- bool validLength(int) const;
- bool checkConsistency(StoreEntry *) const;
-
-private:
- static int md5_mismatches;
-};
-
-#endif /* SQUID_STOREMETAMD5_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETAOBJSIZE_H
-#define SQUID_STOREMETAOBJSIZE_H
-
-#include "StoreMeta.h"
-
-class StoreMetaObjSize : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaObjSize);
-
-public:
- char getType() const {return STORE_META_OBJSIZE;}
-};
-
-#endif /* SQUID_STOREMETAOBJSIZE_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMetaSTD.h"
-
-bool
-StoreMetaSTD::validLength(int len) const
-{
- return len == STORE_HDR_METASIZE_OLD;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETASTD_H
-#define SQUID_STOREMETASTD_H
-
-#include "StoreMeta.h"
-
-class StoreMetaSTD : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaSTD);
-
-public:
- char getType() const {return STORE_META_STD;}
-
- bool validLength(int) const;
- // bool checkConsistency(StoreEntry *) const;
-};
-
-#endif /* SQUID_STOREMETASTD_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMetaSTDLFS.h"
-
-bool
-StoreMetaSTDLFS::validLength(int len) const
-{
- return len == STORE_HDR_METASIZE;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETASTDLFS_H
-#define SQUID_STOREMETASTDLFS_H
-
-#include "StoreMeta.h"
-
-class StoreMetaSTDLFS : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaSTDLFS);
-
-public:
- char getType() const {return STORE_META_STD_LFS;}
-
- bool validLength(int) const;
- // bool checkConsistency(StoreEntry *) const;
-};
-
-#endif /* SQUID_STOREMETASTDLFS_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMetaURL.h"
-
-bool
-StoreMetaURL::checkConsistency(StoreEntry *e) const
-{
- assert (getType() == STORE_META_URL);
-
- if (!e->mem_obj->hasUris())
- return true;
-
- if (strcasecmp(e->mem_obj->urlXXX(), (char *)value)) {
- debugs(20, DBG_IMPORTANT, "storeClientReadHeader: URL mismatch");
- debugs(20, DBG_IMPORTANT, "\t{" << (char *) value << "} != {" << e->mem_obj->urlXXX() << "}");
- return false;
- }
-
- return true;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETAURL_H
-#define SQUID_STOREMETAURL_H
-
-#include "StoreMeta.h"
-
-class StoreMetaURL : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaURL);
-
-public:
- char getType() const {return STORE_META_URL;}
-
- bool checkConsistency(StoreEntry *) const;
-};
-
-#endif /* SQUID_STOREMETAURL_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Unpacker */
-
-#include "squid.h"
-#include "base/TextException.h"
-#include "debug/Stream.h"
-#include "defines.h"
-#include "StoreMeta.h"
-#include "StoreMetaUnpacker.h"
-
-int const StoreMetaUnpacker::MinimumBufferLength = sizeof(char) + sizeof(int);
-
-/// useful for meta stored in pre-initialized (with zeros) db files
-bool
-StoreMetaUnpacker::isBufferZero()
-{
- // We could memcmp the entire buffer, but it is probably safe enough
- // to test a few bytes because if we do not detect a corrupted entry
- // it is not a big deal. Empty entries are not isBufferSane anyway.
- const int depth = 10;
- if (buflen < depth)
- return false; // cannot be sure enough
-
- for (int i = 0; i < depth; ++i) {
- if (buf[i])
- return false;
- }
- return true;
-}
-
-void
-StoreMetaUnpacker::checkBuffer()
-{
- assert(buf); // paranoid; already checked in the constructor
- if (buf[0] != static_cast<char>(STORE_META_OK))
- throw TexcHere("store entry metadata is corrupted");
- /*
- * sanity check on 'buflen' value. It should be at least big
- * enough to hold one type and one length.
- */
- getBufferLength();
- if (*hdr_len < MinimumBufferLength)
- throw TexcHere("store entry metadata is too small");
- if (*hdr_len > buflen)
- throw TexcHere("store entry metadata is too big");
-}
-
-void
-StoreMetaUnpacker::getBufferLength()
-{
- memcpy(hdr_len, &buf[1], sizeof(int));
-}
-
-StoreMetaUnpacker::StoreMetaUnpacker(char const *aBuffer, ssize_t aLen, int *anInt) :
- buf(aBuffer),
- buflen(aLen),
- hdr_len(anInt),
- position(1 + sizeof(int)),
- type('\0'),
- length(0),
- tail(nullptr)
-{
- assert(aBuffer != nullptr);
-}
-
-void
-StoreMetaUnpacker::getType()
-{
- type = buf[position];
- ++position;
-}
-
-void
-StoreMetaUnpacker::getLength()
-{
- memcpy(&length, &buf[position], sizeof(int));
- position += sizeof(int);
-}
-
-bool
-StoreMetaUnpacker::doOneEntry()
-{
- getType();
- getLength();
-
- if (position + length > *hdr_len) {
- debugs(20, DBG_CRITICAL, "storeSwapMetaUnpack: overflow!");
- debugs(20, DBG_CRITICAL, "\ttype=" << type << ", length=" << length << ", *hdr_len=" << *hdr_len << ", offset=" << position);
- return false;
- }
-
- StoreMeta *newNode = StoreMeta::Factory(type, length, &buf[position]);
-
- if (newNode)
- tail = StoreMeta::Add (tail, newNode);
-
- position += length;
-
- return true;
-}
-
-bool
-StoreMetaUnpacker::moreToProcess() const
-{
- return *hdr_len - position - MinimumBufferLength >= 0;
-}
-
-StoreMeta *
-StoreMetaUnpacker::createStoreMeta ()
-{
- tlv *TLV = nullptr;
- tail = &TLV;
- assert(hdr_len != nullptr);
-
- checkBuffer();
-
- getBufferLength();
-
- assert (position == 1 + sizeof(int));
-
- while (moreToProcess()) {
- if (!doOneEntry())
- break;
- }
-
- if (!TLV)
- throw TexcHere("store entry metadata is empty");
-
- assert(TLV);
- return TLV;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_TYPELENGTHVALUEUNPACKER_H
-#define SQUID_TYPELENGTHVALUEUNPACKER_H
-
-class StoreMeta;
-class StoreEntry;
-
-class StoreMetaUnpacker
-{
-
-public:
- StoreMetaUnpacker (const char *buf, ssize_t bufferLength, int *hdrlen);
- StoreMeta *createStoreMeta();
- bool isBufferZero(); ///< all-zeros buffer, checkBuffer() would throw
- /// validates buffer sanity and throws if validation fails
- void checkBuffer();
-
-private:
- static int const MinimumBufferLength;
-
- void getBufferLength();
- void getType();
- void getLength();
- void getTLV();
- bool doOneEntry();
- bool moreToProcess() const;
-
- char const * const buf;
- ssize_t buflen;
- int *hdr_len;
- int position;
- char type;
- int length;
- StoreMeta **tail;
-};
-
-#endif /* SQUID_TYPELENGTHVALUEUNPACKER_H */
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMetaVary.h"
-
-bool
-StoreMetaVary::checkConsistency(StoreEntry *e) const
-{
- assert (getType() == STORE_META_VARY_HEADERS);
-
- if (e->mem_obj->vary_headers.isEmpty()) {
- /* XXX separate this mutator from the query */
- /* Assume the object is OK.. remember the vary request headers */
- e->mem_obj->vary_headers.assign(static_cast<const char *>(value), length);
- /* entries created before SBuf vary handling may include string terminator */
- static const SBuf nul("\0", 1);
- e->mem_obj->vary_headers.trim(nul);
- return true;
- }
-
- if (e->mem_obj->vary_headers.cmp(static_cast<const char *>(value), length) != 0)
- return false;
-
- return true;
-}
-
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#ifndef SQUID_STOREMETAVARY_H
-#define SQUID_STOREMETAVARY_H
-
-#include "StoreMeta.h"
-
-class StoreMetaVary : public StoreMeta
-{
- MEMPROXY_CLASS(StoreMetaVary);
-
-public:
- char getType() const {return STORE_META_VARY_HEADERS;}
-
- bool checkConsistency(StoreEntry *) const;
-};
-
-#endif /* SQUID_STOREMETAVARY_H */
-
\section ImplementationNotes Implementation Notes
\par
* When writing an object to disk, we must first write the meta data.
- * This is done with a couple of functions. First, storeSwapMetaPack()
- * takes a StoreEntry as a parameter and returns a tlv linked
- * list. Second, storeSwapMetaPack() converts the tlv list
+ * Store::PackSwapMeta() serializes that meta data
* into a character buffer that we can write.
*
\note MemObject has a MemObject::swap_hdr_sz.
\note The swap file content includes the HTTP reply headers and the HTTP reply body (if any).
*
\par
- * When reading a swap file, there is a similar process to extract
- * the swap meta data. First, storeSwapMetaUnpack() converts a
- * character buffer into a tlv linked list. It also tells us
+ * When reading a swap file, Store::Unpack*SwapMeta*() functions iterate
+ * over stored swap meta data header fields and also extract
* the value for MemObject->swap_hdr_sz.
*/
};
/// The maximum used DebugMessage::id plus 1. Increase as you add new IDs.
-constexpr DebugMessageId DebugMessageIdUpperBound = 65;
+constexpr DebugMessageId DebugMessageIdUpperBound = 66;
/// a collection of DebugMessage objects (with fast access by message IDs)
class DebugMessages
#define CLIENT_REQ_BUF_SZ 4096
-#define STORE_META_OK 0x03
-
#define IPC_NONE 0
#define IPC_TCP_SOCKET 1
#define IPC_UDP_SOCKET 2
#define IPC_DGRAM IPC_UDP_SOCKET
#endif
-#define STORE_META_KEY STORE_META_KEY_MD5
-
-#define STORE_HDR_METASIZE (4*sizeof(time_t)+2*sizeof(uint16_t)+sizeof(uint64_t))
-#define STORE_HDR_METASIZE_OLD (4*sizeof(time_t)+2*sizeof(uint16_t)+sizeof(size_t))
-
#define COUNT_INTERVAL 60
/*
* keep 60 minutes' worth of per-minute readings (+ current reading)
#include "fs/rock/RockIoState.h"
#include "mime_header.h"
#include "Store.h"
-#include "StoreMetaUnpacker.h"
+#include "store/SwapMetaIn.h"
CBDATA_NAMESPACED_CLASS_INIT(Rock, HeaderUpdater);
Rock::HeaderUpdater::parseReadBytes()
{
if (!staleSwapHeaderSize) {
- StoreMetaUnpacker aBuilder(
- exchangeBuffer.rawContent(),
- exchangeBuffer.length(),
- &staleSwapHeaderSize);
+ staleSwapHeaderSize = Store::UnpackSwapMetaSize(exchangeBuffer);
// Squid assumes that metadata always fits into a single db slot
- aBuilder.checkBuffer(); // cannot update an entry with invalid metadata
debugs(47, 7, "staleSwapHeaderSize=" << staleSwapHeaderSize);
Must(staleSwapHeaderSize > 0);
exchangeBuffer.consume(staleSwapHeaderSize);
SBuf exchangeBuffer; ///< bytes read but not yet discarded or written
uint64_t bytesRead; ///< total entry bytes read from Store so far
- int staleSwapHeaderSize; ///< stored size of the stale entry metadata
+ size_t staleSwapHeaderSize; ///< stored size of the stale entry metadata
SlotId staleSplicingPointNext; ///< non-updatable old HTTP body suffix start
};
#include "Store.h"
#include "tools.h"
+#include <array>
#include <cerrno>
+#include <cstring>
CBDATA_NAMESPACED_CLASS_INIT(Rock, Rebuild);
useNewSlot(slotId, header);
}
+/// whether the given slot buffer is likely to have nothing but zeros, as is
+/// common to slots in pre-initialized (with zeros) db files
+static bool
+ZeroedSlot(const MemBuf &buf)
+{
+ // We could memcmp the entire buffer, but it is probably safe enough to test
+ // a few bytes because even if we do not detect a corrupted entry, it is not
+ // a big deal: Store::UnpackPrefix() rejects all-0s metadata prefix.
+ static const std::array<char, 10> zeros = {};
+
+ if (static_cast<size_t>(buf.contentSize()) < zeros.size())
+ return false; // cannot be sure enough
+
+ return memcmp(buf.content(), zeros.data(), zeros.size()) == 0;
+}
+
/// parse StoreEntry basics and add them to the map, returning true on success
bool
Rock::Rebuild::importEntry(Ipc::StoreMapAnchor &anchor, const sfileno fileno, const DbCellHeader &header)
StoreEntry loadedE;
const uint64_t knownSize = header.entrySize > 0 ?
header.entrySize : anchor.basics.swap_file_sz.load();
+
+ if (ZeroedSlot(buf))
+ return false;
+
if (!storeRebuildParseEntry(buf, loadedE, key, counts, knownSize))
return false;
#include "store/Controller.h"
#include "store/Disk.h"
#include "store/Disks.h"
+#include "store/SwapMetaOut.h"
#include "store_digest.h"
#include "store_key_md5.h"
#include "store_log.h"
#include "store_rebuild.h"
#include "StoreClient.h"
#include "StoreIOState.h"
-#include "StoreMeta.h"
#include "StrList.h"
#include "swap_log_op.h"
#include "tools.h"
void
storeInit(void)
{
- storeKeyInit();
mem_policy = createRemovalPolicy(Config.memPolicy);
storeDigestInit();
storeLogOpen();
char const *
StoreEntry::getSerialisedMetaData(size_t &length) const
{
- StoreMeta *tlv_list = storeSwapMetaBuild(this);
- int swap_hdr_sz;
- char *result = storeSwapMetaPack(tlv_list, &swap_hdr_sz);
- storeSwapTLVFree(tlv_list);
- assert (swap_hdr_sz >= 0);
- length = static_cast<size_t>(swap_hdr_sz);
- return result;
+ return static_cast<const char *>(Store::PackSwapMeta(*this, length).release());
}
/**
LocalSearch.cc \
LocalSearch.h \
Storage.h \
+ SwapMeta.cc \
+ SwapMeta.h \
+ SwapMetaIn.cc \
+ SwapMetaIn.h \
+ SwapMetaOut.cc \
+ SwapMetaOut.h \
+ SwapMetaView.cc \
+ SwapMetaView.h \
forward.h
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#include "squid.h"
+#include "store/SwapMeta.h"
+
+void
+Store::CheckSwapMetaSerialization(const RawSwapMetaType type, const RawSwapMetaLength length, const void *value)
+{
+ // we do not serialize deprecated or reserved types
+ assert(HonoredSwapMetaType(type));
+
+ assert(length >= 0);
+ assert(size_t(length) <= SwapMetaFieldValueLengthMax);
+
+ // cannot write a non-empty value if it is missing
+ assert(!length || value);
+}
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_SRC_STORE_SWAPMETA_H
+#define SQUID_SRC_STORE_SWAPMETA_H
+
+#include "defines.h"
+
+#include <limits>
+
+// store swap metadata interfaces shared by input and output code
+
+namespace Store {
+
+/// "Swap meta" (a.k.a. "swap header") is Store entry metadata stored at the
+/// beginning of each cache_dir entry. This entry-dependent metadata typically
+/// includes entry cache key, URL, size, and various timestamps. Copies of this
+/// information may also be present in Squid process memory, readily accessible
+/// via StoreEntry and MemObject pointers, but swap metadata is about cached
+/// entry information stored in serialized form meant to cross process and
+/// instance boundaries.
+///
+/// Layout of swap metadata (for a single Store entry) in pseudo code:
+/// struct SwapMeta {
+/// struct Prefix {
+/// char magic; // see SwapMetaMagic
+/// int swap_hdr_sz; // total SwapMeta size: prefix and all fields
+/// };
+/// Prefix prefix;
+///
+/// struct TLV {
+/// char type; // value meaning (and format); see SwapMetaType
+/// int length; // value length in octets
+/// char value[length]; // type-specific storage; exactly length bytes
+/// };
+/// TLV fields[]; // as many swap meta fields as swap_hdr_sz accommodates
+///
+/// // XXX: int fields above should have been using a fixed-size type.
+/// };
+///
+/// Stored response (e.g., HTTP headers and body) follows swap metadata.
+
+/// Identifies the meaning (and associated format) of a single swap meta field.
+/// This enumeration only contains identifiers used by the current code.
+/// The gaps in the enum item values below represent deprecated/reserved IDs.
+/// \sa DeprecatedSwapMetaType(), ReservedSwapMetaType()
+enum SwapMetaType {
+ /// An invalid swap metadata field or field with an unknown meaning.
+ /// Never used by swapout code.
+ STORE_META_VOID = 0,
+
+ /// This represents the MD5 cache key that Squid currently uses. When Squid
+ /// opens a disk file for reading, it can check that this MD5 matches the
+ /// MD5 of the user's request. If not, then something went wrong and this is
+ /// probably the wrong object.
+ STORE_META_KEY_MD5 = 3,
+
+ /// The object's URL. This also may be matched against a user's request for
+ /// cache hits to make sure we got the right object.
+ STORE_META_URL = 4,
+
+ /**
+ * This is the "standard metadata" for an object.
+ * Really its just this middle chunk of the StoreEntry structure:
+ \code
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ uint64_t swap_file_sz;
+ uint16_t refcount;
+ uint16_t flags;
+ \endcode
+ */
+ STORE_META_STD = 5,
+
+ /// Stores Vary request headers.
+ STORE_META_VARY_HEADERS = 8,
+
+ /// Modern STORE_META_STD version, with 64-bit swap_file_sz supporting
+ /// objects larger than 2GB.
+ STORE_META_STD_LFS = 9,
+
+ // TODO: Document this type after we start using it; see UnpackHitSwapMeta()
+ STORE_META_OBJSIZE = 10
+};
+
+/// The type of a serialized swap meta field part called "type" (i.e. T in TLV).
+/// Meant for storing the serialized version of SwapMetaType.
+using RawSwapMetaType = char;
+
+/// The type of a serialized swap meta field part called "length" (i.e. L in TLV).
+/// Valid values of this type do not include the size of T and L components.
+/// Low-level serialization code aside, we use size_t for swap meta field sizes.
+using RawSwapMetaLength = int;
+
+/// Store entries with larger (serialized) swap metadata field values are not
+/// swapped out and are considered invalid when validating being-loaded entries.
+/// This arbitrary limit protects code that adds individual swap metadata field
+/// sizes from overflowing and prevents allocation of huge buffers when loading
+/// variable-length fields. Reevaluate this limit when increasing MAX_URL.
+const size_t SwapMetaFieldValueLengthMax = 64*1024;
+
+static_assert(SwapMetaFieldValueLengthMax >= MAX_URL, "MAX_URL will fit in a swap meta field");
+static_assert(SwapMetaFieldValueLengthMax <= uint64_t(std::numeric_limits<RawSwapMetaLength>::max()), "any swap metadata value size can be stored as RawSwapMetaLength");
+
+/// the start of the swap meta section
+const char SwapMetaMagic = 0x03;
+
+/// The type of the serialized "metadata size" field that follows SwapMetaMagic.
+/// Together, the two fields form a swap metadata "prefix". The meaning of this
+/// size field is different from RawSwapMetaLength! Valid values of this field
+/// include the prefix size itself.
+using RawSwapMetaPrefixLength = int;
+
+/// The size of the initial (and required) portion of any swap metadata
+const auto SwapMetaPrefixSize = sizeof(SwapMetaMagic) + sizeof(RawSwapMetaPrefixLength);
+
+/// SwapMetaType IDs will never have this or smaller serialized value.
+/// This is not the smallest RawSwapMetaType value (that is usually -128).
+const RawSwapMetaType RawSwapMetaTypeBottom = 0;
+
+// TODO: Use "inline constexpr ..." with C++17.
+/// Maximum value of a serialized SwapMetaType ID.
+/// This is not the largest RawSwapMetaType value (that is usually +127).
+inline RawSwapMetaType
+RawSwapMetaTypeTop()
+{
+ // This "constant" switch forces developers to update this function when
+ // they add SwapMetaType values [-Wswitch]. It is better than an end_ enum
+ // marker because it does not force us to add that marker to every switch
+ // statement, with an assert(false) or similar "unreachable code" handler.
+ // Optimizing compilers optimize this statement away into a constant.
+ switch (STORE_META_VOID) {
+ case STORE_META_VOID:
+ case STORE_META_KEY_MD5:
+ case STORE_META_URL:
+ case STORE_META_STD:
+ case STORE_META_VARY_HEADERS:
+ case STORE_META_STD_LFS:
+ case STORE_META_OBJSIZE:
+ // always return the last/maximum enum value
+ return STORE_META_OBJSIZE;
+ }
+}
+
+/// Whether the given raw swap meta field type represents a type that we should
+/// inform the admin about (if found in a store) but can otherwise ignore.
+inline bool
+DeprecatedSwapMetaType(const RawSwapMetaType type)
+{
+ enum class DeprecatedMetas {
+ /// \deprecated Using URL as the cache key, as in Squid-1.1.
+ STORE_META_KEY_URL = 1,
+ /// \deprecated Using SHA (secure hash algorithm) as a cache key
+ STORE_META_KEY_SHA = 2,
+ /// \deprecated hit-metering (RFC 2227)
+ STORE_META_HITMETERING = 6,
+ /// \deprecated
+ STORE_META_VALID = 7
+ };
+ return
+ // TODO: simplify with std::underlying_type_t when switching to C++14
+ type == static_cast<RawSwapMetaType>(DeprecatedMetas::STORE_META_KEY_URL) ||
+ type == static_cast<RawSwapMetaType>(DeprecatedMetas::STORE_META_KEY_SHA) ||
+ type == static_cast<RawSwapMetaType>(DeprecatedMetas::STORE_META_HITMETERING) ||
+ type == static_cast<RawSwapMetaType>(DeprecatedMetas::STORE_META_VALID);
+}
+
+/// Whether the given raw swap meta field type represents a type that we should
+/// ignore without informing the admin.
+inline bool
+ReservedSwapMetaType(const RawSwapMetaType type)
+{
+ enum class ReservedMetas {
+ /// the Store-ID url, if different from the normal URL
+ STORE_META_STOREURL = 11,
+ /// unique ID linking variants
+ STORE_META_VARY_ID = 12
+ };
+ return
+ type == static_cast<RawSwapMetaType>(ReservedMetas::STORE_META_STOREURL) ||
+ type == static_cast<RawSwapMetaType>(ReservedMetas::STORE_META_VARY_ID);
+}
+
+/// Whether we store the given swap meta field type (and also interpret the
+/// corresponding swap meta field when the Store loads it). Matches all
+/// SwapMetaType enum values except for the never-stored STORE_META_VOID.
+inline bool
+HonoredSwapMetaType(const RawSwapMetaType type)
+{
+ switch (type) {
+ case STORE_META_VOID:
+ return false;
+
+ case STORE_META_KEY_MD5:
+ case STORE_META_URL:
+ case STORE_META_STD:
+ case STORE_META_VARY_HEADERS:
+ case STORE_META_STD_LFS:
+ case STORE_META_OBJSIZE:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
+/// Whether the given raw swap meta field type can be safely ignored.
+/// \sa HonoredSwapMetaType()
+inline bool
+IgnoredSwapMetaType(const RawSwapMetaType type)
+{
+ return DeprecatedSwapMetaType(type) || ReservedSwapMetaType(type);
+}
+
+/// Expected size of a STORE_META_STD_LFS swap meta field. XXX: The actual size
+/// is environment-specific due to 4 parts that do not use fixed-size types.
+const auto STORE_HDR_METASIZE =
+ 4*sizeof(time_t) + sizeof(uint64_t) + 2*sizeof(uint16_t);
+
+/// Expected size of a STORE_META_STD swap meta field. XXX: The actual size is
+/// environment-specific due to 5 parts that do not use fixed-size types.
+const auto STORE_HDR_METASIZE_OLD =
+ 4*sizeof(time_t) + sizeof(size_t) + 2*sizeof(uint16_t);
+
+/// Ensures that the given serialized swap meta field is valid and can be
+/// subsequently de-serialized (by the same code). Also detects some failures to
+/// update one of the classification functions above when editing SwapMetaType.
+void CheckSwapMetaSerialization(RawSwapMetaType, RawSwapMetaLength, const void *);
+
+} // namespace Store
+
+#endif /* SQUID_SRC_STORE_SWAPMETA_H */
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#include "squid.h"
+#include "base/Raw.h"
+#include "base/TextException.h"
+#include "int.h"
+#include "md5.h"
+#include "MemObject.h"
+#include "sbuf/SBuf.h"
+#include "sbuf/Stream.h"
+#include "SquidMath.h"
+#include "Store.h"
+#include "store/SwapMeta.h"
+#include "store/SwapMetaIn.h"
+#include "store/SwapMetaView.h"
+
+namespace Store {
+
+/// iterates serialized swap meta fields loaded into a given buffer
+class SwapMetaIterator
+{
+public:
+ /* some of the standard iterator traits */
+ using iterator_category = std::forward_iterator_tag;
+ using value_type = const SwapMetaView;
+ using pointer = value_type *;
+ using reference = value_type &;
+
+ /// positions iterator at the start of a swap meta field extending up to end
+ SwapMetaIterator(const void *start, const void *end);
+
+ /* some of the standard iterator methods */
+ reference operator *() const { return meta_; }
+ pointer operator ->() const { return &meta_; }
+ SwapMetaIterator& operator++();
+ bool operator ==(const SwapMetaIterator &them) const { return fieldStart_ == them.fieldStart_; }
+ bool operator !=(const SwapMetaIterator &them) const { return !(*this == them); }
+
+private:
+ void sync();
+
+ const char *fieldStart_; ///< the start of the current field
+ const void * const bufEnd_; ///< last field must end at this boundary
+ SwapMetaView meta_; ///< current field; valid after sync() and before end
+};
+
+/// Store entry metadata view providing a for-range loop meta field iterator API
+class SwapMetaUnpacker
+{
+public:
+ SwapMetaUnpacker(const char *buf, ssize_t bufferLength, size_t &swap_hdr_sz);
+
+ // for-range loop API for iterating over serialized swap metadata fields
+ using Iterator = SwapMetaIterator;
+ Iterator cbegin() const { return Iterator(metas, metas + metasSize); }
+ Iterator cend() const { return Iterator(metas + metasSize, metas + metasSize); }
+ Iterator begin() const { return cbegin(); }
+ Iterator end() const { return cend(); }
+
+private:
+ const char *metas; ///< metadata field(s)
+ size_t metasSize; ///< number of bytes in the metas buffer
+};
+
+/// validates serialized STORE_META_KEY_MD5 swap meta field
+static void
+CheckSwapMetaKey(const SwapMetaView &meta, const StoreEntry &entry)
+{
+ Assure(meta.type == STORE_META_KEY_MD5);
+ meta.checkExpectedLength(SQUID_MD5_DIGEST_LENGTH);
+
+ if (!EBIT_TEST(entry.flags, KEY_PRIVATE) &&
+ memcmp(meta.rawValue, entry.key, SQUID_MD5_DIGEST_LENGTH) != 0) {
+
+ debugs(20, 2, "stored key mismatches " << entry.getMD5Text());
+
+ static unsigned int md5_mismatches = 0;
+ if (isPowTen(++md5_mismatches))
+ debugs(20, DBG_IMPORTANT, "WARNING: " << md5_mismatches << " swapin MD5 mismatches");
+
+ // TODO: Support TextException::frequent = isPowTen(++md5_mismatches)
+ // to suppress reporting, achieving the same effect as above
+ throw TextException("swap meta MD5 mismatch", Here());
+ }
+}
+
+/// deserializes STORE_META_KEY_MD5 swap meta field
+static void
+UnpackSwapMetaKey(const SwapMetaView &meta, cache_key *key)
+{
+ Assure(meta.type == STORE_META_KEY_MD5);
+ meta.checkExpectedLength(SQUID_MD5_DIGEST_LENGTH);
+ Assure(key);
+ memcpy(key, meta.rawValue, SQUID_MD5_DIGEST_LENGTH);
+}
+
+/// validates serialized STORE_META_URL swap meta field
+static void
+CheckSwapMetaUrl(const SwapMetaView &meta, const StoreEntry &entry)
+{
+ Assure(meta.type == STORE_META_URL);
+
+ // PackSwapMeta() terminates; strcasecmp() and reporting below rely on that
+ if (!memrchr(meta.rawValue, '\0', meta.rawLength))
+ throw TextException("unterminated URI or bad URI length", Here());
+
+ const auto &emem = entry.mem();
+
+ if (!emem.hasUris())
+ return; // cannot validate
+
+ const auto storedUrl = static_cast<const char *>(meta.rawValue);
+ // XXX: ensure all Squid URL inputs are properly normalized then use case-sensitive compare here
+ if (strcasecmp(emem.urlXXX(), storedUrl) != 0) {
+ debugs(20, DBG_IMPORTANT, "WARNING: URL mismatch when loading a cached entry:" <<
+ Debug::Extra << "expected: " << emem.urlXXX() <<
+ Debug::Extra << "found: " << storedUrl);
+ throw TextException("URL mismatch", Here());
+ }
+}
+
+/// deserializes STORE_META_VARY_HEADERS swap meta field
+static SBuf
+UnpackNewSwapMetaVaryHeaders(const SwapMetaView &meta, const StoreEntry &entry)
+{
+ Assure(meta.type == STORE_META_VARY_HEADERS);
+ SBuf rawVary(static_cast<const char *>(meta.rawValue), meta.rawLength);
+ // entries created before SBuf-based Vary may include string terminator
+ static const SBuf nul("\0", 1);
+ rawVary.trim(nul, false, true);
+
+ const auto &knownVary = entry.mem().vary_headers;
+ if (knownVary.isEmpty())
+ return rawVary; // new Vary (that we cannot validate)
+
+ if (knownVary == rawVary)
+ return SBuf(); // OK: no new Vary
+
+ throw TextException("Vary mismatch", Here());
+}
+
+/// deserializes entry metadata size from the given buffer
+/// \retval total swap metadata size (a.k.a. swap_hdr_sz)
+static size_t
+UnpackPrefix(const char * const buf, const size_t size)
+{
+ Assure(buf);
+ auto input = buf;
+ const auto end = buf + size;
+
+ char magic = 0;
+ SwapMetaExtract(magic, input, end);
+
+ if (magic != SwapMetaMagic)
+ throw TextException("store entry metadata prefix is corrupted", Here());
+
+ RawSwapMetaPrefixLength rawMetaSize = 0; // metadata size, including the required prefix
+ SwapMetaExtract(rawMetaSize, input, end);
+
+ if (Less(rawMetaSize, SwapMetaPrefixSize))
+ throw TextException("store entry metadata length is corrupted", Here());
+
+ return rawMetaSize; // now safe to use within (buf, buf+size)
+}
+
+} // namespace Store
+
+/* Store::SwapMetaIterator */
+
+Store::SwapMetaIterator::SwapMetaIterator(const void * const start, const void * const end):
+ fieldStart_(static_cast<const char*>(start)),
+ bufEnd_(end)
+{
+ sync();
+}
+
+Store::SwapMetaIterator &
+Store::SwapMetaIterator::operator++()
+{
+ Assure(fieldStart_ != bufEnd_);
+ fieldStart_ += sizeof(RawSwapMetaType); // swap meta type
+ fieldStart_ += sizeof(RawSwapMetaLength); // swap meta value length
+ fieldStart_ += meta_.rawLength; // swap meta value
+
+ sync();
+ return *this;
+}
+
+/// (re)set meta_
+void
+Store::SwapMetaIterator::sync()
+{
+ if (fieldStart_ == bufEnd_)
+ return; // nothing to do when we reach the end of iteration
+
+ // We cannot start beyond the end of the header: We start with valid
+ // begin/end buffer pointers, and each field checks for overreach.
+ Assure(fieldStart_ < bufEnd_);
+
+ meta_ = SwapMetaView(fieldStart_, bufEnd_);
+}
+
+/* Store::SwapMetaUnpacker */
+
+Store::SwapMetaUnpacker::SwapMetaUnpacker(const char * const buf, const ssize_t size, size_t &swap_hdr_sz)
+{
+ Assure(buf);
+ Assure(size >= 0);
+
+ const auto headerSize = UnpackPrefix(buf, size);
+
+ // We assume the caller supplied a reasonable-size buffer. If our assumption
+ // is wrong, then this is a Squid bug rather than input validation failure.
+ if (Less(size, headerSize)) {
+ throw TextException(ToSBuf("store entry metadata is too big",
+ Debug::Extra, "buffer size: ", size,
+ Debug::Extra, "metadata size: ", headerSize),
+ Here());
+ }
+
+ Assure2(headerSize >= SwapMetaPrefixSize, "UnpackPrefix() validates metadata length");
+ metasSize = headerSize - SwapMetaPrefixSize;
+
+ metas = buf + SwapMetaPrefixSize; // skip prefix
+ Assure(metas + metasSize <= buf + size); // paranoid
+
+ swap_hdr_sz = headerSize;
+}
+
+size_t
+Store::UnpackSwapMetaSize(const SBuf &buf)
+{
+ return UnpackPrefix(buf.rawContent(), buf.length());
+}
+
+size_t
+Store::UnpackIndexSwapMeta(const MemBuf &buf, StoreEntry &tmpe, cache_key * const key)
+{
+ size_t swap_hdr_sz = 0;
+
+ const SwapMetaUnpacker metaFields(buf.content(), buf.contentSize(), swap_hdr_sz);
+ for (const auto &meta: metaFields) {
+ switch (meta.type) {
+ case STORE_META_VOID:
+ // this meta.type is the unpacking code signal that it took care of this field
+ break;
+
+ case STORE_META_KEY_MD5:
+ // Optimization: We could postpone setting the caller's key
+ // until all fields are parsed, but that would require copying
+ // it. Instead, we treat key and tmpe.key as storage that can be
+ // safely altered even on parsing failures. This function
+ // description tells the callers that we may do that.
+ UnpackSwapMetaKey(meta, key);
+ Assure(key);
+ tmpe.key = key;
+ break;
+
+ case STORE_META_STD: {
+ // TODO: Remove. Since old_metahdr's members may have different
+ // sizes on different platforms, we cannot guarantee that serialized
+ // types in the being-loaded old cache are the same as these types.
+ meta.checkExpectedLength(STORE_HDR_METASIZE_OLD);
+ struct old_metahdr {
+ // XXX: All serialized members must have fixed-size types.
+ time_t timestamp;
+ time_t lastref;
+ time_t expires;
+ time_t lastmod;
+ size_t swap_file_sz;
+ uint16_t refcount;
+ uint16_t flags;
+ };
+ static_assert(offsetof(old_metahdr, flags) + sizeof(old_metahdr::flags) == STORE_HDR_METASIZE_OLD, "we reproduced old swap meta basics format");
+ auto basics = static_cast<const old_metahdr*>(meta.rawValue);
+ tmpe.timestamp = basics->timestamp;
+ tmpe.lastref = basics->lastref;
+ tmpe.expires = basics->expires;
+ tmpe.lastModified(basics->lastmod);
+ tmpe.swap_file_sz = basics->swap_file_sz;
+ tmpe.refcount = basics->refcount;
+ tmpe.flags = basics->flags;
+ break;
+ }
+
+ case STORE_META_STD_LFS:
+ meta.checkExpectedLength(STORE_HDR_METASIZE);
+ memcpy(&tmpe.timestamp, meta.rawValue, STORE_HDR_METASIZE);
+ break;
+
+ case STORE_META_URL:
+ case STORE_META_VARY_HEADERS:
+ case STORE_META_OBJSIZE:
+ // We do not load this information at cache index rebuild time;
+ // UnpackHitSwapMeta() handles these MemObject fields.
+ break;
+ }
+ }
+
+ return swap_hdr_sz;
+}
+
+void
+Store::UnpackHitSwapMeta(char const * const buf, const ssize_t len, StoreEntry &entry)
+{
+ debugs(90, 7, entry << " buf len: " << len);
+ assert(len >= 0);
+
+ size_t swap_hdr_sz = 0;
+ SBuf varyHeaders;
+
+ const SwapMetaUnpacker metaFields(buf, len, swap_hdr_sz);
+ for (const auto &meta: metaFields) {
+ switch (meta.type) {
+ case STORE_META_VOID:
+ // this meta.type is the unpacking code signal that it took care of the field
+ break;
+
+ case STORE_META_URL:
+ CheckSwapMetaUrl(meta, entry);
+ break;
+
+ case STORE_META_VARY_HEADERS:
+ varyHeaders = UnpackNewSwapMetaVaryHeaders(meta, entry);
+ break;
+
+ case STORE_META_OBJSIZE:
+ // XXX: We swap out but never use this field; set emem.object_sz?
+ break;
+
+ case STORE_META_KEY_MD5:
+ // already handled by UnpackIndexSwapMeta()
+ CheckSwapMetaKey(meta, entry); // paranoid
+ break;
+
+ case STORE_META_STD:
+ case STORE_META_STD_LFS:
+ // already handled by UnpackIndexSwapMeta()
+ break;
+ }
+ }
+
+ auto &emem = entry.mem();
+
+ emem.swap_hdr_sz = swap_hdr_sz;
+ if (entry.swap_file_sz > 0) { // collapsed hits may not know swap_file_sz
+ Assure(entry.swap_file_sz >= swap_hdr_sz);
+ emem.object_sz = entry.swap_file_sz - swap_hdr_sz;
+ }
+ debugs(90, 5, "swap_file_sz=" << entry.swap_file_sz <<
+ " (" << swap_hdr_sz << " + " << emem.object_sz << ")");
+
+ if (!varyHeaders.isEmpty())
+ emem.vary_headers = varyHeaders;
+}
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_SRC_STORE_SWAPMETAIN_H
+#define SQUID_SRC_STORE_SWAPMETAIN_H
+
+#include "sbuf/forward.h"
+#include "store/forward.h"
+
+class MemBuf;
+
+namespace Store {
+
+/// deserializes entry metadata size from the given buffer
+/// \retval total swap metadata size (a.k.a. swap_hdr_len)
+size_t UnpackSwapMetaSize(const SBuf &);
+
+/// deserializes entry metadata from the given buffer into the cache index entry
+/// \retval total swap metadata size (a.k.a. swap_hdr_len)
+size_t UnpackIndexSwapMeta(const MemBuf &, StoreEntry &, cache_key *);
+
+/// deserializes entry metadata from the given buffer into the cache hit entry
+void UnpackHitSwapMeta(char const *, ssize_t, StoreEntry &);
+
+} // namespace Store
+
+#endif /* SQUID_SRC_STORE_SWAPMETAIN_H */
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#include "squid.h"
+#include "md5.h"
+#include "MemObject.h"
+#include "sbuf/Stream.h"
+#include "SquidMath.h"
+#include "Store.h"
+#include "store/SwapMeta.h"
+#include "store/SwapMetaOut.h"
+
+namespace Store {
+
+/// writes a single swap meta field to the given stream
+static void
+PackField(std::ostream &os, const SwapMetaType type, const size_t length, const void *value)
+{
+ // Outside of packing/unpacking code, we correctly use SwapMetaType for
+ // valid swap meta types, but we store these values as RawSwapMetaType.
+ const auto rawType = NaturalCast<RawSwapMetaType>(type);
+
+ if (length > SwapMetaFieldValueLengthMax)
+ throw TextException("swap meta field value too big to store", Here());
+
+ // Outside of packing/unpacking code, we correctly use size_t for value
+ // sizes, we still store these values using RawSwapMetaLength.
+ const auto rawLength = NaturalCast<RawSwapMetaLength>(length);
+
+ CheckSwapMetaSerialization(rawType, rawLength, value);
+
+ if (!os.write(&rawType, sizeof(rawType)) ||
+ !os.write(reinterpret_cast<const char*>(&rawLength), sizeof(rawLength)) ||
+ (length && !os.write(static_cast<const char*>(value), length)))
+ throw TextException("cannot pack a swap meta field", Here());
+}
+
+/// writes all swap meta fields of the given Store entry to the given stream
+static void
+PackFields(const StoreEntry &entry, std::ostream &os)
+{
+ const auto &emem = entry.mem();
+ const auto objsize = emem.expectedReplySize();
+
+ SBuf url;
+ if (emem.request)
+ url = emem.request->storeId();
+ else
+ url = entry.url();
+
+ debugs(20, 3, entry << " URL: " << url);
+
+ PackField(os, STORE_META_KEY_MD5, SQUID_MD5_DIGEST_LENGTH, entry.key);
+
+ PackField(os, STORE_META_STD_LFS, STORE_HDR_METASIZE, &entry.timestamp);
+
+ // XXX: Performance regression, c_str() may reallocate.
+ // XXX: do TLV without the c_str() termination. check readers first though
+ PackField(os, STORE_META_URL, url.length() + 1U, url.c_str());
+
+ if (objsize >= 0)
+ PackField(os, STORE_META_OBJSIZE, sizeof(objsize), &objsize);
+
+ const auto &vary = emem.vary_headers;
+ if (!vary.isEmpty())
+ PackField(os, STORE_META_VARY_HEADERS, vary.length(), vary.rawContent());
+}
+
+} // namespace Store
+
+AllocedBuf
+Store::PackSwapMeta(const StoreEntry &entry, size_t &totalLength)
+{
+ SBufStream os;
+ PackFields(entry, os);
+ const auto metas = os.buf();
+
+ // TODO: Optimize this allocation away by returning (and swapping out) SBuf.
+ const auto bufSize = NaturalSum<size_t>(sizeof(SwapMetaMagic), sizeof(RawSwapMetaPrefixLength), metas.length()).value();
+ AllocedBuf buf(xmalloc(bufSize));
+ const auto bufStart = static_cast<char*>(buf.get());
+
+ auto pos = bufStart; // buf writing position
+
+ *pos = SwapMetaMagic;
+ pos += sizeof(SwapMetaMagic);
+
+ const auto metaSize = NaturalCast<RawSwapMetaLength>(bufSize);
+ memcpy(pos, &metaSize, sizeof(metaSize));
+ pos += sizeof(metaSize);
+
+ Assure(pos + metas.length() == bufStart + bufSize); // paranoid
+ memcpy(pos, metas.rawContent(), metas.length());
+ pos += metas.length();
+
+ totalLength = bufSize;
+ return buf;
+}
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_SRC_STORE_SWAPMETAOUT_H
+#define SQUID_SRC_STORE_SWAPMETAOUT_H
+
+#include "base/HardFun.h"
+#include "store/forward.h"
+
+#include <memory>
+
+// TODO: Use CtoCpp1() from security/LockingPointer.h by moving that into base/ToCpp.h or similar.
+/// C++ wrapper for the legacy xmalloc()/xcalloc() deallocator
+/// \sa xfree_cppwrapper() with a slightly different (FREE-matching) signature.
+extern "C++" inline void xfree_cpp(const void * const x) { xfree(x); }
+
+// TODO: Move AllocedBuf and xfree_cpp() to src/base/Memory.h or similar.
+/// memory allocated by xmalloc() or xcalloc(), to be freed by xfree()
+using AllocedBuf = std::unique_ptr<void, HardFun<void, const void *, &xfree_cpp> >;
+
+namespace Store {
+
+/// swap metadata prefix and all swap metadata fields of the given entry
+/// \param size gets filled with the total swap metadata size
+AllocedBuf PackSwapMeta(const StoreEntry &, size_t &size);
+
+} // namespace Store
+
+#endif /* SQUID_SRC_STORE_SWAPMETAOUT_H */
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#include "squid.h"
+#include "base/Raw.h"
+#include "base/TextException.h"
+#include "debug/Stream.h"
+#include "sbuf/Stream.h"
+#include "store/SwapMetaView.h"
+
+#include <iostream>
+
+namespace Store {
+
+/// properly reports or rejects a problematic raw swap meta field type
+static void
+HandleBadRawType(const RawSwapMetaType type)
+{
+ if (ReservedSwapMetaType(type)) {
+ debugs(20, 3, "ignoring swap meta field with a reserved type: " << int(type));
+ return;
+ }
+
+ if (DeprecatedSwapMetaType(type)) {
+ debugs(20, DBG_IMPORTANT, "ERROR: Ignoring swap meta field with a deprecated type: " << int(type));
+ return;
+ }
+
+ // TODO: Instead of assuming that all future swap meta types can be ignored
+ // (and some should be reported at level-0/1), future-proof this code by
+ // using a type bit to define whether to silently ignore a swap meta field
+ // with that type (or even the whole Store entry with that field).
+
+ const auto typeValuesWeMightAdd = 10;
+ // compute "type > RawSwapMetaTypeTop() + typeValuesWeMightAdd" w/o overflow
+ if (type >= typeValuesWeMightAdd && type - typeValuesWeMightAdd > RawSwapMetaTypeTop()) {
+ debugs(20, DBG_IMPORTANT, "ERROR: Malformed cache storage; ignoring swap meta field with an unexpected type: " << int(type));
+ return;
+ }
+
+ if (type > RawSwapMetaTypeTop()) {
+ debugs(20, 3, "ignoring swap meta field with a presumed future type: " << int(type));
+ return;
+ }
+
+ assert(type <= RawSwapMetaTypeBottom); // handled all other bad types above
+ debugs(20, DBG_IMPORTANT, "ERROR: Malformed cache storage; ignoring swap meta field with an invalid type: " << int(type));
+}
+
+} // namespace Store
+
+Store::SwapMetaView::SwapMetaView(const void * const begin, const void * const end)
+{
+ auto input = static_cast<const char *>(begin);
+
+ SwapMetaExtract(rawType, input, end);
+ if (HonoredSwapMetaType(rawType))
+ type = static_cast<SwapMetaType>(rawType);
+ else
+ HandleBadRawType(rawType); // and leave type as STORE_META_VOID
+
+ RawSwapMetaLength lengthOrGarbage = 0;
+ SwapMetaExtract(lengthOrGarbage, input, end);
+ if (lengthOrGarbage < 0)
+ throw TextException("negative swap meta field length value", Here());
+ if (uint64_t(lengthOrGarbage) > SwapMetaFieldValueLengthMax)
+ throw TextException("huge swap meta field length value", Here());
+ if (input + lengthOrGarbage > end)
+ throw TextException("truncated swap meta field value", Here());
+ rawLength = static_cast<size_t>(lengthOrGarbage);
+
+ assert(input > begin); // we consumed some input but stayed in the buffer range (the lower bound check)
+ assert(input + rawLength <= end); // we stayed in the buffer range (the upper bound check)
+ rawValue = input;
+}
+
+void
+Store::SwapMetaView::checkExpectedLength(const size_t expectedLength) const
+{
+ if (rawLength != expectedLength)
+ throw TextException(ToSBuf("Bad value length in a Store entry meta field expecting a ",
+ expectedLength, "-byte value: ", *this), Here());
+}
+
+std::ostream &
+operator <<(std::ostream &os, const Store::SwapMetaView &meta)
+{
+ os << "type=" << int(meta.rawType);
+ // XXX: Change Raw constructor to take void* data instead of casting here.
+ const auto rawValue = reinterpret_cast<const char*>(meta.rawValue);
+ os << Raw("value", rawValue, meta.rawLength).minLevel(DBG_DATA).hex();
+ return os;
+}
+
--- /dev/null
+/*
+ * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
+ *
+ * Squid software is distributed under GPLv2+ license and includes
+ * contributions from numerous individuals and organizations.
+ * Please see the COPYING and CONTRIBUTORS files for details.
+ */
+
+#ifndef SQUID_SRC_STORE_SWAPMETAVIEW_H
+#define SQUID_SRC_STORE_SWAPMETAVIEW_H
+
+#include "store/SwapMeta.h"
+
+#include <iosfwd>
+
+namespace Store {
+
+/// a swap metadata field inside the buffer given to SwapMetaUnpacker
+class SwapMetaView
+{
+public:
+ /// ensures that our fixed-size field value has the given expected length
+ void checkExpectedLength(size_t) const;
+
+public:
+ /// A serialized rawLength-byte value (i.e. V in swap meta TLV field).
+ /// The value contents may be completely malformed/bogus.
+ /// Not meaningful when type is STORE_META_VOID.
+ const void *rawValue = nullptr;
+
+ /// The number of bytes in rawValue (i.e. L in swap meta TLV field).
+ /// This length may not match the length of valid fields of this type.
+ /// Not meaningful when type is STORE_META_VOID.
+ size_t rawLength = 0;
+
+ /// The serialized type (i.e. T in swap meta TLV field).
+ /// This type value may not match any named by SwapMetaType.
+ RawSwapMetaType rawType = RawSwapMetaTypeBottom;
+
+ /// The sanitized TLV type that always matches one named by SwapMetaType:
+ /// rawType (if matches a value named by SwapMetaType) or STORE_META_VOID.
+ SwapMetaType type = STORE_META_VOID;
+
+private:
+ /*
+ * SwapMetaView objects should always be accessed through SwapMetaIterator
+ * or SwapMetaUnpacker for read-only access to the current view of metadata.
+ */
+ friend class SwapMetaIterator;
+
+ SwapMetaView() = default;
+ SwapMetaView(const SwapMetaView &) = default;
+ SwapMetaView(SwapMetaView &&) = default;
+ SwapMetaView &operator =(const SwapMetaView &) = default;
+ SwapMetaView &operator =(SwapMetaView &&) = default;
+
+ /// positions the view at the first swap meta field in the given buffer
+ /// \param begin is where the buffer and the field starts
+ /// \param end is where the buffer (but not necessarily the field) finishes
+ explicit SwapMetaView(const void *begin, const void * const end);
+};
+
+/// a helper function to safely copy raw end-bounded serialized input into the
+/// given item and advance that input to the next item
+template <typename Item>
+void
+SwapMetaExtract(Item &item, const char * &input, const void *end)
+{
+ if (input + sizeof(item) > end)
+ throw TextException("truncated swap meta part", Here());
+ memcpy(&item, input, sizeof(item));
+ input += sizeof(item);
+}
+
+} // namespace Store
+
+/// writes a short human-readable summary of the given SwapMetaView object
+std::ostream &operator <<(std::ostream &, const Store::SwapMetaView &);
+
+#endif /* SQUID_SRC_STORE_SWAPMETAVIEW_H */
+
#include "SquidConfig.h"
#include "StatCounters.h"
#include "Store.h"
+#include "store/SwapMetaIn.h"
#include "store_swapin.h"
#include "StoreClient.h"
-#include "StoreMeta.h"
-#include "StoreMetaUnpacker.h"
#if USE_DELAY_POOLS
#include "DelayPools.h"
#endif
sc->readBody(buf, len);
}
-bool
-store_client::unpackHeader(char const *buf, ssize_t len)
-{
- debugs(90, 3, "store_client::unpackHeader: len " << len << "");
- assert(len >= 0);
-
- int swap_hdr_sz = 0;
- tlv *tlv_list = nullptr;
- try {
- StoreMetaUnpacker aBuilder(buf, len, &swap_hdr_sz);
- tlv_list = aBuilder.createStoreMeta();
- } catch (const std::exception &e) {
- debugs(90, DBG_IMPORTANT, "WARNING: failed to unpack metadata because " << e.what());
- return false;
- }
- assert(tlv_list);
-
- /*
- * Check the meta data and make sure we got the right object.
- */
- for (tlv *t = tlv_list; t; t = t->next) {
- if (!t->checkConsistency(entry)) {
- storeSwapTLVFree(tlv_list);
- return false;
- }
- }
-
- storeSwapTLVFree(tlv_list);
-
- assert(swap_hdr_sz >= 0);
- entry->mem_obj->swap_hdr_sz = swap_hdr_sz;
- if (entry->swap_file_sz > 0) { // collapsed hits may not know swap_file_sz
- assert(entry->swap_file_sz >= static_cast<uint64_t>(swap_hdr_sz));
- entry->mem_obj->object_sz = entry->swap_file_sz - swap_hdr_sz;
- }
- debugs(90, 5, "store_client::unpackHeader: swap_file_sz=" <<
- entry->swap_file_sz << "( " << swap_hdr_sz << " + " <<
- entry->mem_obj->object_sz << ")");
- return true;
-}
-
void
store_client::readHeader(char const *buf, ssize_t len)
{
if (len < 0)
return fail();
- if (!unpackHeader(buf, len)) {
+ try {
+ Store::UnpackHitSwapMeta(buf, len, *entry);
+ } catch (...) {
+ debugs(90, DBG_IMPORTANT, "ERROR: Failed to unpack Store entry metadata: " << CurrentException);
fail();
return;
}
#include "md5.h"
#include "store_key_md5.h"
-static cache_key null_key[SQUID_MD5_DIGEST_LENGTH];
-
const char *
storeKeyText(const cache_key *key)
{
return n;
}
-int
-storeKeyNull(const cache_key * key)
-{
- if (memcmp(key, null_key, SQUID_MD5_DIGEST_LENGTH) == 0)
- return 1;
- else
- return 0;
-}
-
-void
-storeKeyInit(void)
-{
- memset(null_key, '\0', SQUID_MD5_DIGEST_LENGTH);
-}
-
const cache_key *storeKeyPublicByRequestMethod(HttpRequest *, const HttpRequestMethod&, const KeyScope keyScope = ksDefault);
const cache_key *storeKeyPrivate();
int storeKeyHashBuckets(int);
-int storeKeyNull(const cache_key *);
-void storeKeyInit(void);
extern HASHHASH storeKeyHashHash;
extern HASHCMP storeKeyHashCmp;
#include "squid.h"
#include "debug/Messages.h"
#include "event.h"
+#include "fde.h"
#include "globals.h"
#include "md5.h"
#include "SquidConfig.h"
#include "StatCounters.h"
#include "Store.h"
#include "store/Disk.h"
+#include "store/SwapMetaIn.h"
#include "store_digest.h"
#include "store_key_md5.h"
#include "store_rebuild.h"
}
}
-#include "fde.h"
-#include "Generic.h"
-#include "StoreMeta.h"
-#include "StoreMetaUnpacker.h"
-
-struct InitStoreEntry : public unary_function<StoreMeta, void> {
- InitStoreEntry(StoreEntry *anEntry, cache_key *aKey):what(anEntry),index(aKey) {}
-
- void operator()(StoreMeta const &x) {
- switch (x.getType()) {
-
- case STORE_META_KEY:
- assert(x.length == SQUID_MD5_DIGEST_LENGTH);
- memcpy(index, x.value, SQUID_MD5_DIGEST_LENGTH);
- break;
-
- case STORE_META_STD:
- struct old_metahdr {
- time_t timestamp;
- time_t lastref;
- time_t expires;
- time_t lastmod;
- size_t swap_file_sz;
- uint16_t refcount;
- uint16_t flags;
- } *tmp;
- tmp = (struct old_metahdr *)x.value;
- assert(x.length == STORE_HDR_METASIZE_OLD);
- what->timestamp = tmp->timestamp;
- what->lastref = tmp->lastref;
- what->expires = tmp->expires;
- what->lastModified(tmp->lastmod);
- what->swap_file_sz = tmp->swap_file_sz;
- what->refcount = tmp->refcount;
- what->flags = tmp->flags;
- break;
-
- case STORE_META_STD_LFS:
- assert(x.length == STORE_HDR_METASIZE);
- memcpy(&what->timestamp, x.value, STORE_HDR_METASIZE);
- break;
-
- default:
- break;
- }
- }
-
- StoreEntry *what;
- cache_key *index;
-};
-
bool
storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &)
{
StoreRebuildData &stats,
uint64_t expectedSize)
{
- int swap_hdr_len = 0;
- StoreMetaUnpacker aBuilder(buf.content(), buf.contentSize(), &swap_hdr_len);
- if (aBuilder.isBufferZero()) {
- debugs(47,5, "skipping empty record.");
- return false;
- }
+ uint64_t swap_hdr_len = 0;
+
+ tmpe.key = nullptr;
- StoreMeta *tlv_list = nullptr;
try {
- tlv_list = aBuilder.createStoreMeta();
- } catch (const std::exception &e) {
- debugs(47, DBG_IMPORTANT, "WARNING: Ignoring store entry because " << e.what());
+ swap_hdr_len = Store::UnpackIndexSwapMeta(buf, tmpe, key);
+ } catch (...) {
+ debugs(47, Important(65), "WARNING: Indexer ignores a cache_dir entry: " << CurrentException);
return false;
}
- assert(tlv_list);
// TODO: consume parsed metadata?
debugs(47,7, "successful swap meta unpacking; swap_file_sz=" << tmpe.swap_file_sz);
- memset(key, '\0', SQUID_MD5_DIGEST_LENGTH);
-
- InitStoreEntry visitor(&tmpe, key);
- for_each(*tlv_list, visitor);
- storeSwapTLVFree(tlv_list);
- tlv_list = nullptr;
- if (storeKeyNull(key)) {
+ if (!tmpe.key) {
debugs(47, DBG_IMPORTANT, "WARNING: Ignoring keyless cache entry");
return false;
}
- tmpe.key = key;
/* check sizes */
if (expectedSize > 0) {
/// loads entry from disk; fills supplied memory buffer on success
bool storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &counts);
-/// parses entry buffer and validates entry metadata; fills e on success
+
+/**
+ * Parses the given Store entry metadata, filling e and key.
+ * Optimization: Both e and key parameters may be updated even on failures.
+ *
+ * \param buf memory containing serialized Store entry swap metadata (at least)
+ * \param e caller's temporary StoreEntry object for returning parsed metadata
+ * \param key caller's temporary Store entry key buffer; usable to set e.key
+ * \param expectedSize either total entry size (including swap metadata) or 0
+ *
+ * \retval true success; e/key filled with parsed metadata
+ * \retval false failure; e/key ought to be ignored (may be filled/dirty)
+ */
bool storeRebuildParseEntry(MemBuf &buf, StoreEntry &e, cache_key *key, StoreRebuildData &counts, uint64_t expectedSize);
#endif /* SQUID_STORE_REBUILD_H_ */
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-/* DEBUG: section 20 Storage Manager Swapfile Metadata */
-
-#include "squid.h"
-#include "md5.h"
-#include "MemObject.h"
-#include "Store.h"
-#include "StoreMeta.h"
-#include "StoreMetaUnpacker.h"
-
-#if HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-
-void
-storeSwapTLVFree(tlv * n)
-{
- tlv *t;
-
- while ((t = n) != nullptr) {
- n = t->next;
- xfree(t->value);
- delete t;
- }
-}
-
-/*
- * Build a TLV list for a StoreEntry
- */
-tlv *
-storeSwapMetaBuild(const StoreEntry *e)
-{
- tlv *TLV = nullptr; /* we'll return this */
- tlv **T = &TLV;
- assert(e->mem_obj != nullptr);
- const int64_t objsize = e->mem_obj->expectedReplySize();
-
- // e->mem_obj->request may be nil in this context
- SBuf url;
- if (e->mem_obj->request)
- url = e->mem_obj->request->storeId();
- else
- url = e->url();
-
- debugs(20, 3, "storeSwapMetaBuild URL: " << url);
-
- tlv *t = StoreMeta::Factory (STORE_META_KEY,SQUID_MD5_DIGEST_LENGTH, e->key);
-
- if (!t) {
- storeSwapTLVFree(TLV);
- return nullptr;
- }
-
- T = StoreMeta::Add(T, t);
- t = StoreMeta::Factory(STORE_META_STD_LFS,STORE_HDR_METASIZE,&e->timestamp);
-
- if (!t) {
- storeSwapTLVFree(TLV);
- return nullptr;
- }
-
- // XXX: do TLV without the c_str() termination. check readers first though
- T = StoreMeta::Add(T, t);
- t = StoreMeta::Factory(STORE_META_URL, url.length()+1, url.c_str());
-
- if (!t) {
- storeSwapTLVFree(TLV);
- return nullptr;
- }
-
- if (objsize >= 0) {
- T = StoreMeta::Add(T, t);
- t = StoreMeta::Factory(STORE_META_OBJSIZE, sizeof(objsize), &objsize);
-
- if (!t) {
- storeSwapTLVFree(TLV);
- return nullptr;
- }
- }
-
- T = StoreMeta::Add(T, t);
- SBuf vary(e->mem_obj->vary_headers);
-
- if (!vary.isEmpty()) {
- t = StoreMeta::Factory(STORE_META_VARY_HEADERS, vary.length(), vary.c_str());
-
- if (!t) {
- storeSwapTLVFree(TLV);
- return nullptr;
- }
-
- StoreMeta::Add (T, t);
- }
-
- return TLV;
-}
-
-char *
-storeSwapMetaPack(tlv * tlv_list, int *length)
-{
- int buflen = 0;
- tlv *t;
- int j = 0;
- char *buf;
- assert(length != nullptr);
- ++buflen; /* STORE_META_OK */
- buflen += sizeof(int); /* size of header to follow */
-
- for (t = tlv_list; t; t = t->next)
- buflen += sizeof(char) + sizeof(int) + t->length;
-
- buf = (char *)xmalloc(buflen);
-
- buf[j] = (char) STORE_META_OK;
- ++j;
-
- memcpy(&buf[j], &buflen, sizeof(int));
-
- j += sizeof(int);
-
- for (t = tlv_list; t; t = t->next) {
- buf[j] = t->getType();
- ++j;
- memcpy(&buf[j], &t->length, sizeof(int));
- j += sizeof(int);
- memcpy(&buf[j], t->value, t->length);
- j += t->length;
- }
-
- assert((int) j == buflen);
- *length = buflen;
- return buf;
-}
-
tests/stub_Port.cc \
tests/stub_SBuf.cc \
tests/stub_StatHist.cc \
- tests/stub_StoreMeta.cc \
tests/stub_UdsOp.cc \
tests/stub_access_log.cc \
tests/stub_acl.cc \
tests/stub_store_digest.cc \
tests/stub_store_rebuild.cc \
tests/stub_store_stats.cc \
- tests/stub_store_swapout.cc \
tests/stub_tools.cc \
tests/stub_tunnel.cc \
tests/stub_wccp2.cc \
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#include "squid.h"
-
-#define STUB_API "StoreMeta.cc"
-#include "tests/STUB.h"
-
-#include "StoreMeta.h"
-
-bool StoreMeta::validType(char) STUB_RETVAL(false)
-bool StoreMeta::validLength(int) const STUB_RETVAL(false)
-StoreMeta * StoreMeta::Factory (char, size_t, void const *) STUB_RETVAL(nullptr)
-void StoreMeta::FreeList(StoreMeta **) STUB
-StoreMeta ** StoreMeta::Add(StoreMeta **, StoreMeta *) STUB_RETVAL(nullptr)
-bool StoreMeta::checkConsistency(StoreEntry *) const STUB_RETVAL(false)
-
StoreSearch *NewLocalSearch() STUB_RETVAL(nullptr)
}
+#include "store/SwapMetaIn.h"
+size_t Store::UnpackSwapMetaSize(const SBuf &) STUB_RETVAL(0)
+uint64_t Store::UnpackIndexSwapMeta(const MemBuf &, StoreEntry &, cache_key *) STUB_RETVAL(0)
+void Store::UnpackHitSwapMeta(char const *, ssize_t, StoreEntry &) STUB
+
+#include "store/SwapMetaOut.h"
+AllocedBuf Store::PackSwapMeta(const StoreEntry &, size_t &) STUB_RETVAL(nullptr)
+
+++ /dev/null
-/*
- * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
- *
- * Squid software is distributed under GPLv2+ license and includes
- * contributions from numerous individuals and organizations.
- * Please see the COPYING and CONTRIBUTORS files for details.
- */
-
-#include "squid.h"
-#include "StoreMeta.h"
-
-#define STUB_API "store_swapout.cc"
-#include "tests/STUB.h"
-
-#include <iostream>
-
-char *storeSwapMetaPack(tlv *, int *) STUB_RETVAL(nullptr)
-tlv *storeSwapMetaBuild(const StoreEntry *) STUB_RETVAL(nullptr)
-void storeSwapTLVFree(tlv *) STUB
-
#include "squid.h"
#include "Store.h"
+#include "store/SwapMeta.h"
#include "testStore.h"
#include "unitTestMain.h"
+#include <limits>
+
CPPUNIT_TEST_SUITE_REGISTRATION( testStore );
int
Store::FreeMemory();
}
+namespace Store {
+
+/// check rawType that may be ignored
+static void
+checkIgnorableSwapMetaRawType(const RawSwapMetaType rawType)
+{
+ if (IgnoredSwapMetaType(rawType)) {
+ // an ignored raw type is either deprecated or reserved
+ CPPUNIT_ASSERT(DeprecatedSwapMetaType(rawType) || ReservedSwapMetaType(rawType));
+ CPPUNIT_ASSERT(!(DeprecatedSwapMetaType(rawType) && ReservedSwapMetaType(rawType)));
+ } else {
+ // all other raw types are neither deprecated nor reserved
+ CPPUNIT_ASSERT(!DeprecatedSwapMetaType(rawType) && !ReservedSwapMetaType(rawType));
+ }
+}
+
+/// check a raw swap meta field type below SwapMetaType range or STORE_META_VOID
+static void
+checkTooSmallSwapMetaRawType(const RawSwapMetaType rawType)
+{
+ // RawSwapMetaTypeBottom and smaller values are unrelated to any named
+ // SwapMetaDataType values, including past, current, and future ones
+ CPPUNIT_ASSERT(!HonoredSwapMetaType(rawType)); // current
+ CPPUNIT_ASSERT(!IgnoredSwapMetaType(rawType)); // past and future
+ CPPUNIT_ASSERT(!DeprecatedSwapMetaType(rawType)); // past
+ CPPUNIT_ASSERT(!ReservedSwapMetaType(rawType)); // future
+}
+
+/// check a raw swap meta field type within SwapMetaType range, excluding STORE_META_VOID
+static void
+checkKnownSwapMetaRawType(const RawSwapMetaType rawType)
+{
+ // an in-range rawType other than STORE_META_VOID is either honored or ignored
+ CPPUNIT_ASSERT(HonoredSwapMetaType(rawType) || IgnoredSwapMetaType(rawType));
+ CPPUNIT_ASSERT(!(HonoredSwapMetaType(rawType) && IgnoredSwapMetaType(rawType)));
+ checkIgnorableSwapMetaRawType(rawType);
+}
+
+/// check a raw swap meta field type exceeding RawSwapMetaTypeTop()
+static void
+checkTooBigSwapMetaRawType(const RawSwapMetaType rawType)
+{
+ // values beyond RawSwapMetaTypeTop() cannot be honored but may be ignored
+ CPPUNIT_ASSERT(!HonoredSwapMetaType(rawType));
+ checkIgnorableSwapMetaRawType(rawType);
+}
+
+/// check a given raw swap meta field type
+static void
+checkSwapMetaRawType(const RawSwapMetaType rawType)
+{
+ if (rawType <= RawSwapMetaTypeBottom)
+ checkTooSmallSwapMetaRawType(rawType);
+ else if (rawType > RawSwapMetaTypeTop())
+ checkTooBigSwapMetaRawType(rawType);
+ else
+ checkKnownSwapMetaRawType(rawType);
+}
+
+} // namespace Store
+
+void
+testStore::testSwapMetaTypeClassification()
+{
+ using limits = std::numeric_limits<Store::RawSwapMetaType>;
+ for (auto rawType = limits::min(); true; ++rawType) {
+
+ Store::checkSwapMetaRawType(rawType);
+
+ if (rawType == limits::max())
+ break;
+ }
+
+ // Store::RawSwapMetaTypeTop() is documented as an honored type value
+ CPPUNIT_ASSERT(Store::HonoredSwapMetaType(Store::RawSwapMetaTypeTop()));
+}
+
CPPUNIT_TEST( testUnsetRoot );
CPPUNIT_TEST( testStats );
CPPUNIT_TEST( testMaxSize );
+ CPPUNIT_TEST( testSwapMetaTypeClassification );
CPPUNIT_TEST_SUITE_END();
public:
void testUnsetRoot();
void testStats();
void testMaxSize();
+ void testSwapMetaTypeClassification();
};
/// allows testing of methods without having all the other components live
projects / thirdparty / squid.git / commitdiff
