From: Eduard Bagdasaryan Date: Sat, 30 Aug 2025 22:09:14 +0000 (+0000) Subject: Reduce testHttpRange dependencies (#2148) X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;p=thirdparty%2Fsquid.git Reduce testHttpRange dependencies (#2148) This drastic reduction in testHttpRange dependencies is made possible by splitting HeaderTools code into several parts: * Header-mangling code that pulled in many heavy dependencies but was unused by testHttpRange code. This high-level code does not belong to libhttp, so it was left in src/ (see HeaderMangling.cc). * Simple header manipulation functions without heavy dependencies; some used by testHttpRange code. This low-level code should eventually be moved to libhttp, but it was left in HttpHeaderTools.cc for now because libhttp itself is currently bloated with heavy dependencies -- linking with libhttp requires linking with or stubbing a lot of code unrelated to testHttpRange. * Definition of httpHeaderParseQuotedString() and a few other functions declared in HttpHeader.h were moved to HttpHeader.cc to improve declaration/definition/stubs affinity and avoid linking errors: testHttpRange now depends on stub_HttpHeader.cc that has stubs for these functions. * httpHeaderMaskInit() and getStringPrefix() are only used by HttpHeader.cc, so they were moved to HttpHeader.cc and made static. Moved code was unchanged. A few comments were relocated to better match Squid coding style. A few STUBs were added/adjusted as needed. TODO: Check the remaining two tests that still include header mangling code: testHttpRequest and testCacheManager. --- diff --git a/src/HeaderMangling.cc b/src/HeaderMangling.cc new file mode 100644 index 0000000000..9eba869a47 --- /dev/null +++ b/src/HeaderMangling.cc @@ -0,0 +1,271 @@ +/* + * Copyright (C) 1996-2025 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +/* DEBUG: section 66 HTTP Header Tools */ + +#include "squid.h" +#include "acl/FilledChecklist.h" +#include "acl/Gadgets.h" +#include "base/EnumIterator.h" +#include "fde.h" +#include "globals.h" +#include "http/RegisteredHeaders.h" +#include "HttpHeader.h" +#include "HttpRequest.h" +#include "MemBuf.h" +#include "sbuf/Stream.h" +#include "sbuf/StringConvert.h" +#include "SquidConfig.h" +#include "Store.h" + +static void httpHdrAdd(HttpHeader *heads, HttpRequest *request, const AccessLogEntryPointer &al, HeaderWithAclList &headersAdd); + +/** + * Checks the anonymizer (header_access) configuration. + * + * \retval 0 Header is explicitly blocked for removal + * \retval 1 Header is explicitly allowed + * \retval 1 Header has been replaced, the current version can be used. + * \retval 1 Header has no access controls to test + */ +static int +httpHdrMangle(HttpHeaderEntry * e, HttpRequest * request, HeaderManglers *hms, const AccessLogEntryPointer &al) +{ + int retval; + + assert(e); + + const headerMangler *hm = hms->find(*e); + + /* mangler or checklist went away. default allow */ + if (!hm || !hm->access_list) { + debugs(66, 7, "couldn't find mangler or access list. Allowing"); + return 1; + } + + ACLFilledChecklist checklist(hm->access_list, request); + checklist.updateAle(al); + + // XXX: The two "It was denied" clauses below mishandle cases with no + // matching rules, violating the "If no rules within the set have matching + // ACLs, the header field is left as is" promise in squid.conf. + // TODO: Use Acl::Answer::implicit. See HttpStateData::forwardUpgrade(). + if (checklist.fastCheck().allowed()) { + /* aclCheckFast returns true for allow. */ + debugs(66, 7, "checklist for mangler is positive. Mangle"); + retval = 1; + } else if (nullptr == hm->replacement) { + /* It was denied, and we don't have any replacement */ + debugs(66, 7, "checklist denied, we have no replacement. Pass"); + // XXX: We said "Pass", but the caller will delete on zero retval. + retval = 0; + } else { + /* It was denied, but we have a replacement. Replace the + * header on the fly, and return that the new header + * is allowed. + */ + debugs(66, 7, "checklist denied but we have replacement. Replace"); + e->value = hm->replacement; + retval = 1; + } + + return retval; +} + +/** Mangles headers for a list of headers. */ +void +httpHdrMangleList(HttpHeader *l, HttpRequest *request, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep) +{ + HttpHeaderEntry *e; + HttpHeaderPos p = HttpHeaderInitPos; + + /* check with anonymizer tables */ + HeaderManglers *hms = nullptr; + HeaderWithAclList *headersAdd = nullptr; + + switch (req_or_rep) { + case ROR_REQUEST: + hms = Config.request_header_access; + headersAdd = Config.request_header_add; + break; + case ROR_REPLY: + hms = Config.reply_header_access; + headersAdd = Config.reply_header_add; + break; + } + + if (hms) { + int headers_deleted = 0; + while ((e = l->getEntry(&p))) { + if (httpHdrMangle(e, request, hms, al) == 0) + l->delAt(p, headers_deleted); + } + + if (headers_deleted) + l->refreshMask(); + } + + if (headersAdd && !headersAdd->empty()) { + httpHdrAdd(l, request, al, *headersAdd); + } +} + +static +void header_mangler_clean(headerMangler &m) +{ + aclDestroyAccessList(&m.access_list); + safe_free(m.replacement); +} + +static +void header_mangler_dump_access(StoreEntry * entry, const char *option, + const headerMangler &m, const char *name) +{ + if (m.access_list != nullptr) { + storeAppendPrintf(entry, "%s ", option); + dump_acl_access(entry, name, m.access_list); + } +} + +static +void header_mangler_dump_replacement(StoreEntry * entry, const char *option, + const headerMangler &m, const char *name) +{ + if (m.replacement) + storeAppendPrintf(entry, "%s %s %s\n", option, name, m.replacement); +} + +HeaderManglers::HeaderManglers() +{ + memset(known, 0, sizeof(known)); + memset(&all, 0, sizeof(all)); +} + +HeaderManglers::~HeaderManglers() +{ + for (auto i : WholeEnum()) + header_mangler_clean(known[i]); + + for (auto i : custom) + header_mangler_clean(i.second); + + header_mangler_clean(all); +} + +void +HeaderManglers::dumpAccess(StoreEntry * entry, const char *name) const +{ + for (auto id : WholeEnum()) + header_mangler_dump_access(entry, name, known[id], Http::HeaderLookupTable.lookup(id).name); + + for (auto i : custom) + header_mangler_dump_access(entry, name, i.second, i.first.c_str()); + + header_mangler_dump_access(entry, name, all, "All"); +} + +void +HeaderManglers::dumpReplacement(StoreEntry * entry, const char *name) const +{ + for (auto id : WholeEnum()) { + header_mangler_dump_replacement(entry, name, known[id], Http::HeaderLookupTable.lookup(id).name); + } + + for (auto i: custom) { + header_mangler_dump_replacement(entry, name, i.second, i.first.c_str()); + } + + header_mangler_dump_replacement(entry, name, all, "All"); +} + +headerMangler * +HeaderManglers::track(const char *name) +{ + if (strcmp(name, "All") == 0) + return &all; + + const Http::HdrType id = Http::HeaderLookupTable.lookup(SBuf(name)).id; + + if (id != Http::HdrType::BAD_HDR) + return &known[id]; + + if (strcmp(name, "Other") == 0) + return &known[Http::HdrType::OTHER]; + + return &custom[name]; +} + +void +HeaderManglers::setReplacement(const char *name, const char *value) +{ + // for backword compatibility, we allow replacements to be configured + // for headers w/o access rules, but such replacements are ignored + headerMangler *m = track(name); + + safe_free(m->replacement); // overwrite old value if any + m->replacement = xstrdup(value); +} + +const headerMangler * +HeaderManglers::find(const HttpHeaderEntry &e) const +{ + // a known header with a configured ACL list + if (e.id != Http::HdrType::OTHER && Http::any_HdrType_enum_value(e.id) && + known[e.id].access_list) + return &known[e.id]; + + // a custom header + if (e.id == Http::HdrType::OTHER) { + // does it have an ACL list configured? + // Optimize: use a name type that we do not need to convert to here + SBuf tmp(e.name); // XXX: performance regression. c_str() reallocates + const ManglersByName::const_iterator i = custom.find(tmp.c_str()); + if (i != custom.end()) + return &i->second; + } + + // Next-to-last resort: "Other" rules match any custom header + if (e.id == Http::HdrType::OTHER && known[Http::HdrType::OTHER].access_list) + return &known[Http::HdrType::OTHER]; + + // Last resort: "All" rules match any header + if (all.access_list) + return &all; + + return nullptr; +} + +void +httpHdrAdd(HttpHeader *heads, HttpRequest *request, const AccessLogEntryPointer &al, HeaderWithAclList &headersAdd) +{ + ACLFilledChecklist checklist(nullptr, request); + checklist.updateAle(al); + + for (HeaderWithAclList::const_iterator hwa = headersAdd.begin(); hwa != headersAdd.end(); ++hwa) { + if (!hwa->aclList || checklist.fastCheck(hwa->aclList).allowed()) { + const char *fieldValue = nullptr; + MemBuf mb; + if (hwa->quoted) { + if (al != nullptr) { + mb.init(); + hwa->valueFormat->assemble(mb, al, 0); + fieldValue = mb.content(); + } + } else { + fieldValue = hwa->fieldValue.c_str(); + } + + if (!fieldValue || fieldValue[0] == '\0') + fieldValue = "-"; + + HttpHeaderEntry *e = new HttpHeaderEntry(hwa->fieldId, SBuf(hwa->fieldName), fieldValue); + heads->addEntry(e); + } + } +} + diff --git a/src/HeaderMangling.h b/src/HeaderMangling.h new file mode 100644 index 0000000000..9f1c84b7a9 --- /dev/null +++ b/src/HeaderMangling.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 1996-2025 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef SQUID_SRC_HEADERMANGLING_H +#define SQUID_SRC_HEADERMANGLING_H + +#include "acl/forward.h" +#include "format/Format.h" +#include "http/RegisteredHeaders.h" + +#include +#include +#include +#include +#if HAVE_STRINGS_H +#include +#endif + +class HeaderWithAcl; +class HttpHeader; +class HttpHeaderEntry; +class StoreEntry; + +typedef std::list HeaderWithAclList; + +/* Distinguish between Request and Reply (for header mangling) */ +typedef enum { + ROR_REQUEST, + ROR_REPLY +} req_or_rep_t; + +// Currently a POD +class headerMangler +{ +public: + acl_access *access_list; + char *replacement; +}; + +/// A collection of headerMangler objects for a given message kind. +class HeaderManglers +{ +public: + HeaderManglers(); + ~HeaderManglers(); + + /// returns a header mangler for field e or nil if none was specified + const headerMangler *find(const HttpHeaderEntry &e) const; + + /// returns a mangler for the named header (known or custom) + headerMangler *track(const char *name); + + /// updates mangler for the named header with a replacement value + void setReplacement(const char *name, const char *replacementValue); + + /// report the *_header_access part of the configuration + void dumpAccess(StoreEntry *entry, const char *optionName) const; + /// report the *_header_replace part of the configuration + void dumpReplacement(StoreEntry *entry, const char *optionName) const; + +private: + /// Case-insensitive std::string "less than" comparison functor. + /// Fast version recommended by Meyers' "Effective STL" for ASCII c-strings. + class NoCaseLessThan + { + public: + bool operator()(const std::string &lhs, const std::string &rhs) const { + return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; + } + }; + + /// a name:mangler map; optimize: use unordered map or some such + typedef std::map ManglersByName; + + /// one mangler for each known header + headerMangler known[static_cast(Http::HdrType::enumEnd_)]; + + /// one mangler for each custom header + ManglersByName custom; + + /// configured if some mangling ACL applies to all header names + headerMangler all; + +private: + /* not implemented */ + HeaderManglers(const HeaderManglers &); + HeaderManglers &operator =(const HeaderManglers &); +}; + +class HeaderWithAcl +{ +public: + HeaderWithAcl() : aclList(nullptr), valueFormat(nullptr), fieldId(Http::HdrType::BAD_HDR), quoted(false) {} + + /// HTTP header field name + std::string fieldName; + + /// HTTP header field value, possibly with macros + std::string fieldValue; + + /// when the header field should be added (always if nil) + ACLList *aclList; + + /// compiled HTTP header field value (no macros) + Format::Format *valueFormat; + + /// internal ID for "known" headers or HDR_OTHER + Http::HdrType fieldId; + + /// whether fieldValue may contain macros + bool quoted; +}; + +void httpHdrMangleList(HttpHeader *, HttpRequest *, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep); + +#endif /* SQUID_SRC_HEADERMANGLING_H */ diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc index c52f44d357..31d06fcb60 100644 --- a/src/HttpHeader.cc +++ b/src/HttpHeader.cc @@ -26,6 +26,7 @@ #include "MemBuf.h" #include "mgr/Registration.h" #include "mime_header.h" +#include "sbuf/Stream.h" #include "sbuf/StringConvert.h" #include "SquidConfig.h" #include "StatHist.h" @@ -117,6 +118,22 @@ httpHeaderRegisterWithCacheManager(void) httpHeaderStoreReport, 0, 1); } +static void +httpHeaderMaskInit(HttpHeaderMask * mask, int value) +{ + memset(mask, value, sizeof(*mask)); +} + +/** handy to printf prefixes of potentially very long buffers */ +static const char * +getStringPrefix(const char *str, size_t sz) +{ +#define SHORT_PREFIX_SIZE 512 + LOCAL_ARRAY(char, buf, SHORT_PREFIX_SIZE); + xstrncpy(buf, str, (sz+1 > SHORT_PREFIX_SIZE) ? SHORT_PREFIX_SIZE : sz); + return buf; +} + void httpHeaderInitModule(void) { @@ -140,6 +157,122 @@ httpHeaderInitModule(void) httpHeaderRegisterWithCacheManager(); } +/** + * Parses a quoted-string field (RFC 2616 section 2.2), complains if + * something went wrong, returns non-zero on success. + * Un-escapes quoted-pair characters found within the string. + * start should point at the first double-quote. + */ +int +httpHeaderParseQuotedString(const char *start, const int len, String *val) +{ + const char *end, *pos; + val->clean(); + if (*start != '"') { + debugs(66, 2, "failed to parse a quoted-string header field near '" << start << "'"); + return 0; + } + pos = start + 1; + + while (*pos != '"' && len > (pos-start)) { + + if (*pos =='\r') { + ++pos; + if ((pos-start) > len || *pos != '\n') { + debugs(66, 2, "failed to parse a quoted-string header field with '\\r' octet " << (start-pos) + << " bytes into '" << start << "'"); + val->clean(); + return 0; + } + } + + if (*pos == '\n') { + ++pos; + if ( (pos-start) > len || (*pos != ' ' && *pos != '\t')) { + debugs(66, 2, "failed to parse multiline quoted-string header field '" << start << "'"); + val->clean(); + return 0; + } + // TODO: replace the entire LWS with a space + val->append(" "); + ++pos; + debugs(66, 2, "len < pos-start => " << len << " < " << (pos-start)); + continue; + } + + bool quoted = (*pos == '\\'); + if (quoted) { + ++pos; + if (!*pos || (pos-start) > len) { + debugs(66, 2, "failed to parse a quoted-string header field near '" << start << "'"); + val->clean(); + return 0; + } + } + end = pos; + while (end < (start+len) && *end != '\\' && *end != '\"' && (unsigned char)*end > 0x1F && *end != 0x7F) + ++end; + if (((unsigned char)*end <= 0x1F && *end != '\r' && *end != '\n') || *end == 0x7F) { + debugs(66, 2, "failed to parse a quoted-string header field with CTL octet " << (start-pos) + << " bytes into '" << start << "'"); + val->clean(); + return 0; + } + val->append(pos, end-pos); + pos = end; + } + + if (*pos != '\"') { + debugs(66, 2, "failed to parse a quoted-string header field which did not end with \" "); + val->clean(); + return 0; + } + /* Make sure it's defined even if empty "" */ + if (!val->termedBuf()) + val->assign("", 0); + return 1; +} + +SBuf +httpHeaderQuoteString(const char *raw) +{ + assert(raw); + + // TODO: Optimize by appending a sequence of characters instead of a char. + // This optimization may be easier with Tokenizer after raw becomes SBuf. + + // RFC 7230 says a "sender SHOULD NOT generate a quoted-pair in a + // quoted-string except where necessary" (i.e., DQUOTE and backslash) + bool needInnerQuote = false; + for (const char *s = raw; !needInnerQuote && *s; ++s) + needInnerQuote = *s == '"' || *s == '\\'; + + SBuf quotedStr; + quotedStr.append('"'); + + if (needInnerQuote) { + for (const char *s = raw; *s; ++s) { + if (*s == '"' || *s == '\\') + quotedStr.append('\\'); + quotedStr.append(*s); + } + } else { + quotedStr.append(raw); + } + + quotedStr.append('"'); + return quotedStr; +} + +SBuf +Http::SlowlyParseQuotedString(const char * const description, const char * const start, const size_t length) +{ + String s; + if (!httpHeaderParseQuotedString(start, length, &s)) + throw TextException(ToSBuf("Cannot parse ", description, " as a quoted string"), Here()); + return StringToSBuf(s); +} + /* * HttpHeader Implementation */ diff --git a/src/HttpHeaderMask.h b/src/HttpHeaderMask.h index 2de317106a..077fd9bc4d 100644 --- a/src/HttpHeaderMask.h +++ b/src/HttpHeaderMask.h @@ -12,7 +12,5 @@ /* big mask for http headers */ typedef char HttpHeaderMask[12]; -void httpHeaderMaskInit(HttpHeaderMask * mask, int value); - #endif /* SQUID_SRC_HTTPHEADERMASK_H */ diff --git a/src/HttpHeaderTools.cc b/src/HttpHeaderTools.cc index b913ee4efe..a8459ead10 100644 --- a/src/HttpHeaderTools.cc +++ b/src/HttpHeaderTools.cc @@ -9,45 +9,16 @@ /* DEBUG: section 66 HTTP Header Tools */ #include "squid.h" -#include "acl/FilledChecklist.h" -#include "acl/Gadgets.h" -#include "base/EnumIterator.h" -#include "client_side.h" -#include "client_side_request.h" -#include "comm/Connection.h" #include "compat/strtoll.h" -#include "ConfigParser.h" -#include "fde.h" -#include "globals.h" -#include "http/RegisteredHeaders.h" -#include "http/Stream.h" #include "HttpHdrContRange.h" #include "HttpHeader.h" #include "HttpHeaderTools.h" -#include "HttpRequest.h" #include "MemBuf.h" -#include "sbuf/Stream.h" -#include "sbuf/StringConvert.h" -#include "SquidConfig.h" -#include "Store.h" #include "StrList.h" -#if USE_OPENSSL -#include "ssl/support.h" -#endif - -#include #include -#include static void httpHeaderPutStrvf(HttpHeader * hdr, Http::HdrType id, const char *fmt, va_list vargs); -static void httpHdrAdd(HttpHeader *heads, HttpRequest *request, const AccessLogEntryPointer &al, HeaderWithAclList &headersAdd); - -void -httpHeaderMaskInit(HttpHeaderMask * mask, int value) -{ - memset(mask, value, sizeof(*mask)); -} /* same as httpHeaderPutStr, but formats the string using snprintf first */ void @@ -105,16 +76,6 @@ httpHeaderHasConnDir(const HttpHeader * hdr, const SBuf &directive) return false; } -/** handy to printf prefixes of potentially very long buffers */ -const char * -getStringPrefix(const char *str, size_t sz) -{ -#define SHORT_PREFIX_SIZE 512 - LOCAL_ARRAY(char, buf, SHORT_PREFIX_SIZE); - xstrncpy(buf, str, (sz+1 > SHORT_PREFIX_SIZE) ? SHORT_PREFIX_SIZE : sz); - return buf; -} - /** * parses an int field, complains if something went wrong, returns true on * success @@ -158,363 +119,3 @@ httpHeaderParseOffset(const char *start, int64_t *value, char **endPtr) return true; } -/** - * Parses a quoted-string field (RFC 2616 section 2.2), complains if - * something went wrong, returns non-zero on success. - * Un-escapes quoted-pair characters found within the string. - * start should point at the first double-quote. - */ -int -httpHeaderParseQuotedString(const char *start, const int len, String *val) -{ - const char *end, *pos; - val->clean(); - if (*start != '"') { - debugs(66, 2, "failed to parse a quoted-string header field near '" << start << "'"); - return 0; - } - pos = start + 1; - - while (*pos != '"' && len > (pos-start)) { - - if (*pos =='\r') { - ++pos; - if ((pos-start) > len || *pos != '\n') { - debugs(66, 2, "failed to parse a quoted-string header field with '\\r' octet " << (start-pos) - << " bytes into '" << start << "'"); - val->clean(); - return 0; - } - } - - if (*pos == '\n') { - ++pos; - if ( (pos-start) > len || (*pos != ' ' && *pos != '\t')) { - debugs(66, 2, "failed to parse multiline quoted-string header field '" << start << "'"); - val->clean(); - return 0; - } - // TODO: replace the entire LWS with a space - val->append(" "); - ++pos; - debugs(66, 2, "len < pos-start => " << len << " < " << (pos-start)); - continue; - } - - bool quoted = (*pos == '\\'); - if (quoted) { - ++pos; - if (!*pos || (pos-start) > len) { - debugs(66, 2, "failed to parse a quoted-string header field near '" << start << "'"); - val->clean(); - return 0; - } - } - end = pos; - while (end < (start+len) && *end != '\\' && *end != '\"' && (unsigned char)*end > 0x1F && *end != 0x7F) - ++end; - if (((unsigned char)*end <= 0x1F && *end != '\r' && *end != '\n') || *end == 0x7F) { - debugs(66, 2, "failed to parse a quoted-string header field with CTL octet " << (start-pos) - << " bytes into '" << start << "'"); - val->clean(); - return 0; - } - val->append(pos, end-pos); - pos = end; - } - - if (*pos != '\"') { - debugs(66, 2, "failed to parse a quoted-string header field which did not end with \" "); - val->clean(); - return 0; - } - /* Make sure it's defined even if empty "" */ - if (!val->termedBuf()) - val->assign("", 0); - return 1; -} - -SBuf -Http::SlowlyParseQuotedString(const char * const description, const char * const start, const size_t length) -{ - String s; - if (!httpHeaderParseQuotedString(start, length, &s)) - throw TextException(ToSBuf("Cannot parse ", description, " as a quoted string"), Here()); - return StringToSBuf(s); -} - -SBuf -httpHeaderQuoteString(const char *raw) -{ - assert(raw); - - // TODO: Optimize by appending a sequence of characters instead of a char. - // This optimization may be easier with Tokenizer after raw becomes SBuf. - - // RFC 7230 says a "sender SHOULD NOT generate a quoted-pair in a - // quoted-string except where necessary" (i.e., DQUOTE and backslash) - bool needInnerQuote = false; - for (const char *s = raw; !needInnerQuote && *s; ++s) - needInnerQuote = *s == '"' || *s == '\\'; - - SBuf quotedStr; - quotedStr.append('"'); - - if (needInnerQuote) { - for (const char *s = raw; *s; ++s) { - if (*s == '"' || *s == '\\') - quotedStr.append('\\'); - quotedStr.append(*s); - } - } else { - quotedStr.append(raw); - } - - quotedStr.append('"'); - return quotedStr; -} - -/** - * Checks the anonymizer (header_access) configuration. - * - * \retval 0 Header is explicitly blocked for removal - * \retval 1 Header is explicitly allowed - * \retval 1 Header has been replaced, the current version can be used. - * \retval 1 Header has no access controls to test - */ -static int -httpHdrMangle(HttpHeaderEntry * e, HttpRequest * request, HeaderManglers *hms, const AccessLogEntryPointer &al) -{ - int retval; - - assert(e); - - const headerMangler *hm = hms->find(*e); - - /* mangler or checklist went away. default allow */ - if (!hm || !hm->access_list) { - debugs(66, 7, "couldn't find mangler or access list. Allowing"); - return 1; - } - - ACLFilledChecklist checklist(hm->access_list, request); - checklist.updateAle(al); - - // XXX: The two "It was denied" clauses below mishandle cases with no - // matching rules, violating the "If no rules within the set have matching - // ACLs, the header field is left as is" promise in squid.conf. - // TODO: Use Acl::Answer::implicit. See HttpStateData::forwardUpgrade(). - if (checklist.fastCheck().allowed()) { - /* aclCheckFast returns true for allow. */ - debugs(66, 7, "checklist for mangler is positive. Mangle"); - retval = 1; - } else if (nullptr == hm->replacement) { - /* It was denied, and we don't have any replacement */ - debugs(66, 7, "checklist denied, we have no replacement. Pass"); - // XXX: We said "Pass", but the caller will delete on zero retval. - retval = 0; - } else { - /* It was denied, but we have a replacement. Replace the - * header on the fly, and return that the new header - * is allowed. - */ - debugs(66, 7, "checklist denied but we have replacement. Replace"); - e->value = hm->replacement; - retval = 1; - } - - return retval; -} - -/** Mangles headers for a list of headers. */ -void -httpHdrMangleList(HttpHeader *l, HttpRequest *request, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep) -{ - HttpHeaderEntry *e; - HttpHeaderPos p = HttpHeaderInitPos; - - /* check with anonymizer tables */ - HeaderManglers *hms = nullptr; - HeaderWithAclList *headersAdd = nullptr; - - switch (req_or_rep) { - case ROR_REQUEST: - hms = Config.request_header_access; - headersAdd = Config.request_header_add; - break; - case ROR_REPLY: - hms = Config.reply_header_access; - headersAdd = Config.reply_header_add; - break; - } - - if (hms) { - int headers_deleted = 0; - while ((e = l->getEntry(&p))) { - if (httpHdrMangle(e, request, hms, al) == 0) - l->delAt(p, headers_deleted); - } - - if (headers_deleted) - l->refreshMask(); - } - - if (headersAdd && !headersAdd->empty()) { - httpHdrAdd(l, request, al, *headersAdd); - } -} - -static -void header_mangler_clean(headerMangler &m) -{ - aclDestroyAccessList(&m.access_list); - safe_free(m.replacement); -} - -static -void header_mangler_dump_access(StoreEntry * entry, const char *option, - const headerMangler &m, const char *name) -{ - if (m.access_list != nullptr) { - storeAppendPrintf(entry, "%s ", option); - dump_acl_access(entry, name, m.access_list); - } -} - -static -void header_mangler_dump_replacement(StoreEntry * entry, const char *option, - const headerMangler &m, const char *name) -{ - if (m.replacement) - storeAppendPrintf(entry, "%s %s %s\n", option, name, m.replacement); -} - -HeaderManglers::HeaderManglers() -{ - memset(known, 0, sizeof(known)); - memset(&all, 0, sizeof(all)); -} - -HeaderManglers::~HeaderManglers() -{ - for (auto i : WholeEnum()) - header_mangler_clean(known[i]); - - for (auto i : custom) - header_mangler_clean(i.second); - - header_mangler_clean(all); -} - -void -HeaderManglers::dumpAccess(StoreEntry * entry, const char *name) const -{ - for (auto id : WholeEnum()) - header_mangler_dump_access(entry, name, known[id], Http::HeaderLookupTable.lookup(id).name); - - for (auto i : custom) - header_mangler_dump_access(entry, name, i.second, i.first.c_str()); - - header_mangler_dump_access(entry, name, all, "All"); -} - -void -HeaderManglers::dumpReplacement(StoreEntry * entry, const char *name) const -{ - for (auto id : WholeEnum()) { - header_mangler_dump_replacement(entry, name, known[id], Http::HeaderLookupTable.lookup(id).name); - } - - for (auto i: custom) { - header_mangler_dump_replacement(entry, name, i.second, i.first.c_str()); - } - - header_mangler_dump_replacement(entry, name, all, "All"); -} - -headerMangler * -HeaderManglers::track(const char *name) -{ - if (strcmp(name, "All") == 0) - return &all; - - const Http::HdrType id = Http::HeaderLookupTable.lookup(SBuf(name)).id; - - if (id != Http::HdrType::BAD_HDR) - return &known[id]; - - if (strcmp(name, "Other") == 0) - return &known[Http::HdrType::OTHER]; - - return &custom[name]; -} - -void -HeaderManglers::setReplacement(const char *name, const char *value) -{ - // for backword compatibility, we allow replacements to be configured - // for headers w/o access rules, but such replacements are ignored - headerMangler *m = track(name); - - safe_free(m->replacement); // overwrite old value if any - m->replacement = xstrdup(value); -} - -const headerMangler * -HeaderManglers::find(const HttpHeaderEntry &e) const -{ - // a known header with a configured ACL list - if (e.id != Http::HdrType::OTHER && Http::any_HdrType_enum_value(e.id) && - known[e.id].access_list) - return &known[e.id]; - - // a custom header - if (e.id == Http::HdrType::OTHER) { - // does it have an ACL list configured? - // Optimize: use a name type that we do not need to convert to here - SBuf tmp(e.name); // XXX: performance regression. c_str() reallocates - const ManglersByName::const_iterator i = custom.find(tmp.c_str()); - if (i != custom.end()) - return &i->second; - } - - // Next-to-last resort: "Other" rules match any custom header - if (e.id == Http::HdrType::OTHER && known[Http::HdrType::OTHER].access_list) - return &known[Http::HdrType::OTHER]; - - // Last resort: "All" rules match any header - if (all.access_list) - return &all; - - return nullptr; -} - -void -httpHdrAdd(HttpHeader *heads, HttpRequest *request, const AccessLogEntryPointer &al, HeaderWithAclList &headersAdd) -{ - ACLFilledChecklist checklist(nullptr, request); - checklist.updateAle(al); - - for (HeaderWithAclList::const_iterator hwa = headersAdd.begin(); hwa != headersAdd.end(); ++hwa) { - if (!hwa->aclList || checklist.fastCheck(hwa->aclList).allowed()) { - const char *fieldValue = nullptr; - MemBuf mb; - if (hwa->quoted) { - if (al != nullptr) { - mb.init(); - hwa->valueFormat->assemble(mb, al, 0); - fieldValue = mb.content(); - } - } else { - fieldValue = hwa->fieldValue.c_str(); - } - - if (!fieldValue || fieldValue[0] == '\0') - fieldValue = "-"; - - HttpHeaderEntry *e = new HttpHeaderEntry(hwa->fieldId, SBuf(hwa->fieldName), fieldValue); - heads->addEntry(e); - } - } -} - diff --git a/src/HttpHeaderTools.h b/src/HttpHeaderTools.h index f2f32a611d..bc5a448a93 100644 --- a/src/HttpHeaderTools.h +++ b/src/HttpHeaderTools.h @@ -9,113 +9,10 @@ #ifndef SQUID_SRC_HTTPHEADERTOOLS_H #define SQUID_SRC_HTTPHEADERTOOLS_H -#include "acl/forward.h" -#include "format/Format.h" -#include "HttpHeader.h" +#include "http/RegisteredHeaders.h" #include "sbuf/forward.h" -#include -#include -#include -#include -#if HAVE_STRINGS_H -#include -#endif - -class HeaderWithAcl; class HttpHeader; -class HttpRequest; -class StoreEntry; - -typedef std::list HeaderWithAclList; - -/* Distinguish between Request and Reply (for header mangling) */ -typedef enum { - ROR_REQUEST, - ROR_REPLY -} req_or_rep_t; - -// Currently a POD -class headerMangler -{ -public: - acl_access *access_list; - char *replacement; -}; - -/// A collection of headerMangler objects for a given message kind. -class HeaderManglers -{ -public: - HeaderManglers(); - ~HeaderManglers(); - - /// returns a header mangler for field e or nil if none was specified - const headerMangler *find(const HttpHeaderEntry &e) const; - - /// returns a mangler for the named header (known or custom) - headerMangler *track(const char *name); - - /// updates mangler for the named header with a replacement value - void setReplacement(const char *name, const char *replacementValue); - - /// report the *_header_access part of the configuration - void dumpAccess(StoreEntry *entry, const char *optionName) const; - /// report the *_header_replace part of the configuration - void dumpReplacement(StoreEntry *entry, const char *optionName) const; - -private: - /// Case-insensitive std::string "less than" comparison functor. - /// Fast version recommended by Meyers' "Effective STL" for ASCII c-strings. - class NoCaseLessThan - { - public: - bool operator()(const std::string &lhs, const std::string &rhs) const { - return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; - } - }; - - /// a name:mangler map; optimize: use unordered map or some such - typedef std::map ManglersByName; - - /// one mangler for each known header - headerMangler known[static_cast(Http::HdrType::enumEnd_)]; - - /// one mangler for each custom header - ManglersByName custom; - - /// configured if some mangling ACL applies to all header names - headerMangler all; - -private: - /* not implemented */ - HeaderManglers(const HeaderManglers &); - HeaderManglers &operator =(const HeaderManglers &); -}; - -class HeaderWithAcl -{ -public: - HeaderWithAcl() : aclList(nullptr), valueFormat(nullptr), fieldId(Http::HdrType::BAD_HDR), quoted(false) {} - - /// HTTP header field name - std::string fieldName; - - /// HTTP header field value, possibly with macros - std::string fieldValue; - - /// when the header field should be added (always if nil) - ACLList *aclList; - - /// compiled HTTP header field value (no macros) - Format::Format *valueFormat; - - /// internal ID for "known" headers or HDR_OTHER - Http::HdrType fieldId; - - /// whether fieldValue may contain macros - bool quoted; -}; /// A strtoll(10) wrapper that checks for strtoll() failures and other problems. /// XXX: This function is not fully compatible with some HTTP syntax rules. @@ -129,9 +26,5 @@ bool httpHeaderHasConnDir(const HttpHeader * hdr, const SBuf &directive); int httpHeaderParseInt(const char *start, int *val); void httpHeaderPutStrf(HttpHeader * hdr, Http::HdrType id, const char *fmt,...) PRINTF_FORMAT_ARG3; -const char *getStringPrefix(const char *str, size_t len); - -void httpHdrMangleList(HttpHeader *, HttpRequest *, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep); - #endif /* SQUID_SRC_HTTPHEADERTOOLS_H */ diff --git a/src/Makefile.am b/src/Makefile.am index b6c0eebd23..5d85a96981 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -259,6 +259,8 @@ squid_SOURCES = \ Generic.h \ HappyConnOpener.cc \ HappyConnOpener.h \ + HeaderMangling.cc \ + HeaderMangling.h \ HierarchyLogEntry.h \ HttpBody.cc \ HttpBody.h \ @@ -902,8 +904,8 @@ nodist_tests_testURL_SOURCES = \ tests/stub_libhttp.cc \ tests/stub_libmem.cc tests_testURL_LDADD = \ - libsquid.la \ anyp/libanyp.la \ + libsquid.la \ parser/libparser.la \ base/libbase.la \ ip/libip.la \ @@ -1753,239 +1755,24 @@ tests_testHtmlQuote_LDFLAGS = $(LIBADD_DL) check_PROGRAMS += tests/testHttpRange tests_testHttpRange_SOURCES = \ - $(DELAY_POOL_SOURCE) \ - $(DNSSOURCE) \ - $(HTCPSOURCE) \ - $(IPC_SOURCE) \ - $(SNMP_SOURCE) \ - $(UNLINKDSOURCE) \ - $(WIN32_SOURCE) \ - AccessLogEntry.cc \ - AuthReg.h \ - BodyPipe.cc \ - tests/stub_CacheDigest.cc \ - CacheDigest.h \ - CachePeer.cc \ - CachePeer.h \ - CachePeers.cc \ - CachePeers.h \ - ClientInfo.h \ - tests/stub_CollapsedForwarding.cc \ - ConfigOption.cc \ - ConfigParser.cc \ - CpuAffinityMap.cc \ - CpuAffinityMap.h \ - CpuAffinitySet.cc \ - CpuAffinitySet.h \ - tests/stub_ETag.cc \ - tests/stub_EventLoop.cc \ - FadingCounter.cc \ - FileMap.h \ - FwdState.cc \ - FwdState.h \ - HappyConnOpener.cc \ - HappyConnOpener.h \ - HttpBody.cc \ - HttpBody.h \ - tests/stub_HttpControlMsg.cc \ - HttpHdrCc.cc \ - HttpHdrCc.h \ HttpHdrContRange.cc \ HttpHdrRange.cc \ - HttpHdrSc.cc \ - HttpHdrScTarget.cc \ - HttpHeader.cc \ - HttpHeader.h \ - HttpHeaderFieldStat.h \ + tests/stub_HttpHeader.cc \ HttpHeaderTools.cc \ - HttpHeaderTools.h \ tests/testHttpRange.cc \ - HttpReply.cc \ - HttpRequest.cc \ - tests/stub_HttpUpgradeProtocolAccess.cc \ - tests/stub_Instance.cc \ - IoStats.h \ - tests/stub_IpcIoFile.cc \ - LogTags.cc \ - MasterXaction.cc \ - MasterXaction.h \ - MemBuf.cc \ - MemObject.cc \ - tests/stub_MemStore.cc \ - Notes.cc \ - Notes.h \ - Parsing.cc \ - PeerPoolMgr.cc \ - PeerPoolMgr.h \ - Pipeline.cc \ - Pipeline.h \ - RefreshPattern.h \ - RemovalPolicy.cc \ - RequestFlags.cc \ - RequestFlags.h \ - ResolvedPeers.cc \ - ResolvedPeers.h \ - SquidMath.cc \ - SquidMath.h \ - StatCounters.cc \ - StatCounters.h \ - StatHist.cc \ - StatHist.h \ - StoreFileSystem.cc \ - StoreIOState.cc \ - StoreSwapLogData.cc \ + tests/stub_HttpReply.cc \ + tests/stub_MemBuf.cc \ StrList.cc \ - StrList.h \ String.cc \ - Transients.cc \ - tests/stub_cache_cf.cc \ - cache_cf.h \ - cache_manager.cc \ - tests/stub_carp.cc \ - carp.h \ - cbdata.cc \ - clientStream.cc \ - tests/stub_client_db.cc \ - client_side.cc \ - client_side.h \ - client_side_reply.cc \ - client_side_request.cc \ - dlink.cc \ - dlink.h \ - errorpage.cc \ - event.cc \ - tests/stub_external_acl.cc \ - tests/stub_fatal.cc \ - fatal.h \ - fd.cc \ - fd.h \ - fde.cc \ - filemap.cc \ - fqdncache.cc \ - fqdncache.h \ - fs_io.cc \ - fs_io.h \ - helper.cc \ - hier_code.h \ - http.cc \ - icp_v2.cc \ - icp_v3.cc \ - int.cc \ - int.h \ - internal.cc \ - internal.h \ - tests/stub_ipc_Forwarder.cc \ - ipcache.cc \ - tests/stub_libauth.cc \ - tests/stub_libdiskio.cc \ - tests/stub_liberror.cc \ - tests/stub_libeui.cc \ - tests/stub_libmem.cc \ - tests/stub_libsecurity.cc \ - tests/stub_libstore.cc \ - tests/stub_main_cc.cc \ - mem_node.cc \ - mime.cc \ - mime.h \ - mime_header.cc \ - mime_header.h \ - multicast.cc \ - multicast.h \ - neighbors.cc \ - neighbors.h \ - pconn.cc \ - peer_digest.cc \ - peer_proxy_negotiate_auth.cc \ - peer_proxy_negotiate_auth.h \ - peer_select.cc \ - peer_sourcehash.cc \ - peer_sourcehash.h \ - peer_userhash.cc \ - peer_userhash.h \ - tests/stub_redirect.cc \ - redirect.h \ - refresh.cc \ - refresh.h \ - repl_modules.h \ - stat.cc \ - stat.h \ - stmem.cc \ - store.cc \ - store_client.cc \ - tests/stub_store_digest.cc \ - store_digest.h \ - store_io.cc \ - store_key_md5.cc \ - store_key_md5.h \ - store_log.cc \ - store_log.h \ - store_rebuild.cc \ - store_rebuild.h \ - tests/stub_store_stats.cc \ - store_swapin.cc \ - store_swapin.h \ - store_swapout.cc \ - tools.cc \ - tools.h \ - tests/stub_tunnel.cc \ - tunnel.h \ - urn.cc \ - urn.h \ - tests/stub_wccp2.cc \ - wccp2.h \ - wordlist.cc \ - wordlist.h -nodist_tests_testHttpRange_SOURCES = \ - $(BUILT_SOURCES) \ - tests/stub_libtime.cc + tests/stub_cbdata.cc \ + tests/stub_debug.cc \ + tests/stub_libhttp.cc \ + tests/stub_libmem.cc tests_testHttpRange_LDADD = \ - libsquid.la \ - clients/libclients.la \ - servers/libservers.la \ - ftp/libftp.la \ - helper/libhelper.la \ - http/libhttp.la \ - parser/libparser.la \ - acl/libacls.la \ - acl/libstate.la \ - acl/libapi.la \ - proxyp/libproxyp.la \ - parser/libparser.la \ - fs/libfs.la \ - anyp/libanyp.la \ - icmp/libicmp.la \ - comm/libcomm.la \ - ip/libip.la \ - log/liblog.la \ - format/libformat.la \ - $(REPL_OBJS) \ - $(ADAPTATION_LIBS) \ - $(SSL_LIBS) \ - ipc/libipc.la \ - dns/libdns.la \ - base/libbase.la \ - mgr/libmgr.la \ - html/libhtml.la \ sbuf/libsbuf.la \ - debug/libdebug.la \ - store/libstore.la \ - $(SNMP_LIBS) \ - $(top_builddir)/lib/libmisccontainers.la \ - $(top_builddir)/lib/libmiscencoding.la \ - $(top_builddir)/lib/libmiscutil.la \ - $(LIBCAP_LIBS) \ - $(LIBGNUTLS_LIBS) \ - $(LIBHEIMDAL_KRB5_LIBS) \ - $(REGEXLIB) \ - $(SSLLIB) \ + base/libbase.la \ $(LIBCPPUNIT_LIBS) \ - $(LIBSYSTEMD_LIBS) \ $(COMPAT_LIB) \ - $(LIBGSS_LIBS) \ - $(LIBMIT_KRB5_LIBS) \ - $(LIBNETFILTER_CONNTRACK_LIBS) \ - $(LIBNETTLE_LIBS) \ - $(LIBPSAPI_LIBS) \ $(XTRA_LIBS) tests_testHttpRange_LDFLAGS = $(LIBADD_DL) @@ -2173,6 +1960,8 @@ tests_testHttpRequest_SOURCES = \ FwdState.h \ HappyConnOpener.cc \ HappyConnOpener.h \ + HeaderMangling.cc \ + HeaderMangling.h \ HttpBody.cc \ HttpBody.h \ tests/stub_HttpControlMsg.cc \ @@ -2473,6 +2262,8 @@ tests_testCacheManager_SOURCES = \ FwdState.h \ HappyConnOpener.cc \ HappyConnOpener.h \ + HeaderMangling.cc \ + HeaderMangling.h \ HttpBody.cc \ HttpBody.h \ tests/stub_HttpControlMsg.cc \ diff --git a/src/SquidConfig.h b/src/SquidConfig.h index 0fd54a7f65..ae2d72cc76 100644 --- a/src/SquidConfig.h +++ b/src/SquidConfig.h @@ -16,8 +16,8 @@ #include "ClientDelayConfig.h" #include "DelayConfig.h" #endif +#include "HeaderMangling.h" #include "helper/ChildConfig.h" -#include "HttpHeaderTools.h" #include "ip/Address.h" #if USE_DELAY_POOLS #include "MessageDelayPools.h" diff --git a/src/acl/HttpHeaderData.cc b/src/acl/HttpHeaderData.cc index fe62136a1e..8abdd312f7 100644 --- a/src/acl/HttpHeaderData.cc +++ b/src/acl/HttpHeaderData.cc @@ -17,7 +17,6 @@ #include "cache_cf.h" #include "ConfigParser.h" #include "debug/Stream.h" -#include "HttpHeaderTools.h" #include "sbuf/SBuf.h" #include "sbuf/StringConvert.h" diff --git a/src/auth/negotiate/UserRequest.cc b/src/auth/negotiate/UserRequest.cc index 4a41bc2324..4b0a5ea5d0 100644 --- a/src/auth/negotiate/UserRequest.cc +++ b/src/auth/negotiate/UserRequest.cc @@ -21,7 +21,6 @@ #include "helper.h" #include "helper/Reply.h" #include "http/Stream.h" -#include "HttpHeaderTools.h" #include "HttpReply.h" #include "HttpRequest.h" #include "MemBuf.h" diff --git a/src/cache_cf.cc b/src/cache_cf.cc index da21910189..412a14e952 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -41,7 +41,7 @@ #include "fqdncache.h" #include "ftp/Elements.h" #include "globals.h" -#include "HttpHeaderTools.h" +#include "HeaderMangling.h" #include "HttpUpgradeProtocolAccess.h" #include "icmp/IcmpConfig.h" #include "ip/Intercept.h" diff --git a/src/client_side.cc b/src/client_side.cc index 5122cfb6f0..cb48faba47 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -94,7 +94,6 @@ #include "http/one/TeChunkedParser.h" #include "http/Stream.h" #include "HttpHdrContRange.h" -#include "HttpHeaderTools.h" #include "HttpReply.h" #include "HttpRequest.h" #include "internal.h" diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc index 9af712660e..f031e8e584 100644 --- a/src/client_side_reply.cc +++ b/src/client_side_reply.cc @@ -21,6 +21,7 @@ #include "format/Token.h" #include "FwdState.h" #include "globals.h" +#include "HeaderMangling.h" #include "http/Stream.h" #include "HttpHeaderTools.h" #include "HttpReply.h" diff --git a/src/external_acl.cc b/src/external_acl.cc index 47e9729c76..48917fbfee 100644 --- a/src/external_acl.cc +++ b/src/external_acl.cc @@ -23,7 +23,6 @@ #include "helper.h" #include "helper/Reply.h" #include "http/Stream.h" -#include "HttpHeaderTools.h" #include "HttpReply.h" #include "HttpRequest.h" #include "ip/tools.h" diff --git a/src/http.cc b/src/http.cc index fdc9af4e9d..f7e7f2735b 100644 --- a/src/http.cc +++ b/src/http.cc @@ -30,6 +30,7 @@ #include "fd.h" #include "fde.h" #include "globals.h" +#include "HeaderMangling.h" #include "http.h" #include "http/one/ResponseParser.h" #include "http/one/TeChunkedParser.h" diff --git a/src/servers/Http1Server.cc b/src/servers/Http1Server.cc index e07d2017cb..d780076519 100644 --- a/src/servers/Http1Server.cc +++ b/src/servers/Http1Server.cc @@ -15,9 +15,9 @@ #include "client_side_request.h" #include "clientStream.h" #include "comm/Write.h" +#include "HeaderMangling.h" #include "http/one/RequestParser.h" #include "http/Stream.h" -#include "HttpHeaderTools.h" #include "servers/Http1Server.h" #include "SquidConfig.h" #include "Store.h" diff --git a/src/tests/stub_libhttp.cc b/src/tests/stub_libhttp.cc index 794a5d0cb2..09f5fa9bc6 100644 --- a/src/tests/stub_libhttp.cc +++ b/src/tests/stub_libhttp.cc @@ -9,7 +9,6 @@ #include "squid.h" #include "client_side.h" #include "comm/Connection.h" -#include "SquidConfig.h" #define STUB_API "http/libhttp.la" #include "tests/STUB.h" @@ -21,7 +20,7 @@ namespace Http Http::ContentLengthInterpreter::ContentLengthInterpreter(): value(-1), headerWideProblem(nullptr), - debugLevel(Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2), + debugLevel(DBG_IMPORTANT), sawBad(false), needsSanitizing(false), sawGood(false), diff --git a/src/tests/testHttpRange.cc b/src/tests/testHttpRange.cc index ffe611aec4..bec4324115 100644 --- a/src/tests/testHttpRange.cc +++ b/src/tests/testHttpRange.cc @@ -9,6 +9,7 @@ #include "squid.h" #include "compat/cppunit.h" #include "HttpHeaderRange.h" +#include "HttpHeaderTools.h" #include "unitTestMain.h" class TestHttpRange : public CPPUNIT_NS::TestFixture