2 * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
9 /* DEBUG: section 55 HTTP Header */
12 #include "base/EnumIterator.h"
15 #include "HttpHdrCc.h"
16 #include "HttpHdrContRange.h"
17 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
18 #include "HttpHeader.h"
19 #include "HttpHeaderFieldInfo.h"
20 #include "HttpHeaderStat.h"
21 #include "HttpHeaderTools.h"
23 #include "mgr/Registration.h"
24 #include "profiler/Profiler.h"
26 #include "SquidConfig.h"
27 //#include "SquidString.h" // pulled by HttpHdrCc.h
31 #include "TimeOrTag.h"
36 /* XXX: the whole set of API managing the entries vector should be rethought
37 * after the parse4r-ng effort is complete.
41 * On naming conventions:
43 * HTTP/1.1 defines message-header as
45 * message-header = field-name ":" [ field-value ] CRLF
47 * field-value = *( field-content | LWS )
49 * HTTP/1.1 does not give a name name a group of all message-headers in a message.
50 * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
52 * HttpHeader is an object that represents all message-headers in a message.
53 * HttpHeader does not manage start-line.
55 * HttpHeader is implemented as a collection of header "entries".
56 * An entry is a (field_id, field_name, field_value) triplet.
60 * local constants and vars
63 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
64 std::vector
<HttpHeaderFieldStat
> headerStatsTable(Http::HdrType::enumEnd_
);
66 /* request-only headers. Used for cachemgr */
67 static HttpHeaderMask RequestHeadersMask
; /* set run-time using RequestHeaders */
69 /* reply-only headers. Used for cachemgr */
70 static HttpHeaderMask ReplyHeadersMask
; /* set run-time using ReplyHeaders */
72 /* header accounting */
73 // NP: keep in sync with enum http_hdr_owner_type
74 static HttpHeaderStat HttpHeaderStats
[] = {
75 HttpHeaderStat(/*hoNone*/ "all", NULL
),
77 HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask
),
79 HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask
),
80 HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask
)
86 static int HttpHeaderStatCount
= countof(HttpHeaderStats
);
88 static int HeaderEntryParsedCount
= 0;
91 * forward declarations and local routines
96 // update parse statistics for header id; if error is true also account
97 // for errors and write to debug log what happened
98 static void httpHeaderNoteParsedEntry(Http::HdrType id
, String
const &value
, bool error
);
99 static void httpHeaderStatDump(const HttpHeaderStat
* hs
, StoreEntry
* e
);
100 /** store report about current header usage and other stats */
101 static void httpHeaderStoreReport(StoreEntry
* e
);
104 * Module initialization routines
108 httpHeaderRegisterWithCacheManager(void)
110 Mgr::RegisterAction("http_headers",
111 "HTTP Header Statistics",
112 httpHeaderStoreReport
, 0, 1);
116 httpHeaderInitModule(void)
118 /* check that we have enough space for masks */
119 assert(8 * sizeof(HttpHeaderMask
) >= Http::HdrType::enumEnd_
);
121 // masks are needed for stats page still
122 for (auto h
: WholeEnum
<Http::HdrType
>()) {
123 if (Http::HeaderLookupTable
.lookup(h
).request
)
124 CBIT_SET(RequestHeadersMask
,h
);
125 if (Http::HeaderLookupTable
.lookup(h
).reply
)
126 CBIT_SET(ReplyHeadersMask
,h
);
129 /* header stats initialized by class constructor */
130 assert(HttpHeaderStatCount
== hoReply
+ 1);
132 /* init dependent modules */
133 httpHdrCcInitModule();
134 httpHdrScInitModule();
136 httpHeaderRegisterWithCacheManager();
140 * HttpHeader Implementation
143 HttpHeader::HttpHeader() : owner (hoNone
), len (0), conflictingContentLength_(false)
145 httpHeaderMaskInit(&mask
, 0);
148 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner
): owner(anOwner
), len(0), conflictingContentLength_(false)
150 assert(anOwner
> hoNone
&& anOwner
< hoEnd
);
151 debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner
);
152 httpHeaderMaskInit(&mask
, 0);
155 HttpHeader::HttpHeader(const HttpHeader
&other
): owner(other
.owner
), len(other
.len
), conflictingContentLength_(false)
157 httpHeaderMaskInit(&mask
, 0);
158 update(&other
); // will update the mask as well
161 HttpHeader::~HttpHeader()
167 HttpHeader::operator =(const HttpHeader
&other
)
169 if (this != &other
) {
170 // we do not really care, but the caller probably does
171 assert(owner
== other
.owner
);
173 update(&other
); // will update the mask as well
175 conflictingContentLength_
= other
.conflictingContentLength_
;
184 assert(owner
> hoNone
&& owner
< hoEnd
);
185 debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner
);
187 PROF_start(HttpHeaderClean
);
189 if (owner
<= hoReply
) {
191 * An unfortunate bug. The entries array is initialized
192 * such that count is set to zero. httpHeaderClean() seems to
193 * be called both when 'hdr' is created, and destroyed. Thus,
194 * we accumulate a large number of zero counts for 'hdr' before
195 * it is ever used. Can't think of a good way to fix it, except
196 * adding a state variable that indicates whether or not 'hdr'
197 * has been used. As a hack, just never count zero-sized header
200 if (!entries
.empty())
201 HttpHeaderStats
[owner
].hdrUCountDistr
.count(entries
.size());
203 ++ HttpHeaderStats
[owner
].destroyedCount
;
205 HttpHeaderStats
[owner
].busyDestroyedCount
+= entries
.size() > 0;
206 } // if (owner <= hoReply)
208 for (HttpHeaderEntry
*e
: entries
) {
211 if (!Http::any_valid_header(e
->id
)) {
212 debugs(55, DBG_CRITICAL
, "BUG: invalid entry (" << e
->id
<< "). Ignored.");
214 if (owner
<= hoReply
)
215 HttpHeaderStats
[owner
].fieldTypeDistr
.count(e
->id
);
221 httpHeaderMaskInit(&mask
, 0);
223 conflictingContentLength_
= false;
224 PROF_stop(HttpHeaderClean
);
227 /* append entries (also see httpHeaderUpdate) */
229 HttpHeader::append(const HttpHeader
* src
)
233 debugs(55, 7, "appending hdr: " << this << " += " << src
);
235 for (auto e
: src
->entries
) {
237 addEntry(e
->clone());
242 HttpHeader::updateWarnings()
245 HttpHeaderPos pos
= HttpHeaderInitPos
;
247 // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
248 while (HttpHeaderEntry
*e
= getEntry(&pos
)) {
249 if (e
->id
== Http::HdrType::WARNING
&& (e
->getInt()/100 == 1) )
255 HttpHeader::skipUpdateHeader(const Http::HdrType id
) const
257 // RFC 7234, section 4.3.4: use fields other from Warning for update
258 return id
== Http::HdrType::WARNING
;
262 HttpHeader::update(HttpHeader
const *fresh
)
264 const HttpHeaderEntry
*e
;
265 HttpHeaderPos pos
= HttpHeaderInitPos
;
267 assert(this != fresh
);
271 while ((e
= fresh
->getEntry(&pos
))) {
272 /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
274 if (skipUpdateHeader(e
->id
))
277 if (e
->id
!= Http::HdrType::OTHER
)
280 delByName(e
->name
.termedBuf());
283 pos
= HttpHeaderInitPos
;
284 while ((e
= fresh
->getEntry(&pos
))) {
285 /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
287 if (skipUpdateHeader(e
->id
))
290 debugs(55, 7, "Updating header '" << Http::HeaderLookupTable
.lookup(e
->id
).name
<< "' in cached entry");
292 addEntry(e
->clone());
297 HttpHeader::parse(const char *header_start
, size_t hdrLen
)
299 const char *field_ptr
= header_start
;
300 const char *header_end
= header_start
+ hdrLen
; // XXX: remove
301 HttpHeaderEntry
*e
, *e2
;
302 int warnOnError
= (Config
.onoff
.relaxed_header_parser
<= 0 ? DBG_IMPORTANT
: 2);
304 PROF_start(HttpHeaderParse
);
306 assert(header_start
&& header_end
);
307 debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl
<< getStringPrefix(header_start
, hdrLen
));
308 ++ HttpHeaderStats
[owner
].parsedCount
;
311 if ((nulpos
= (char*)memchr(header_start
, '\0', hdrLen
))) {
312 debugs(55, DBG_IMPORTANT
, "WARNING: HTTP header contains NULL characters {" <<
313 getStringPrefix(header_start
, nulpos
-header_start
) << "}\nNULL\n{" << getStringPrefix(nulpos
+1, hdrLen
-(nulpos
-header_start
)-1));
314 PROF_stop(HttpHeaderParse
);
319 /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
320 * continuation lines start with a (single) space or tab */
321 while (field_ptr
< header_end
) {
322 const char *field_start
= field_ptr
;
323 const char *field_end
;
326 const char *this_line
= field_ptr
;
327 field_ptr
= (const char *)memchr(field_ptr
, '\n', header_end
- field_ptr
);
331 PROF_stop(HttpHeaderParse
);
336 field_end
= field_ptr
;
338 ++field_ptr
; /* Move to next line */
340 if (field_end
> this_line
&& field_end
[-1] == '\r') {
341 --field_end
; /* Ignore CR LF */
343 if (owner
== hoRequest
&& field_end
> this_line
) {
345 for (const char *p
= this_line
; p
< field_end
&& cr_only
; ++p
) {
350 debugs(55, DBG_IMPORTANT
, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
351 "header field to prevent request smuggling attacks: {" <<
352 getStringPrefix(header_start
, hdrLen
) << "}");
353 PROF_stop(HttpHeaderParse
);
360 /* Barf on stray CR characters */
361 if (memchr(this_line
, '\r', field_end
- this_line
)) {
362 debugs(55, warnOnError
, "WARNING: suspicious CR characters in HTTP header {" <<
363 getStringPrefix(field_start
, field_end
-field_start
) << "}");
365 if (Config
.onoff
.relaxed_header_parser
) {
366 char *p
= (char *) this_line
; /* XXX Warning! This destroys original header content and violates specifications somewhat */
368 while ((p
= (char *)memchr(p
, '\r', field_end
- p
)) != NULL
) {
373 PROF_stop(HttpHeaderParse
);
379 if (this_line
+ 1 == field_end
&& this_line
> field_start
) {
380 debugs(55, warnOnError
, "WARNING: Blank continuation line in HTTP header {" <<
381 getStringPrefix(header_start
, hdrLen
) << "}");
382 PROF_stop(HttpHeaderParse
);
386 } while (field_ptr
< header_end
&& (*field_ptr
== ' ' || *field_ptr
== '\t'));
388 if (field_start
== field_end
) {
389 if (field_ptr
< header_end
) {
390 debugs(55, warnOnError
, "WARNING: unparseable HTTP header field near {" <<
391 getStringPrefix(field_start
, hdrLen
-(field_start
-header_start
)) << "}");
392 PROF_stop(HttpHeaderParse
);
397 break; /* terminating blank line */
400 if ((e
= HttpHeaderEntry::parse(field_start
, field_end
)) == NULL
) {
401 debugs(55, warnOnError
, "WARNING: unparseable HTTP header field {" <<
402 getStringPrefix(field_start
, field_end
-field_start
) << "}");
403 debugs(55, warnOnError
, " in {" << getStringPrefix(header_start
, hdrLen
) << "}");
405 if (Config
.onoff
.relaxed_header_parser
)
408 PROF_stop(HttpHeaderParse
);
413 // XXX: RFC 7230 Section 3.3.3 item #4 requires sending a 502 error in
414 // several cases that we do not yet cover. TODO: Rewrite to cover more.
415 if (e
->id
== Http::HdrType::CONTENT_LENGTH
&& (e2
= findEntry(e
->id
)) != nullptr) {
416 if (e
->value
!= e2
->value
) {
418 debugs(55, warnOnError
, "WARNING: found two conflicting content-length headers in {" <<
419 getStringPrefix(header_start
, hdrLen
) << "}");
421 if (!Config
.onoff
.relaxed_header_parser
) {
423 PROF_stop(HttpHeaderParse
);
428 if (!httpHeaderParseOffset(e
->value
.termedBuf(), &l1
)) {
429 debugs(55, DBG_IMPORTANT
, "WARNING: Unparseable content-length '" << e
->value
<< "'");
432 } else if (!httpHeaderParseOffset(e2
->value
.termedBuf(), &l2
)) {
433 debugs(55, DBG_IMPORTANT
, "WARNING: Unparseable content-length '" << e2
->value
<< "'");
437 conflictingContentLength_
= true;
442 debugs(55, warnOnError
, "NOTICE: found double content-length header");
445 if (Config
.onoff
.relaxed_header_parser
)
448 PROF_stop(HttpHeaderParse
);
454 if (e
->id
== Http::HdrType::OTHER
&& stringHasWhitespace(e
->name
.termedBuf())) {
455 debugs(55, warnOnError
, "WARNING: found whitespace in HTTP header name {" <<
456 getStringPrefix(field_start
, field_end
-field_start
) << "}");
458 if (!Config
.onoff
.relaxed_header_parser
) {
460 PROF_stop(HttpHeaderParse
);
470 // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
471 delById(Http::HdrType::CONTENT_LENGTH
);
472 // RFC 7230 section 3.3.3 #4: ignore Content-Length conflicts with Transfer-Encoding
473 conflictingContentLength_
= false;
474 } else if (conflictingContentLength_
) {
475 // ensure our callers do not see the conflicting Content-Length value
476 delById(Http::HdrType::CONTENT_LENGTH
);
479 PROF_stop(HttpHeaderParse
);
480 return 1; /* even if no fields where found, it is a valid header */
483 /* packs all the entries using supplied packer */
485 HttpHeader::packInto(Packable
* p
, bool mask_sensitive_info
) const
487 HttpHeaderPos pos
= HttpHeaderInitPos
;
488 const HttpHeaderEntry
*e
;
490 debugs(55, 7, this << " into " << p
<<
491 (mask_sensitive_info
? " while masking" : ""));
492 /* pack all entries one by one */
493 while ((e
= getEntry(&pos
))) {
494 if (!mask_sensitive_info
) {
499 bool maskThisEntry
= false;
501 case Http::HdrType::AUTHORIZATION
:
502 case Http::HdrType::PROXY_AUTHORIZATION
:
503 maskThisEntry
= true;
506 case Http::HdrType::FTP_ARGUMENTS
:
507 if (const HttpHeaderEntry
*cmd
= findEntry(Http::HdrType::FTP_COMMAND
))
508 maskThisEntry
= (cmd
->value
== "PASS");
515 p
->append(e
->name
.rawBuf(), e
->name
.size());
516 p
->append(": ** NOT DISPLAYED **\r\n", 23);
522 /* Pack in the "special" entries */
527 /* returns next valid entry */
529 HttpHeader::getEntry(HttpHeaderPos
* pos
) const
532 assert(*pos
>= HttpHeaderInitPos
&& *pos
< static_cast<ssize_t
>(entries
.size()));
534 for (++(*pos
); *pos
< static_cast<ssize_t
>(entries
.size()); ++(*pos
)) {
536 return static_cast<HttpHeaderEntry
*>(entries
[*pos
]);
543 * returns a pointer to a specified entry if any
544 * note that we return one entry so it does not make much sense to ask for
548 HttpHeader::findEntry(Http::HdrType id
) const
550 assert(any_registered_header(id
));
551 assert(!Http::HeaderLookupTable
.lookup(id
).list
);
553 /* check mask first */
555 if (!CBIT_TEST(mask
, id
))
558 /* looks like we must have it, do linear search */
559 for (auto e
: entries
) {
560 if (e
&& e
->id
== id
)
564 /* hm.. we thought it was there, but it was not found */
566 return nullptr; /* not reached */
570 * same as httpHeaderFindEntry
573 HttpHeader::findLastEntry(Http::HdrType id
) const
575 assert(any_registered_header(id
));
576 assert(!Http::HeaderLookupTable
.lookup(id
).list
);
578 /* check mask first */
579 if (!CBIT_TEST(mask
, id
))
582 for (auto e
= entries
.rbegin(); e
!= entries
.rend(); ++e
) {
583 if (*e
&& (*e
)->id
== id
)
587 /* hm.. we thought it was there, but it was not found */
589 return nullptr; /* not reached */
593 * deletes all fields with a given name if any, returns #fields deleted;
596 HttpHeader::delByName(const char *name
)
599 HttpHeaderPos pos
= HttpHeaderInitPos
;
601 httpHeaderMaskInit(&mask
, 0); /* temporal inconsistency */
602 debugs(55, 9, "deleting '" << name
<< "' fields in hdr " << this);
604 while ((e
= getEntry(&pos
))) {
605 if (!e
->name
.caseCmp(name
))
608 CBIT_SET(mask
, e
->id
);
614 /* deletes all entries with a given id, returns the #entries deleted */
616 HttpHeader::delById(Http::HdrType id
)
618 debugs(55, 8, this << " del-by-id " << id
);
619 assert(any_registered_header(id
));
621 if (!CBIT_TEST(mask
, id
))
626 HttpHeaderPos pos
= HttpHeaderInitPos
;
627 while (HttpHeaderEntry
*e
= getEntry(&pos
)) {
629 delAt(pos
, count
); // deletes e
638 * deletes an entry at pos and leaves a gap; leaving a gap makes it
639 * possible to iterate(search) and delete fields at the same time
640 * NOTE: Does not update the header mask. Caller must follow up with
641 * a call to refreshMask() if headers_deleted was incremented.
644 HttpHeader::delAt(HttpHeaderPos pos
, int &headers_deleted
)
647 assert(pos
>= HttpHeaderInitPos
&& pos
< static_cast<ssize_t
>(entries
.size()));
648 e
= static_cast<HttpHeaderEntry
*>(entries
[pos
]);
650 /* decrement header length, allow for ": " and crlf */
651 len
-= e
->name
.size() + 2 + e
->value
.size() + 2;
658 * Compacts the header storage
661 HttpHeader::compact()
663 // TODO: optimize removal, or possibly make it so that's not needed.
664 entries
.erase( std::remove(entries
.begin(), entries
.end(), nullptr),
669 * Refreshes the header mask. Required after delAt() calls.
672 HttpHeader::refreshMask()
674 httpHeaderMaskInit(&mask
, 0);
675 debugs(55, 7, "refreshing the mask in hdr " << this);
676 for (auto e
: entries
) {
678 CBIT_SET(mask
, e
->id
);
683 * does not call e->clone() so one should not reuse "*e"
686 HttpHeader::addEntry(HttpHeaderEntry
* e
)
689 assert(any_HdrType_enum_value(e
->id
));
690 assert(e
->name
.size());
692 debugs(55, 7, this << " adding entry: " << e
->id
<< " at " << entries
.size());
694 if (e
->id
!= Http::HdrType::BAD_HDR
) {
695 if (CBIT_TEST(mask
, e
->id
)) {
696 ++ headerStatsTable
[e
->id
].repCount
;
698 CBIT_SET(mask
, e
->id
);
702 entries
.push_back(e
);
704 /* increment header length, allow for ": " and crlf */
705 len
+= e
->name
.size() + 2 + e
->value
.size() + 2;
709 * does not call e->clone() so one should not reuse "*e"
712 HttpHeader::insertEntry(HttpHeaderEntry
* e
)
715 assert(any_valid_header(e
->id
));
717 debugs(55, 7, this << " adding entry: " << e
->id
<< " at " << entries
.size());
719 // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
720 if (CBIT_TEST(mask
, e
->id
)) {
721 ++ headerStatsTable
[e
->id
].repCount
;
723 CBIT_SET(mask
, e
->id
);
726 entries
.insert(entries
.begin(),e
);
728 /* increment header length, allow for ": " and crlf */
729 len
+= e
->name
.size() + 2 + e
->value
.size() + 2;
733 HttpHeader::getList(Http::HdrType id
, String
*s
) const
735 debugs(55, 9, this << " joining for id " << id
);
736 /* only fields from ListHeaders array can be "listed" */
737 assert(Http::HeaderLookupTable
.lookup(id
).list
);
739 if (!CBIT_TEST(mask
, id
))
742 for (auto e
: entries
) {
743 if (e
&& e
->id
== id
)
744 strListAdd(s
, e
->value
.termedBuf(), ',');
748 * note: we might get an empty (size==0) string if there was an "empty"
749 * header. This results in an empty length String, which may have a NULL
752 /* temporary warning: remove it? (Is it useful for diagnostics ?) */
754 debugs(55, 3, "empty list header: " << Http::HeaderLookupTable
.lookup(id
).name
<< "(" << id
<< ")");
756 debugs(55, 6, this << ": joined for id " << id
<< ": " << s
);
761 /* return a list of entries with the same id separated by ',' and ws */
763 HttpHeader::getList(Http::HdrType id
) const
766 HttpHeaderPos pos
= HttpHeaderInitPos
;
767 debugs(55, 9, this << "joining for id " << id
);
768 /* only fields from ListHeaders array can be "listed" */
769 assert(Http::HeaderLookupTable
.lookup(id
).list
);
771 if (!CBIT_TEST(mask
, id
))
776 while ((e
= getEntry(&pos
))) {
778 strListAdd(&s
, e
->value
.termedBuf(), ',');
782 * note: we might get an empty (size==0) string if there was an "empty"
783 * header. This results in an empty length String, which may have a NULL
786 /* temporary warning: remove it? (Is it useful for diagnostics ?) */
788 debugs(55, 3, "empty list header: " << Http::HeaderLookupTable
.lookup(id
).name
<< "(" << id
<< ")");
790 debugs(55, 6, this << ": joined for id " << id
<< ": " << s
);
795 /* return a string or list of entries with the same id separated by ',' and ws */
797 HttpHeader::getStrOrList(Http::HdrType id
) const
801 if (Http::HeaderLookupTable
.lookup(id
).list
)
804 if ((e
= findEntry(id
)))
811 * Returns the value of the specified header and/or an undefined String.
814 HttpHeader::getByName(const char *name
) const
817 // ignore presence: return undefined string if an empty header is present
818 (void)getByNameIfPresent(name
, strlen(name
), result
);
823 HttpHeader::getByName(const SBuf
&name
) const
826 // ignore presence: return undefined string if an empty header is present
827 (void)getByNameIfPresent(name
, result
);
832 HttpHeader::getById(Http::HdrType id
) const
835 (void)getByIdIfPresent(id
,result
);
840 HttpHeader::getByNameIfPresent(const SBuf
&s
, String
&result
) const
842 return getByNameIfPresent(s
.rawContent(), s
.length(), result
);
846 HttpHeader::getByIdIfPresent(Http::HdrType id
, String
&result
) const
848 if (id
== Http::HdrType::BAD_HDR
)
852 result
= getStrOrList(id
);
857 HttpHeader::getByNameIfPresent(const char *name
, int namelen
, String
&result
) const
860 HttpHeaderPos pos
= HttpHeaderInitPos
;
865 /* First try the quick path */
866 id
= Http::HeaderLookupTable
.lookup(name
,namelen
).id
;
868 if (id
!= Http::HdrType::BAD_HDR
) {
869 if (getByIdIfPresent(id
, result
))
873 /* Sorry, an unknown header name. Do linear search */
875 while ((e
= getEntry(&pos
))) {
876 if (e
->id
== Http::HdrType::OTHER
&& e
->name
.caseCmp(name
) == 0) {
878 strListAdd(&result
, e
->value
.termedBuf(), ',');
886 * Returns a the value of the specified list member, if any.
889 HttpHeader::getByNameListMember(const char *name
, const char *member
, const char separator
) const
892 const char *pos
= NULL
;
895 int mlen
= strlen(member
);
899 header
= getByName(name
);
903 while (strListGetItem(&header
, separator
, &item
, &ilen
, &pos
)) {
904 if (strncmp(item
, member
, mlen
) == 0 && item
[mlen
] == '=') {
905 result
.append(item
+ mlen
+ 1, ilen
- mlen
- 1);
914 * returns a the value of the specified list member, if any.
917 HttpHeader::getListMember(Http::HdrType id
, const char *member
, const char separator
) const
920 const char *pos
= NULL
;
923 int mlen
= strlen(member
);
925 assert(any_registered_header(id
));
927 header
= getStrOrList(id
);
930 while (strListGetItem(&header
, separator
, &item
, &ilen
, &pos
)) {
931 if (strncmp(item
, member
, mlen
) == 0 && item
[mlen
] == '=') {
932 result
.append(item
+ mlen
+ 1, ilen
- mlen
- 1);
941 /* test if a field is present */
943 HttpHeader::has(Http::HdrType id
) const
945 assert(any_registered_header(id
));
946 debugs(55, 9, this << " lookup for " << id
);
947 return CBIT_TEST(mask
, id
);
951 HttpHeader::putInt(Http::HdrType id
, int number
)
953 assert(any_registered_header(id
));
954 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftInt
); /* must be of an appropriate type */
956 addEntry(new HttpHeaderEntry(id
, NULL
, xitoa(number
)));
960 HttpHeader::putInt64(Http::HdrType id
, int64_t number
)
962 assert(any_registered_header(id
));
963 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftInt64
); /* must be of an appropriate type */
965 addEntry(new HttpHeaderEntry(id
, NULL
, xint64toa(number
)));
969 HttpHeader::putTime(Http::HdrType id
, time_t htime
)
971 assert(any_registered_header(id
));
972 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftDate_1123
); /* must be of an appropriate type */
974 addEntry(new HttpHeaderEntry(id
, NULL
, mkrfc1123(htime
)));
978 HttpHeader::putStr(Http::HdrType id
, const char *str
)
980 assert(any_registered_header(id
));
981 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftStr
); /* must be of an appropriate type */
983 addEntry(new HttpHeaderEntry(id
, NULL
, str
));
987 HttpHeader::putAuth(const char *auth_scheme
, const char *realm
)
989 assert(auth_scheme
&& realm
);
990 httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE
, "%s realm=\"%s\"", auth_scheme
, realm
);
994 HttpHeader::putCc(const HttpHdrCc
* cc
)
997 /* remove old directives if any */
998 delById(Http::HdrType::CACHE_CONTROL
);
1004 addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL
, NULL
, mb
.buf
));
1010 HttpHeader::putContRange(const HttpHdrContRange
* cr
)
1013 /* remove old directives if any */
1014 delById(Http::HdrType::CONTENT_RANGE
);
1018 httpHdrContRangePackInto(cr
, &mb
);
1020 addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE
, NULL
, mb
.buf
));
1026 HttpHeader::putRange(const HttpHdrRange
* range
)
1029 /* remove old directives if any */
1030 delById(Http::HdrType::RANGE
);
1034 range
->packInto(&mb
);
1036 addEntry(new HttpHeaderEntry(Http::HdrType::RANGE
, NULL
, mb
.buf
));
1042 HttpHeader::putSc(HttpHdrSc
*sc
)
1045 /* remove old directives if any */
1046 delById(Http::HdrType::SURROGATE_CONTROL
);
1052 addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL
, NULL
, mb
.buf
));
1058 HttpHeader::putWarning(const int code
, const char *const text
)
1061 snprintf(buf
, sizeof(buf
), "%i %s \"%s\"", code
, visible_appname_string
, text
);
1062 putStr(Http::HdrType::WARNING
, buf
);
1065 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1067 HttpHeader::putExt(const char *name
, const char *value
)
1069 assert(name
&& value
);
1070 debugs(55, 8, this << " adds ext entry " << name
<< " : " << value
);
1071 addEntry(new HttpHeaderEntry(Http::HdrType::OTHER
, name
, value
));
1075 HttpHeader::getInt(Http::HdrType id
) const
1077 assert(any_registered_header(id
));
1078 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftInt
); /* must be of an appropriate type */
1081 if ((e
= findEntry(id
)))
1088 HttpHeader::getInt64(Http::HdrType id
) const
1090 assert(any_registered_header(id
));
1091 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftInt64
); /* must be of an appropriate type */
1094 if ((e
= findEntry(id
)))
1095 return e
->getInt64();
1101 HttpHeader::getTime(Http::HdrType id
) const
1105 assert(any_registered_header(id
));
1106 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftDate_1123
); /* must be of an appropriate type */
1108 if ((e
= findEntry(id
))) {
1109 value
= parse_rfc1123(e
->value
.termedBuf());
1110 httpHeaderNoteParsedEntry(e
->id
, e
->value
, value
< 0);
1116 /* sync with httpHeaderGetLastStr */
1118 HttpHeader::getStr(Http::HdrType id
) const
1121 assert(any_registered_header(id
));
1122 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftStr
); /* must be of an appropriate type */
1124 if ((e
= findEntry(id
))) {
1125 httpHeaderNoteParsedEntry(e
->id
, e
->value
, false); /* no errors are possible */
1126 return e
->value
.termedBuf();
1134 HttpHeader::getLastStr(Http::HdrType id
) const
1137 assert(any_registered_header(id
));
1138 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftStr
); /* must be of an appropriate type */
1140 if ((e
= findLastEntry(id
))) {
1141 httpHeaderNoteParsedEntry(e
->id
, e
->value
, false); /* no errors are possible */
1142 return e
->value
.termedBuf();
1149 HttpHeader::getCc() const
1151 if (!CBIT_TEST(mask
, Http::HdrType::CACHE_CONTROL
))
1153 PROF_start(HttpHeader_getCc
);
1156 getList(Http::HdrType::CACHE_CONTROL
, &s
);
1158 HttpHdrCc
*cc
=new HttpHdrCc();
1160 if (!cc
->parse(s
)) {
1165 ++ HttpHeaderStats
[owner
].ccParsedCount
;
1168 httpHdrCcUpdateStats(cc
, &HttpHeaderStats
[owner
].ccTypeDistr
);
1170 httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL
, s
, !cc
);
1172 PROF_stop(HttpHeader_getCc
);
1178 HttpHeader::getRange() const
1180 HttpHdrRange
*r
= NULL
;
1182 /* some clients will send "Request-Range" _and_ *matching* "Range"
1183 * who knows, some clients might send Request-Range only;
1184 * this "if" should work correctly in both cases;
1185 * hopefully no clients send mismatched headers! */
1187 if ((e
= findEntry(Http::HdrType::RANGE
)) ||
1188 (e
= findEntry(Http::HdrType::REQUEST_RANGE
))) {
1189 r
= HttpHdrRange::ParseCreate(&e
->value
);
1190 httpHeaderNoteParsedEntry(e
->id
, e
->value
, !r
);
1197 HttpHeader::getSc() const
1199 if (!CBIT_TEST(mask
, Http::HdrType::SURROGATE_CONTROL
))
1204 (void) getList(Http::HdrType::SURROGATE_CONTROL
, &s
);
1206 HttpHdrSc
*sc
= httpHdrScParseCreate(s
);
1208 ++ HttpHeaderStats
[owner
].ccParsedCount
;
1211 sc
->updateStats(&HttpHeaderStats
[owner
].scTypeDistr
);
1213 httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL
, s
, !sc
);
1219 HttpHeader::getContRange() const
1221 HttpHdrContRange
*cr
= NULL
;
1224 if ((e
= findEntry(Http::HdrType::CONTENT_RANGE
))) {
1225 cr
= httpHdrContRangeParseCreate(e
->value
.termedBuf());
1226 httpHeaderNoteParsedEntry(e
->id
, e
->value
, !cr
);
1233 HttpHeader::getAuth(Http::HdrType id
, const char *auth_scheme
) const
1237 assert(auth_scheme
);
1240 if (!field
) /* no authorization field */
1243 l
= strlen(auth_scheme
);
1245 if (!l
|| strncasecmp(field
, auth_scheme
, l
)) /* wrong scheme */
1250 if (!xisspace(*field
)) /* wrong scheme */
1253 /* skip white space */
1254 for (; field
&& xisspace(*field
); ++field
);
1256 if (!*field
) /* no authorization cookie */
1259 static char decodedAuthToken
[8192];
1260 struct base64_decode_ctx ctx
;
1261 base64_decode_init(&ctx
);
1262 size_t decodedLen
= 0;
1263 if (!base64_decode_update(&ctx
, &decodedLen
, reinterpret_cast<uint8_t*>(decodedAuthToken
), strlen(field
), reinterpret_cast<const uint8_t*>(field
)) ||
1264 !base64_decode_final(&ctx
)) {
1267 decodedAuthToken
[decodedLen
] = '\0';
1268 return decodedAuthToken
;
1272 HttpHeader::getETag(Http::HdrType id
) const
1274 ETag etag
= {NULL
, -1};
1276 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftETag
); /* must be of an appropriate type */
1278 if ((e
= findEntry(id
)))
1279 etagParseInit(&etag
, e
->value
.termedBuf());
1285 HttpHeader::getTimeOrTag(Http::HdrType id
) const
1289 assert(Http::HeaderLookupTable
.lookup(id
).type
== Http::HdrFieldType::ftDate_1123_or_ETag
); /* must be of an appropriate type */
1290 memset(&tot
, 0, sizeof(tot
));
1292 if ((e
= findEntry(id
))) {
1293 const char *str
= e
->value
.termedBuf();
1294 /* try as an ETag */
1296 if (etagParseInit(&tot
.tag
, str
)) {
1297 tot
.valid
= tot
.tag
.str
!= NULL
;
1300 /* or maybe it is time? */
1301 tot
.time
= parse_rfc1123(str
);
1302 tot
.valid
= tot
.time
>= 0;
1307 assert(tot
.time
< 0 || !tot
.tag
.str
); /* paranoid */
1315 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId
, const char *aName
, const char *aValue
)
1317 assert(any_HdrType_enum_value(anId
));
1320 if (id
!= Http::HdrType::OTHER
)
1321 name
= Http::HeaderLookupTable
.lookup(id
).name
;
1327 if (id
!= Http::HdrType::BAD_HDR
)
1328 ++ headerStatsTable
[id
].aliveCount
;
1330 debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name
<< " : " << value
);
1333 HttpHeaderEntry::~HttpHeaderEntry()
1335 debugs(55, 9, "destroying entry " << this << ": '" << name
<< ": " << value
<< "'");
1337 if (id
!= Http::HdrType::BAD_HDR
) {
1338 assert(headerStatsTable
[id
].aliveCount
);
1339 -- headerStatsTable
[id
].aliveCount
;
1340 id
= Http::HdrType::BAD_HDR
; // it already is BAD_HDR, no sense in resetting it
1345 /* parses and inits header entry, returns true/false */
1347 HttpHeaderEntry::parse(const char *field_start
, const char *field_end
)
1349 /* note: name_start == field_start */
1350 const char *name_end
= (const char *)memchr(field_start
, ':', field_end
- field_start
);
1351 int name_len
= name_end
? name_end
- field_start
:0;
1352 const char *value_start
= field_start
+ name_len
+ 1; /* skip ':' */
1353 /* note: value_end == field_end */
1355 ++ HeaderEntryParsedCount
;
1357 /* do we have a valid field name within this field? */
1359 if (!name_len
|| name_end
> field_end
)
1362 if (name_len
> 65534) {
1363 /* String must be LESS THAN 64K and it adds a terminating NULL */
1364 debugs(55, DBG_IMPORTANT
, "WARNING: ignoring header name of " << name_len
<< " bytes");
1368 if (Config
.onoff
.relaxed_header_parser
&& xisspace(field_start
[name_len
- 1])) {
1369 debugs(55, Config
.onoff
.relaxed_header_parser
<= 0 ? 1 : 2,
1370 "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start
, field_end
-field_start
) << "'");
1372 while (name_len
> 0 && xisspace(field_start
[name_len
- 1]))
1379 /* now we know we can parse it */
1381 debugs(55, 9, "parsing HttpHeaderEntry: near '" << getStringPrefix(field_start
, field_end
-field_start
) << "'");
1383 /* is it a "known" field? */
1384 Http::HdrType id
= Http::HeaderLookupTable
.lookup(field_start
,name_len
).id
;
1385 debugs(55, 9, "got hdr-id=" << id
);
1391 if (id
== Http::HdrType::BAD_HDR
)
1392 id
= Http::HdrType::OTHER
;
1394 /* set field name */
1395 if (id
== Http::HdrType::OTHER
)
1396 name
.limitInit(field_start
, name_len
);
1398 name
= Http::HeaderLookupTable
.lookup(id
).name
;
1400 /* trim field value */
1401 while (value_start
< field_end
&& xisspace(*value_start
))
1404 while (value_start
< field_end
&& xisspace(field_end
[-1]))
1407 if (field_end
- value_start
> 65534) {
1408 /* String must be LESS THAN 64K and it adds a terminating NULL */
1409 debugs(55, DBG_IMPORTANT
, "WARNING: ignoring '" << name
<< "' header of " << (field_end
- value_start
) << " bytes");
1411 if (id
== Http::HdrType::OTHER
)
1417 /* set field value */
1418 value
.limitInit(value_start
, field_end
- value_start
);
1420 if (id
!= Http::HdrType::BAD_HDR
)
1421 ++ headerStatsTable
[id
].seenCount
;
1423 debugs(55, 9, "parsed HttpHeaderEntry: '" << name
<< ": " << value
<< "'");
1425 return new HttpHeaderEntry(id
, name
.termedBuf(), value
.termedBuf());
1429 HttpHeaderEntry::clone() const
1431 return new HttpHeaderEntry(id
, name
.termedBuf(), value
.termedBuf());
1435 HttpHeaderEntry::packInto(Packable
* p
) const
1438 p
->append(name
.rawBuf(), name
.size());
1440 p
->append(value
.rawBuf(), value
.size());
1441 p
->append("\r\n", 2);
1445 HttpHeaderEntry::getInt() const
1448 int ok
= httpHeaderParseInt(value
.termedBuf(), &val
);
1449 httpHeaderNoteParsedEntry(id
, value
, ok
== 0);
1450 /* XXX: Should we check ok - ie
1451 * return ok ? -1 : value;
1457 HttpHeaderEntry::getInt64() const
1460 int ok
= httpHeaderParseOffset(value
.termedBuf(), &val
);
1461 httpHeaderNoteParsedEntry(id
, value
, ok
== 0);
1462 /* XXX: Should we check ok - ie
1463 * return ok ? -1 : value;
1469 httpHeaderNoteParsedEntry(Http::HdrType id
, String
const &context
, bool error
)
1471 if (id
!= Http::HdrType::BAD_HDR
)
1472 ++ headerStatsTable
[id
].parsCount
;
1475 if (id
!= Http::HdrType::BAD_HDR
)
1476 ++ headerStatsTable
[id
].errCount
;
1477 debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable
.lookup(id
).name
<< ": " << context
<< "'");
1485 /* tmp variable used to pass stat info to dumpers */
1486 extern const HttpHeaderStat
*dump_stat
; /* argh! */
1487 const HttpHeaderStat
*dump_stat
= NULL
;
1490 httpHeaderFieldStatDumper(StoreEntry
* sentry
, int, double val
, double, int count
)
1492 const int id
= static_cast<int>(val
);
1493 const bool valid_id
= Http::any_valid_header(static_cast<Http::HdrType
>(id
));
1494 const char *name
= valid_id
? Http::HeaderLookupTable
.lookup(static_cast<Http::HdrType
>(id
)).name
: "INVALID";
1495 int visible
= count
> 0;
1496 /* for entries with zero count, list only those that belong to current type of message */
1498 if (!visible
&& valid_id
&& dump_stat
->owner_mask
)
1499 visible
= CBIT_TEST(*dump_stat
->owner_mask
, id
);
1502 storeAppendPrintf(sentry
, "%2d\t %-20s\t %5d\t %6.2f\n",
1503 id
, name
, count
, xdiv(count
, dump_stat
->busyDestroyedCount
));
1507 httpHeaderFldsPerHdrDumper(StoreEntry
* sentry
, int idx
, double val
, double, int count
)
1510 storeAppendPrintf(sentry
, "%2d\t %5d\t %5d\t %6.2f\n",
1511 idx
, (int) val
, count
,
1512 xpercent(count
, dump_stat
->destroyedCount
));
1516 httpHeaderStatDump(const HttpHeaderStat
* hs
, StoreEntry
* e
)
1522 storeAppendPrintf(e
, "\nHeader Stats: %s\n", hs
->label
);
1523 storeAppendPrintf(e
, "\nField type distribution\n");
1524 storeAppendPrintf(e
, "%2s\t %-20s\t %5s\t %6s\n",
1525 "id", "name", "count", "#/header");
1526 hs
->fieldTypeDistr
.dump(e
, httpHeaderFieldStatDumper
);
1527 storeAppendPrintf(e
, "\nCache-control directives distribution\n");
1528 storeAppendPrintf(e
, "%2s\t %-20s\t %5s\t %6s\n",
1529 "id", "name", "count", "#/cc_field");
1530 hs
->ccTypeDistr
.dump(e
, httpHdrCcStatDumper
);
1531 storeAppendPrintf(e
, "\nSurrogate-control directives distribution\n");
1532 storeAppendPrintf(e
, "%2s\t %-20s\t %5s\t %6s\n",
1533 "id", "name", "count", "#/sc_field");
1534 hs
->scTypeDistr
.dump(e
, httpHdrScStatDumper
);
1535 storeAppendPrintf(e
, "\nNumber of fields per header distribution\n");
1536 storeAppendPrintf(e
, "%2s\t %-5s\t %5s\t %6s\n",
1537 "id", "#flds", "count", "%total");
1538 hs
->hdrUCountDistr
.dump(e
, httpHeaderFldsPerHdrDumper
);
1539 storeAppendPrintf(e
, "\n");
1544 httpHeaderStoreReport(StoreEntry
* e
)
1549 HttpHeaderStats
[0].parsedCount
=
1550 HttpHeaderStats
[hoRequest
].parsedCount
+ HttpHeaderStats
[hoReply
].parsedCount
;
1551 HttpHeaderStats
[0].ccParsedCount
=
1552 HttpHeaderStats
[hoRequest
].ccParsedCount
+ HttpHeaderStats
[hoReply
].ccParsedCount
;
1553 HttpHeaderStats
[0].destroyedCount
=
1554 HttpHeaderStats
[hoRequest
].destroyedCount
+ HttpHeaderStats
[hoReply
].destroyedCount
;
1555 HttpHeaderStats
[0].busyDestroyedCount
=
1556 HttpHeaderStats
[hoRequest
].busyDestroyedCount
+ HttpHeaderStats
[hoReply
].busyDestroyedCount
;
1558 for (i
= 1; i
< HttpHeaderStatCount
; ++i
) {
1559 httpHeaderStatDump(HttpHeaderStats
+ i
, e
);
1562 /* field stats for all messages */
1563 storeAppendPrintf(e
, "\nHttp Fields Stats (replies and requests)\n");
1565 storeAppendPrintf(e
, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1566 "id", "name", "#alive", "%err", "%repeat");
1568 // scan heaaderTable and output
1569 for (auto h
: WholeEnum
<Http::HdrType
>()) {
1570 auto stats
= headerStatsTable
[h
];
1571 storeAppendPrintf(e
, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1572 Http::HeaderLookupTable
.lookup(h
).id
,
1573 Http::HeaderLookupTable
.lookup(h
).name
,
1575 xpercent(stats
.errCount
, stats
.parsCount
),
1576 xpercent(stats
.repCount
, stats
.seenCount
));
1579 storeAppendPrintf(e
, "Headers Parsed: %d + %d = %d\n",
1580 HttpHeaderStats
[hoRequest
].parsedCount
,
1581 HttpHeaderStats
[hoReply
].parsedCount
,
1582 HttpHeaderStats
[0].parsedCount
);
1583 storeAppendPrintf(e
, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount
);
1587 HttpHeader::hasListMember(Http::HdrType id
, const char *member
, const char separator
) const
1590 const char *pos
= NULL
;
1593 int mlen
= strlen(member
);
1595 assert(any_registered_header(id
));
1597 String
header (getStrOrList(id
));
1599 while (strListGetItem(&header
, separator
, &item
, &ilen
, &pos
)) {
1600 if (strncasecmp(item
, member
, mlen
) == 0
1601 && (item
[mlen
] == '=' || item
[mlen
] == separator
|| item
[mlen
] == ';' || item
[mlen
] == '\0')) {
1611 HttpHeader::hasByNameListMember(const char *name
, const char *member
, const char separator
) const
1614 const char *pos
= NULL
;
1617 int mlen
= strlen(member
);
1621 String
header (getByName(name
));
1623 while (strListGetItem(&header
, separator
, &item
, &ilen
, &pos
)) {
1624 if (strncasecmp(item
, member
, mlen
) == 0
1625 && (item
[mlen
] == '=' || item
[mlen
] == separator
|| item
[mlen
] == ';' || item
[mlen
] == '\0')) {
1635 HttpHeader::removeHopByHopEntries()
1637 removeConnectionHeaderEntries();
1639 const HttpHeaderEntry
*e
;
1640 HttpHeaderPos pos
= HttpHeaderInitPos
;
1641 int headers_deleted
= 0;
1642 while ((e
= getEntry(&pos
))) {
1643 Http::HdrType id
= e
->id
;
1644 if (Http::HeaderLookupTable
.lookup(id
).hopbyhop
) {
1645 delAt(pos
, headers_deleted
);
1652 HttpHeader::removeConnectionHeaderEntries()
1654 if (has(Http::HdrType::CONNECTION
)) {
1655 /* anything that matches Connection list member will be deleted */
1656 String strConnection
;
1658 (void) getList(Http::HdrType::CONNECTION
, &strConnection
);
1659 const HttpHeaderEntry
*e
;
1660 HttpHeaderPos pos
= HttpHeaderInitPos
;
1662 * think: on-average-best nesting of the two loops (hdrEntry
1663 * and strListItem) @?@
1666 * maybe we should delete standard stuff ("keep-alive","close")
1667 * from strConnection first?
1670 int headers_deleted
= 0;
1671 while ((e
= getEntry(&pos
))) {
1672 if (strListIsMember(&strConnection
, e
->name
.termedBuf(), ','))
1673 delAt(pos
, headers_deleted
);
1675 if (headers_deleted
)