src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2019 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/CharacterSet.h"
  13 #include "base/EnumIterator.h"
  14 #include "base64.h"
  15 #include "globals.h"
  16 #include "http/ContentLengthInterpreter.h"
  17 #include "HttpHdrCc.h"
  18 #include "HttpHdrContRange.h"
  19 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
  20 #include "HttpHeader.h"
  21 #include "HttpHeaderFieldInfo.h"
  22 #include "HttpHeaderStat.h"
  23 #include "HttpHeaderTools.h"
  24 #include "MemBuf.h"
  25 #include "mgr/Registration.h"
  26 #include "mime_header.h"
  27 #include "profiler/Profiler.h"
  28 #include "rfc1123.h"
  29 #include "sbuf/StringConvert.h"
  30 #include "SquidConfig.h"
  31 #include "StatHist.h"
  32 #include "Store.h"
  33 #include "StrList.h"
  34 #include "TimeOrTag.h"
  35 #include "util.h"
  36
  37 #include <algorithm>
  38
  39 /* XXX: the whole set of API managing the entries vector should be rethought
  40  *      after the parse4r-ng effort is complete.
  41  */
  42
  43 /*
  44  * On naming conventions:
  45  *
  46  * HTTP/1.1 defines message-header as
  47  *
  48  * message-header = field-name ":" [ field-value ] CRLF
  49  * field-name     = token
  50  * field-value    = *( field-content | LWS )
  51  *
  52  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  53  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  54  *
  55  * HttpHeader is an object that represents all message-headers in a message.
  56  * HttpHeader does not manage start-line.
  57  *
  58  * HttpHeader is implemented as a collection of header "entries".
  59  * An entry is a (field_id, field_name, field_value) triplet.
  60  */
  61
  62 /*
  63  * local constants and vars
  64  */
  65
  66 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
  67 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
  68
  69 /* request-only headers. Used for cachemgr */
  70 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
  71
  72 /* reply-only headers. Used for cachemgr */
  73 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
  74
  75 /* header accounting */
  76 // NP: keep in sync with enum http_hdr_owner_type
  77 static HttpHeaderStat HttpHeaderStats[] = {
  78     HttpHeaderStat(/*hoNone*/ "all", NULL),
  79 #if USE_HTCP
  80     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
  81 #endif
  82     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
  83     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
  84 #if USE_OPENSSL
  85     /* hoErrorDetail */
  86 #endif
  87     /* hoEnd */
  88 };
  89 static int HttpHeaderStatCount = countof(HttpHeaderStats);
  90
  91 static int HeaderEntryParsedCount = 0;
  92
  93 /*
  94  * forward declarations and local routines
  95  */
  96
  97 class StoreEntry;
  98
  99 // update parse statistics for header id; if error is true also account
 100 // for errors and write to debug log what happened
 101 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
 102 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 103 /** store report about current header usage and other stats */
 104 static void httpHeaderStoreReport(StoreEntry * e);
 105
 106 /*
 107  * Module initialization routines
 108  */
 109
 110 static void
 111 httpHeaderRegisterWithCacheManager(void)
 112 {
 113     Mgr::RegisterAction("http_headers",
 114                         "HTTP Header Statistics",
 115                         httpHeaderStoreReport, 0, 1);
 116 }
 117
 118 void
 119 httpHeaderInitModule(void)
 120 {
 121     /* check that we have enough space for masks */
 122     assert(8 * sizeof(HttpHeaderMask) >= Http::HdrType::enumEnd_);
 123
 124     // masks are needed for stats page still
 125     for (auto h : WholeEnum<Http::HdrType>()) {
 126         if (Http::HeaderLookupTable.lookup(h).request)
 127             CBIT_SET(RequestHeadersMask,h);
 128         if (Http::HeaderLookupTable.lookup(h).reply)
 129             CBIT_SET(ReplyHeadersMask,h);
 130     }
 131
 132     /* header stats initialized by class constructor */
 133     assert(HttpHeaderStatCount == hoReply + 1);
 134
 135     /* init dependent modules */
 136     httpHdrCcInitModule();
 137     httpHdrScInitModule();
 138
 139     httpHeaderRegisterWithCacheManager();
 140 }
 141
 142 /*
 143  * HttpHeader Implementation
 144  */
 145
 146 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
 147 {
 148     entries.reserve(32);
 149     httpHeaderMaskInit(&mask, 0);
 150 }
 151
 152 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
 153 {
 154     assert(anOwner > hoNone && anOwner < hoEnd);
 155     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 156     entries.reserve(32);
 157     httpHeaderMaskInit(&mask, 0);
 158 }
 159
 160 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
 161 {
 162     entries.reserve(other.entries.capacity());
 163     httpHeaderMaskInit(&mask, 0);
 164     update(&other); // will update the mask as well
 165 }
 166
 167 HttpHeader::~HttpHeader()
 168 {
 169     clean();
 170 }
 171
 172 HttpHeader &
 173 HttpHeader::operator =(const HttpHeader &other)
 174 {
 175     if (this != &other) {
 176         // we do not really care, but the caller probably does
 177         assert(owner == other.owner);
 178         clean();
 179         update(&other); // will update the mask as well
 180         len = other.len;
 181         conflictingContentLength_ = other.conflictingContentLength_;
 182     }
 183     return *this;
 184 }
 185
 186 void
 187 HttpHeader::clean()
 188 {
 189
 190     assert(owner > hoNone && owner < hoEnd);
 191     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 192
 193     PROF_start(HttpHeaderClean);
 194
 195     if (owner <= hoReply) {
 196         /*
 197          * An unfortunate bug.  The entries array is initialized
 198          * such that count is set to zero.  httpHeaderClean() seems to
 199          * be called both when 'hdr' is created, and destroyed.  Thus,
 200          * we accumulate a large number of zero counts for 'hdr' before
 201          * it is ever used.  Can't think of a good way to fix it, except
 202          * adding a state variable that indicates whether or not 'hdr'
 203          * has been used.  As a hack, just never count zero-sized header
 204          * arrays.
 205          */
 206         if (!entries.empty())
 207             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 208
 209         ++ HttpHeaderStats[owner].destroyedCount;
 210
 211         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 212     } // if (owner <= hoReply)
 213
 214     for (HttpHeaderEntry *e : entries) {
 215         if (e == nullptr)
 216             continue;
 217         if (!Http::any_valid_header(e->id)) {
 218             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 219         } else {
 220             if (owner <= hoReply)
 221                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 222             delete e;
 223         }
 224     }
 225
 226     entries.clear();
 227     httpHeaderMaskInit(&mask, 0);
 228     len = 0;
 229     conflictingContentLength_ = false;
 230     PROF_stop(HttpHeaderClean);
 231 }
 232
 233 /* append entries (also see httpHeaderUpdate) */
 234 void
 235 HttpHeader::append(const HttpHeader * src)
 236 {
 237     assert(src);
 238     assert(src != this);
 239     debugs(55, 7, "appending hdr: " << this << " += " << src);
 240
 241     for (auto e : src->entries) {
 242         if (e)
 243             addEntry(e->clone());
 244     }
 245 }
 246
 247 /// check whether the fresh header has any new/changed updatable fields
 248 bool
 249 HttpHeader::needUpdate(HttpHeader const *fresh) const
 250 {
 251     for (const auto e: fresh->entries) {
 252         if (!e || skipUpdateHeader(e->id))
 253             continue;
 254         String value;
 255         if (!hasNamed(e->name, &value) ||
 256                 (value != fresh->getByName(e->name)))
 257             return true;
 258     }
 259     return false;
 260 }
 261
 262 void
 263 HttpHeader::updateWarnings()
 264 {
 265     int count = 0;
 266     HttpHeaderPos pos = HttpHeaderInitPos;
 267
 268     // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
 269     while (HttpHeaderEntry *e = getEntry(&pos)) {
 270         if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
 271             delAt(pos, count);
 272     }
 273 }
 274
 275 bool
 276 HttpHeader::skipUpdateHeader(const Http::HdrType id) const
 277 {
 278     // RFC 7234, section 4.3.4: use fields other from Warning for update
 279     return id == Http::HdrType::WARNING;
 280 }
 281
 282 bool
 283 HttpHeader::update(HttpHeader const *fresh)
 284 {
 285     assert(fresh);
 286     assert(this != fresh);
 287
 288     // Optimization: Finding whether a header field changed is expensive
 289     // and probably not worth it except for collapsed revalidation needs.
 290     if (Config.onoff.collapsed_forwarding && !needUpdate(fresh))
 291         return false;
 292
 293     updateWarnings();
 294
 295     const HttpHeaderEntry *e;
 296     HttpHeaderPos pos = HttpHeaderInitPos;
 297
 298     while ((e = fresh->getEntry(&pos))) {
 299         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 300
 301         if (skipUpdateHeader(e->id))
 302             continue;
 303
 304         if (e->id != Http::HdrType::OTHER)
 305             delById(e->id);
 306         else
 307             delByName(e->name);
 308     }
 309
 310     pos = HttpHeaderInitPos;
 311     while ((e = fresh->getEntry(&pos))) {
 312         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 313
 314         if (skipUpdateHeader(e->id))
 315             continue;
 316
 317         debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
 318
 319         addEntry(e->clone());
 320     }
 321     return true;
 322 }
 323
 324 bool
 325 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
 326 {
 327     /*
 328      * parse_start points to the first line of HTTP message *headers*,
 329      * not including the request or status lines
 330      */
 331     const size_t end = headersEnd(*parse_start, l);
 332
 333     if (end) {
 334         *blk_start = *parse_start;
 335         *blk_end = *parse_start + end - 1;
 336         assert(**blk_end == '\n');
 337         // Point blk_end to the first character after the last header field.
 338         // In other words, blk_end should point to the CR?LF header terminator.
 339         if (end > 1 && *(*blk_end - 1) == '\r')
 340             --(*blk_end);
 341         *parse_start += end;
 342     }
 343     return end;
 344 }
 345
 346 int
 347 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz, Http::ContentLengthInterpreter &clen)
 348 {
 349     const char *parse_start = buf;
 350     const char *blk_start, *blk_end;
 351     hdr_sz = 0;
 352
 353     if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
 354         // XXX: do not parse non-isolated headers even if the connection is closed.
 355         // Treat unterminated headers as "partial headers" framing errors.
 356         if (!atEnd)
 357             return 0;
 358         blk_start = parse_start;
 359         blk_end = blk_start + strlen(blk_start);
 360     }
 361
 362     if (parse(blk_start, blk_end - blk_start, clen)) {
 363         hdr_sz = parse_start - buf;
 364         return 1;
 365     }
 366     return -1;
 367 }
 368
 369 int
 370 HttpHeader::parse(const char *header_start, size_t hdrLen, Http::ContentLengthInterpreter &clen)
 371 {
 372     const char *field_ptr = header_start;
 373     const char *header_end = header_start + hdrLen; // XXX: remove
 374     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 375
 376     PROF_start(HttpHeaderParse);
 377
 378     assert(header_start && header_end);
 379     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 380     ++ HttpHeaderStats[owner].parsedCount;
 381
 382     char *nulpos;
 383     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 384         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 385                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 386         PROF_stop(HttpHeaderParse);
 387         clean();
 388         return 0;
 389     }
 390
 391     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 392      * continuation lines start with a (single) space or tab */
 393     while (field_ptr < header_end) {
 394         const char *field_start = field_ptr;
 395         const char *field_end;
 396
 397         do {
 398             const char *this_line = field_ptr;
 399             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 400
 401             if (!field_ptr) {
 402                 // missing <LF>
 403                 PROF_stop(HttpHeaderParse);
 404                 clean();
 405                 return 0;
 406             }
 407
 408             field_end = field_ptr;
 409
 410             ++field_ptr;    /* Move to next line */
 411
 412             if (field_end > this_line && field_end[-1] == '\r') {
 413                 --field_end;    /* Ignore CR LF */
 414
 415                 if (owner == hoRequest && field_end > this_line) {
 416                     bool cr_only = true;
 417                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 418                         if (*p != '\r')
 419                             cr_only = false;
 420                     }
 421                     if (cr_only) {
 422                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 423                                "header field to prevent request smuggling attacks: {" <<
 424                                getStringPrefix(header_start, hdrLen) << "}");
 425                         PROF_stop(HttpHeaderParse);
 426                         clean();
 427                         return 0;
 428                     }
 429                 }
 430             }
 431
 432             /* Barf on stray CR characters */
 433             if (memchr(this_line, '\r', field_end - this_line)) {
 434                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 435                        getStringPrefix(field_start, field_end-field_start) << "}");
 436
 437                 if (Config.onoff.relaxed_header_parser) {
 438                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 439
 440                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 441                         *p = ' ';
 442                         ++p;
 443                     }
 444                 } else {
 445                     PROF_stop(HttpHeaderParse);
 446                     clean();
 447                     return 0;
 448                 }
 449             }
 450
 451             if (this_line + 1 == field_end && this_line > field_start) {
 452                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 453                        getStringPrefix(header_start, hdrLen) << "}");
 454                 PROF_stop(HttpHeaderParse);
 455                 clean();
 456                 return 0;
 457             }
 458         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 459
 460         if (field_start == field_end) {
 461             if (field_ptr < header_end) {
 462                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 463                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 464                 PROF_stop(HttpHeaderParse);
 465                 clean();
 466                 return 0;
 467             }
 468
 469             break;      /* terminating blank line */
 470         }
 471
 472         HttpHeaderEntry *e;
 473         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 474             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 475                    getStringPrefix(field_start, field_end-field_start) << "}");
 476             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 477
 478             if (Config.onoff.relaxed_header_parser)
 479                 continue;
 480
 481             PROF_stop(HttpHeaderParse);
 482             clean();
 483             return 0;
 484         }
 485
 486         if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
 487             delete e;
 488
 489             if (Config.onoff.relaxed_header_parser)
 490                 continue; // clen has printed any necessary warnings
 491
 492             PROF_stop(HttpHeaderParse);
 493             clean();
 494             return 0;
 495         }
 496
 497         /* AYJ 2017-05-23: I suspect we need to change this whitespace check to conform to the
 498            updated WSP character set in RFC 7230/7231. For now I left it as the
 499            characters in w_space which the previous code was using. */
 500         static CharacterSet wsp = (CharacterSet::WSP + CharacterSet::CR + CharacterSet::LF);
 501         if (e->id == Http::HdrType::OTHER && e->name.findFirstOf(wsp) != SBuf::npos) {
 502             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 503                    getStringPrefix(field_start, field_end-field_start) << "}");
 504
 505             if (!Config.onoff.relaxed_header_parser) {
 506                 delete e;
 507                 PROF_stop(HttpHeaderParse);
 508                 clean();
 509                 return 0;
 510             }
 511         }
 512
 513         addEntry(e);
 514     }
 515
 516     if (clen.headerWideProblem) {
 517         debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
 518                " Content-Length field values in" <<
 519                Raw("header", header_start, hdrLen));
 520     }
 521
 522     if (clen.prohibitedAndIgnored()) {
 523         // RFC 7230 section 3.3.2: A server MUST NOT send a Content-Length
 524         // header field in any response with a status code of 1xx (Informational)
 525         // or 204 (No Content). And RFC 7230 3.3.3#1 tells recipients to ignore
 526         // such Content-Lengths.
 527         if (delById(Http::HdrType::CONTENT_LENGTH))
 528             debugs(55, 3, "Content-Length is " << clen.prohibitedAndIgnored());
 529     } else if (chunked()) {
 530         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 531         // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
 532         delById(Http::HdrType::CONTENT_LENGTH);
 533         // and clen state becomes irrelevant
 534     } else if (clen.sawBad) {
 535         // ensure our callers do not accidentally see bad Content-Length values
 536         delById(Http::HdrType::CONTENT_LENGTH);
 537         conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
 538     } else if (clen.needsSanitizing) {
 539         // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
 540         // ensure our callers see a clean Content-Length value or none at all
 541         delById(Http::HdrType::CONTENT_LENGTH);
 542         if (clen.sawGood) {
 543             putInt64(Http::HdrType::CONTENT_LENGTH, clen.value);
 544             debugs(55, 5, "sanitized Content-Length to be " << clen.value);
 545         }
 546     }
 547
 548     PROF_stop(HttpHeaderParse);
 549     return 1;           /* even if no fields where found, it is a valid header */
 550 }
 551
 552 /* packs all the entries using supplied packer */
 553 void
 554 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 555 {
 556     HttpHeaderPos pos = HttpHeaderInitPos;
 557     const HttpHeaderEntry *e;
 558     assert(p);
 559     debugs(55, 7, this << " into " << p <<
 560            (mask_sensitive_info ? " while masking" : ""));
 561     /* pack all entries one by one */
 562     while ((e = getEntry(&pos))) {
 563         if (!mask_sensitive_info) {
 564             e->packInto(p);
 565             continue;
 566         }
 567
 568         bool maskThisEntry = false;
 569         switch (e->id) {
 570         case Http::HdrType::AUTHORIZATION:
 571         case Http::HdrType::PROXY_AUTHORIZATION:
 572             maskThisEntry = true;
 573             break;
 574
 575         case Http::HdrType::FTP_ARGUMENTS:
 576             if (const HttpHeaderEntry *cmd = findEntry(Http::HdrType::FTP_COMMAND))
 577                 maskThisEntry = (cmd->value == "PASS");
 578             break;
 579
 580         default:
 581             break;
 582         }
 583         if (maskThisEntry) {
 584             p->append(e->name.rawContent(), e->name.length());
 585             p->append(": ** NOT DISPLAYED **\r\n", 23);
 586         } else {
 587             e->packInto(p);
 588         }
 589
 590     }
 591     /* Pack in the "special" entries */
 592
 593     /* Cache-Control */
 594 }
 595
 596 /* returns next valid entry */
 597 HttpHeaderEntry *
 598 HttpHeader::getEntry(HttpHeaderPos * pos) const
 599 {
 600     assert(pos);
 601     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 602
 603     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 604         if (entries[*pos])
 605             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 606     }
 607
 608     return NULL;
 609 }
 610
 611 /*
 612  * returns a pointer to a specified entry if any
 613  * note that we return one entry so it does not make much sense to ask for
 614  * "list" headers
 615  */
 616 HttpHeaderEntry *
 617 HttpHeader::findEntry(Http::HdrType id) const
 618 {
 619     assert(any_registered_header(id));
 620     assert(!Http::HeaderLookupTable.lookup(id).list);
 621
 622     /* check mask first */
 623
 624     if (!CBIT_TEST(mask, id))
 625         return NULL;
 626
 627     /* looks like we must have it, do linear search */
 628     for (auto e : entries) {
 629         if (e && e->id == id)
 630             return e;
 631     }
 632
 633     /* hm.. we thought it was there, but it was not found */
 634     assert(false);
 635     return nullptr;        /* not reached */
 636 }
 637
 638 /*
 639  * same as httpHeaderFindEntry
 640  */
 641 HttpHeaderEntry *
 642 HttpHeader::findLastEntry(Http::HdrType id) const
 643 {
 644     assert(any_registered_header(id));
 645     assert(!Http::HeaderLookupTable.lookup(id).list);
 646
 647     /* check mask first */
 648     if (!CBIT_TEST(mask, id))
 649         return NULL;
 650
 651     for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
 652         if (*e && (*e)->id == id)
 653             return *e;
 654     }
 655
 656     /* hm.. we thought it was there, but it was not found */
 657     assert(false);
 658     return nullptr; /* not reached */
 659 }
 660
 661 int
 662 HttpHeader::delByName(const SBuf &name)
 663 {
 664     int count = 0;
 665     HttpHeaderPos pos = HttpHeaderInitPos;
 666     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 667     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 668
 669     while (const HttpHeaderEntry *e = getEntry(&pos)) {
 670         if (!e->name.caseCmp(name))
 671             delAt(pos, count);
 672         else
 673             CBIT_SET(mask, e->id);
 674     }
 675
 676     return count;
 677 }
 678
 679 /* deletes all entries with a given id, returns the #entries deleted */
 680 int
 681 HttpHeader::delById(Http::HdrType id)
 682 {
 683     debugs(55, 8, this << " del-by-id " << id);
 684     assert(any_registered_header(id));
 685
 686     if (!CBIT_TEST(mask, id))
 687         return 0;
 688
 689     int count = 0;
 690
 691     HttpHeaderPos pos = HttpHeaderInitPos;
 692     while (HttpHeaderEntry *e = getEntry(&pos)) {
 693         if (e->id == id)
 694             delAt(pos, count); // deletes e
 695     }
 696
 697     CBIT_CLR(mask, id);
 698     assert(count);
 699     return count;
 700 }
 701
 702 /*
 703  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 704  * possible to iterate(search) and delete fields at the same time
 705  * NOTE: Does not update the header mask. Caller must follow up with
 706  * a call to refreshMask() if headers_deleted was incremented.
 707  */
 708 void
 709 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 710 {
 711     HttpHeaderEntry *e;
 712     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 713     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 714     entries[pos] = NULL;
 715     /* decrement header length, allow for ": " and crlf */
 716     len -= e->name.length() + 2 + e->value.size() + 2;
 717     assert(len >= 0);
 718     delete e;
 719     ++headers_deleted;
 720 }
 721
 722 /*
 723  * Compacts the header storage
 724  */
 725 void
 726 HttpHeader::compact()
 727 {
 728     // TODO: optimize removal, or possibly make it so that's not needed.
 729     entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
 730                    entries.end());
 731 }
 732
 733 /*
 734  * Refreshes the header mask. Required after delAt() calls.
 735  */
 736 void
 737 HttpHeader::refreshMask()
 738 {
 739     httpHeaderMaskInit(&mask, 0);
 740     debugs(55, 7, "refreshing the mask in hdr " << this);
 741     for (auto e : entries) {
 742         if (e)
 743             CBIT_SET(mask, e->id);
 744     }
 745 }
 746
 747 /* appends an entry;
 748  * does not call e->clone() so one should not reuse "*e"
 749  */
 750 void
 751 HttpHeader::addEntry(HttpHeaderEntry * e)
 752 {
 753     assert(e);
 754     assert(any_HdrType_enum_value(e->id));
 755     assert(e->name.length());
 756
 757     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 758
 759     if (e->id != Http::HdrType::BAD_HDR) {
 760         if (CBIT_TEST(mask, e->id)) {
 761             ++ headerStatsTable[e->id].repCount;
 762         } else {
 763             CBIT_SET(mask, e->id);
 764         }
 765     }
 766
 767     entries.push_back(e);
 768
 769     /* increment header length, allow for ": " and crlf */
 770     len += e->name.length() + 2 + e->value.size() + 2;
 771 }
 772
 773 /* inserts an entry;
 774  * does not call e->clone() so one should not reuse "*e"
 775  */
 776 void
 777 HttpHeader::insertEntry(HttpHeaderEntry * e)
 778 {
 779     assert(e);
 780     assert(any_valid_header(e->id));
 781
 782     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 783
 784     // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
 785     if (CBIT_TEST(mask, e->id)) {
 786         ++ headerStatsTable[e->id].repCount;
 787     } else {
 788         CBIT_SET(mask, e->id);
 789     }
 790
 791     entries.insert(entries.begin(),e);
 792
 793     /* increment header length, allow for ": " and crlf */
 794     len += e->name.length() + 2 + e->value.size() + 2;
 795 }
 796
 797 bool
 798 HttpHeader::getList(Http::HdrType id, String *s) const
 799 {
 800     debugs(55, 9, this << " joining for id " << id);
 801     /* only fields from ListHeaders array can be "listed" */
 802     assert(Http::HeaderLookupTable.lookup(id).list);
 803
 804     if (!CBIT_TEST(mask, id))
 805         return false;
 806
 807     for (auto e: entries) {
 808         if (e && e->id == id)
 809             strListAdd(s, e->value.termedBuf(), ',');
 810     }
 811
 812     /*
 813      * note: we might get an empty (size==0) string if there was an "empty"
 814      * header. This results in an empty length String, which may have a NULL
 815      * buffer.
 816      */
 817     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 818     if (!s->size())
 819         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 820     else
 821         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 822
 823     return true;
 824 }
 825
 826 /* return a list of entries with the same id separated by ',' and ws */
 827 String
 828 HttpHeader::getList(Http::HdrType id) const
 829 {
 830     HttpHeaderEntry *e;
 831     HttpHeaderPos pos = HttpHeaderInitPos;
 832     debugs(55, 9, this << "joining for id " << id);
 833     /* only fields from ListHeaders array can be "listed" */
 834     assert(Http::HeaderLookupTable.lookup(id).list);
 835
 836     if (!CBIT_TEST(mask, id))
 837         return String();
 838
 839     String s;
 840
 841     while ((e = getEntry(&pos))) {
 842         if (e->id == id)
 843             strListAdd(&s, e->value.termedBuf(), ',');
 844     }
 845
 846     /*
 847      * note: we might get an empty (size==0) string if there was an "empty"
 848      * header. This results in an empty length String, which may have a NULL
 849      * buffer.
 850      */
 851     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 852     if (!s.size())
 853         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 854     else
 855         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 856
 857     return s;
 858 }
 859
 860 /* return a string or list of entries with the same id separated by ',' and ws */
 861 String
 862 HttpHeader::getStrOrList(Http::HdrType id) const
 863 {
 864     HttpHeaderEntry *e;
 865
 866     if (Http::HeaderLookupTable.lookup(id).list)
 867         return getList(id);
 868
 869     if ((e = findEntry(id)))
 870         return e->value;
 871
 872     return String();
 873 }
 874
 875 /*
 876  * Returns the value of the specified header and/or an undefined String.
 877  */
 878 String
 879 HttpHeader::getByName(const char *name) const
 880 {
 881     String result;
 882     // ignore presence: return undefined string if an empty header is present
 883     (void)hasNamed(name, strlen(name), &result);
 884     return result;
 885 }
 886
 887 String
 888 HttpHeader::getByName(const SBuf &name) const
 889 {
 890     String result;
 891     // ignore presence: return undefined string if an empty header is present
 892     (void)hasNamed(name, &result);
 893     return result;
 894 }
 895
 896 String
 897 HttpHeader::getById(Http::HdrType id) const
 898 {
 899     String result;
 900     (void)getByIdIfPresent(id, &result);
 901     return result;
 902 }
 903
 904 bool
 905 HttpHeader::hasNamed(const SBuf &s, String *result) const
 906 {
 907     return hasNamed(s.rawContent(), s.length(), result);
 908 }
 909
 910 bool
 911 HttpHeader::getByIdIfPresent(Http::HdrType id, String *result) const
 912 {
 913     if (id == Http::HdrType::BAD_HDR)
 914         return false;
 915     if (!has(id))
 916         return false;
 917     if (result)
 918         *result = getStrOrList(id);
 919     return true;
 920 }
 921
 922 bool
 923 HttpHeader::hasNamed(const char *name, unsigned int namelen, String *result) const
 924 {
 925     Http::HdrType id;
 926     HttpHeaderPos pos = HttpHeaderInitPos;
 927     HttpHeaderEntry *e;
 928
 929     assert(name);
 930
 931     /* First try the quick path */
 932     id = Http::HeaderLookupTable.lookup(name,namelen).id;
 933
 934     if (id != Http::HdrType::BAD_HDR) {
 935         if (getByIdIfPresent(id, result))
 936             return true;
 937     }
 938
 939     /* Sorry, an unknown header name. Do linear search */
 940     bool found = false;
 941     while ((e = getEntry(&pos))) {
 942         if (e->id == Http::HdrType::OTHER && e->name.length() == namelen && e->name.caseCmp(name, namelen) == 0) {
 943             found = true;
 944             if (!result)
 945                 break;
 946             strListAdd(result, e->value.termedBuf(), ',');
 947         }
 948     }
 949
 950     return found;
 951 }
 952
 953 /*
 954  * Returns a the value of the specified list member, if any.
 955  */
 956 SBuf
 957 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
 958 {
 959     assert(name);
 960     const auto header = getByName(name);
 961     return ::getListMember(header, member, separator);
 962 }
 963
 964 /*
 965  * returns a the value of the specified list member, if any.
 966  */
 967 SBuf
 968 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
 969 {
 970     assert(any_registered_header(id));
 971     const auto header = getStrOrList(id);
 972     return ::getListMember(header, member, separator);
 973 }
 974
 975 /* test if a field is present */
 976 int
 977 HttpHeader::has(Http::HdrType id) const
 978 {
 979     assert(any_registered_header(id));
 980     debugs(55, 9, this << " lookup for " << id);
 981     return CBIT_TEST(mask, id);
 982 }
 983
 984 void
 985 HttpHeader::addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from)
 986 {
 987     // TODO: do not add Via header for messages where Squid itself
 988     // generated the message (i.e., Downloader or ESI) there should be no Via header added at all.
 989
 990     if (Config.onoff.via) {
 991         SBuf buf;
 992         // RFC 7230 section 5.7.1.: protocol-name is omitted when
 993         // the received protocol is HTTP.
 994         if (ver.protocol > AnyP::PROTO_NONE && ver.protocol < AnyP::PROTO_UNKNOWN &&
 995                 ver.protocol != AnyP::PROTO_HTTP && ver.protocol != AnyP::PROTO_HTTPS)
 996             buf.appendf("%s/", AnyP::ProtocolType_str[ver.protocol]);
 997         buf.appendf("%d.%d %s", ver.major, ver.minor, ThisCache);
 998         const HttpHeader *hdr = from ? from : this;
 999         SBuf strVia = StringToSBuf(hdr->getList(Http::HdrType::VIA));
1000         if (!strVia.isEmpty())
1001             strVia.append(", ", 2);
1002         strVia.append(buf);
1003         // XXX: putStr() still suffers from String size limits
1004         Must(strVia.length() < String::SizeMaxXXX());
1005         delById(Http::HdrType::VIA);
1006         putStr(Http::HdrType::VIA, strVia.c_str());
1007     }
1008 }
1009
1010 void
1011 HttpHeader::putInt(Http::HdrType id, int number)
1012 {
1013     assert(any_registered_header(id));
1014     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1015     assert(number >= 0);
1016     addEntry(new HttpHeaderEntry(id, SBuf(), xitoa(number)));
1017 }
1018
1019 void
1020 HttpHeader::putInt64(Http::HdrType id, int64_t number)
1021 {
1022     assert(any_registered_header(id));
1023     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1024     assert(number >= 0);
1025     addEntry(new HttpHeaderEntry(id, SBuf(), xint64toa(number)));
1026 }
1027
1028 void
1029 HttpHeader::putTime(Http::HdrType id, time_t htime)
1030 {
1031     assert(any_registered_header(id));
1032     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1033     assert(htime >= 0);
1034     addEntry(new HttpHeaderEntry(id, SBuf(), mkrfc1123(htime)));
1035 }
1036
1037 void
1038 HttpHeader::putStr(Http::HdrType id, const char *str)
1039 {
1040     assert(any_registered_header(id));
1041     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1042     assert(str);
1043     addEntry(new HttpHeaderEntry(id, SBuf(), str));
1044 }
1045
1046 void
1047 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1048 {
1049     assert(auth_scheme && realm);
1050     httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1051 }
1052
1053 void
1054 HttpHeader::putCc(const HttpHdrCc * cc)
1055 {
1056     assert(cc);
1057     /* remove old directives if any */
1058     delById(Http::HdrType::CACHE_CONTROL);
1059     /* pack into mb */
1060     MemBuf mb;
1061     mb.init();
1062     cc->packInto(&mb);
1063     /* put */
1064     addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL, SBuf(), mb.buf));
1065     /* cleanup */
1066     mb.clean();
1067 }
1068
1069 void
1070 HttpHeader::putContRange(const HttpHdrContRange * cr)
1071 {
1072     assert(cr);
1073     /* remove old directives if any */
1074     delById(Http::HdrType::CONTENT_RANGE);
1075     /* pack into mb */
1076     MemBuf mb;
1077     mb.init();
1078     httpHdrContRangePackInto(cr, &mb);
1079     /* put */
1080     addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE, SBuf(), mb.buf));
1081     /* cleanup */
1082     mb.clean();
1083 }
1084
1085 void
1086 HttpHeader::putRange(const HttpHdrRange * range)
1087 {
1088     assert(range);
1089     /* remove old directives if any */
1090     delById(Http::HdrType::RANGE);
1091     /* pack into mb */
1092     MemBuf mb;
1093     mb.init();
1094     range->packInto(&mb);
1095     /* put */
1096     addEntry(new HttpHeaderEntry(Http::HdrType::RANGE, SBuf(), mb.buf));
1097     /* cleanup */
1098     mb.clean();
1099 }
1100
1101 void
1102 HttpHeader::putSc(HttpHdrSc *sc)
1103 {
1104     assert(sc);
1105     /* remove old directives if any */
1106     delById(Http::HdrType::SURROGATE_CONTROL);
1107     /* pack into mb */
1108     MemBuf mb;
1109     mb.init();
1110     sc->packInto(&mb);
1111     /* put */
1112     addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL, SBuf(), mb.buf));
1113     /* cleanup */
1114     mb.clean();
1115 }
1116
1117 void
1118 HttpHeader::putWarning(const int code, const char *const text)
1119 {
1120     char buf[512];
1121     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1122     putStr(Http::HdrType::WARNING, buf);
1123 }
1124
1125 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1126 void
1127 HttpHeader::putExt(const char *name, const char *value)
1128 {
1129     assert(name && value);
1130     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1131     addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, SBuf(name), value));
1132 }
1133
1134 int
1135 HttpHeader::getInt(Http::HdrType id) const
1136 {
1137     assert(any_registered_header(id));
1138     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1139     HttpHeaderEntry *e;
1140
1141     if ((e = findEntry(id)))
1142         return e->getInt();
1143
1144     return -1;
1145 }
1146
1147 int64_t
1148 HttpHeader::getInt64(Http::HdrType id) const
1149 {
1150     assert(any_registered_header(id));
1151     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1152     HttpHeaderEntry *e;
1153
1154     if ((e = findEntry(id)))
1155         return e->getInt64();
1156
1157     return -1;
1158 }
1159
1160 time_t
1161 HttpHeader::getTime(Http::HdrType id) const
1162 {
1163     HttpHeaderEntry *e;
1164     time_t value = -1;
1165     assert(any_registered_header(id));
1166     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1167
1168     if ((e = findEntry(id))) {
1169         value = parse_rfc1123(e->value.termedBuf());
1170         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1171     }
1172
1173     return value;
1174 }
1175
1176 /* sync with httpHeaderGetLastStr */
1177 const char *
1178 HttpHeader::getStr(Http::HdrType id) const
1179 {
1180     HttpHeaderEntry *e;
1181     assert(any_registered_header(id));
1182     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1183
1184     if ((e = findEntry(id))) {
1185         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1186         return e->value.termedBuf();
1187     }
1188
1189     return NULL;
1190 }
1191
1192 /* unusual */
1193 const char *
1194 HttpHeader::getLastStr(Http::HdrType id) const
1195 {
1196     HttpHeaderEntry *e;
1197     assert(any_registered_header(id));
1198     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1199
1200     if ((e = findLastEntry(id))) {
1201         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1202         return e->value.termedBuf();
1203     }
1204
1205     return NULL;
1206 }
1207
1208 HttpHdrCc *
1209 HttpHeader::getCc() const
1210 {
1211     if (!CBIT_TEST(mask, Http::HdrType::CACHE_CONTROL))
1212         return NULL;
1213     PROF_start(HttpHeader_getCc);
1214
1215     String s;
1216     getList(Http::HdrType::CACHE_CONTROL, &s);
1217
1218     HttpHdrCc *cc=new HttpHdrCc();
1219
1220     if (!cc->parse(s)) {
1221         delete cc;
1222         cc = NULL;
1223     }
1224
1225     ++ HttpHeaderStats[owner].ccParsedCount;
1226
1227     if (cc)
1228         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1229
1230     httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL, s, !cc);
1231
1232     PROF_stop(HttpHeader_getCc);
1233
1234     return cc;
1235 }
1236
1237 HttpHdrRange *
1238 HttpHeader::getRange() const
1239 {
1240     HttpHdrRange *r = NULL;
1241     HttpHeaderEntry *e;
1242     /* some clients will send "Request-Range" _and_ *matching* "Range"
1243      * who knows, some clients might send Request-Range only;
1244      * this "if" should work correctly in both cases;
1245      * hopefully no clients send mismatched headers! */
1246
1247     if ((e = findEntry(Http::HdrType::RANGE)) ||
1248             (e = findEntry(Http::HdrType::REQUEST_RANGE))) {
1249         r = HttpHdrRange::ParseCreate(&e->value);
1250         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1251     }
1252
1253     return r;
1254 }
1255
1256 HttpHdrSc *
1257 HttpHeader::getSc() const
1258 {
1259     if (!CBIT_TEST(mask, Http::HdrType::SURROGATE_CONTROL))
1260         return NULL;
1261
1262     String s;
1263
1264     (void) getList(Http::HdrType::SURROGATE_CONTROL, &s);
1265
1266     HttpHdrSc *sc = httpHdrScParseCreate(s);
1267
1268     ++ HttpHeaderStats[owner].ccParsedCount;
1269
1270     if (sc)
1271         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1272
1273     httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL, s, !sc);
1274
1275     return sc;
1276 }
1277
1278 HttpHdrContRange *
1279 HttpHeader::getContRange() const
1280 {
1281     HttpHdrContRange *cr = NULL;
1282     HttpHeaderEntry *e;
1283
1284     if ((e = findEntry(Http::HdrType::CONTENT_RANGE))) {
1285         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1286         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1287     }
1288
1289     return cr;
1290 }
1291
1292 const char *
1293 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1294 {
1295     const char *field;
1296     int l;
1297     assert(auth_scheme);
1298     field = getStr(id);
1299
1300     if (!field)         /* no authorization field */
1301         return NULL;
1302
1303     l = strlen(auth_scheme);
1304
1305     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1306         return NULL;
1307
1308     field += l;
1309
1310     if (!xisspace(*field))  /* wrong scheme */
1311         return NULL;
1312
1313     /* skip white space */
1314     for (; field && xisspace(*field); ++field);
1315
1316     if (!*field)        /* no authorization cookie */
1317         return NULL;
1318
1319     static char decodedAuthToken[8192];
1320     struct base64_decode_ctx ctx;
1321     base64_decode_init(&ctx);
1322     size_t decodedLen = 0;
1323     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), field) ||
1324             !base64_decode_final(&ctx)) {
1325         return NULL;
1326     }
1327     decodedAuthToken[decodedLen] = '\0';
1328     return decodedAuthToken;
1329 }
1330
1331 ETag
1332 HttpHeader::getETag(Http::HdrType id) const
1333 {
1334     ETag etag = {NULL, -1};
1335     HttpHeaderEntry *e;
1336     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag);     /* must be of an appropriate type */
1337
1338     if ((e = findEntry(id)))
1339         etagParseInit(&etag, e->value.termedBuf());
1340
1341     return etag;
1342 }
1343
1344 TimeOrTag
1345 HttpHeader::getTimeOrTag(Http::HdrType id) const
1346 {
1347     TimeOrTag tot;
1348     HttpHeaderEntry *e;
1349     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag);    /* must be of an appropriate type */
1350     memset(&tot, 0, sizeof(tot));
1351
1352     if ((e = findEntry(id))) {
1353         const char *str = e->value.termedBuf();
1354         /* try as an ETag */
1355
1356         if (etagParseInit(&tot.tag, str)) {
1357             tot.valid = tot.tag.str != NULL;
1358             tot.time = -1;
1359         } else {
1360             /* or maybe it is time? */
1361             tot.time = parse_rfc1123(str);
1362             tot.valid = tot.time >= 0;
1363             tot.tag.str = NULL;
1364         }
1365     }
1366
1367     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1368     return tot;
1369 }
1370
1371 /*
1372  * HttpHeaderEntry
1373  */
1374
1375 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const SBuf &aName, const char *aValue)
1376 {
1377     assert(any_HdrType_enum_value(anId));
1378     id = anId;
1379
1380     if (id != Http::HdrType::OTHER)
1381         name = Http::HeaderLookupTable.lookup(id).name;
1382     else
1383         name = aName;
1384
1385     value = aValue;
1386
1387     if (id != Http::HdrType::BAD_HDR)
1388         ++ headerStatsTable[id].aliveCount;
1389
1390     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1391 }
1392
1393 HttpHeaderEntry::~HttpHeaderEntry()
1394 {
1395     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1396
1397     if (id != Http::HdrType::BAD_HDR) {
1398         assert(headerStatsTable[id].aliveCount);
1399         -- headerStatsTable[id].aliveCount;
1400         id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1401     }
1402
1403 }
1404
1405 /* parses and inits header entry, returns true/false */
1406 HttpHeaderEntry *
1407 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1408 {
1409     /* note: name_start == field_start */
1410     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1411     int name_len = name_end ? name_end - field_start :0;
1412     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1413     /* note: value_end == field_end */
1414
1415     ++ HeaderEntryParsedCount;
1416
1417     /* do we have a valid field name within this field? */
1418
1419     if (!name_len || name_end > field_end)
1420         return NULL;
1421
1422     if (name_len > 65534) {
1423         /* String must be LESS THAN 64K and it adds a terminating NULL */
1424         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1425         return NULL;
1426     }
1427
1428     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1429         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1430                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1431
1432         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1433             --name_len;
1434
1435         if (!name_len)
1436             return NULL;
1437     }
1438
1439     /* now we know we can parse it */
1440
1441     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1442
1443     /* is it a "known" field? */
1444     Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1445     debugs(55, 9, "got hdr-id=" << id);
1446
1447     SBuf theName;
1448
1449     String value;
1450
1451     if (id == Http::HdrType::BAD_HDR)
1452         id = Http::HdrType::OTHER;
1453
1454     /* set field name */
1455     if (id == Http::HdrType::OTHER)
1456         theName.append(field_start, name_len);
1457     else
1458         theName = Http::HeaderLookupTable.lookup(id).name;
1459
1460     /* trim field value */
1461     while (value_start < field_end && xisspace(*value_start))
1462         ++value_start;
1463
1464     while (value_start < field_end && xisspace(field_end[-1]))
1465         --field_end;
1466
1467     if (field_end - value_start > 65534) {
1468         /* String must be LESS THAN 64K and it adds a terminating NULL */
1469         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << theName << "' header of " << (field_end - value_start) << " bytes");
1470         return NULL;
1471     }
1472
1473     /* set field value */
1474     value.limitInit(value_start, field_end - value_start);
1475
1476     if (id != Http::HdrType::BAD_HDR)
1477         ++ headerStatsTable[id].seenCount;
1478
1479     debugs(55, 9, "parsed HttpHeaderEntry: '" << theName << ": " << value << "'");
1480
1481     return new HttpHeaderEntry(id, theName, value.termedBuf());
1482 }
1483
1484 HttpHeaderEntry *
1485 HttpHeaderEntry::clone() const
1486 {
1487     return new HttpHeaderEntry(id, name, value.termedBuf());
1488 }
1489
1490 void
1491 HttpHeaderEntry::packInto(Packable * p) const
1492 {
1493     assert(p);
1494     p->append(name.rawContent(), name.length());
1495     p->append(": ", 2);
1496     p->append(value.rawBuf(), value.size());
1497     p->append("\r\n", 2);
1498 }
1499
1500 int
1501 HttpHeaderEntry::getInt() const
1502 {
1503     int val = -1;
1504     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1505     httpHeaderNoteParsedEntry(id, value, ok == 0);
1506     /* XXX: Should we check ok - ie
1507      * return ok ? -1 : value;
1508      */
1509     return val;
1510 }
1511
1512 int64_t
1513 HttpHeaderEntry::getInt64() const
1514 {
1515     int64_t val = -1;
1516     const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1517     httpHeaderNoteParsedEntry(id, value, !ok);
1518     return val; // remains -1 if !ok (XXX: bad method API)
1519 }
1520
1521 static void
1522 httpHeaderNoteParsedEntry(Http::HdrType id, String const &context, bool error)
1523 {
1524     if (id != Http::HdrType::BAD_HDR)
1525         ++ headerStatsTable[id].parsCount;
1526
1527     if (error) {
1528         if (id != Http::HdrType::BAD_HDR)
1529             ++ headerStatsTable[id].errCount;
1530         debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1531     }
1532 }
1533
1534 /*
1535  * Reports
1536  */
1537
1538 /* tmp variable used to pass stat info to dumpers */
1539 extern const HttpHeaderStat *dump_stat;     /* argh! */
1540 const HttpHeaderStat *dump_stat = NULL;
1541
1542 void
1543 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1544 {
1545     const int id = static_cast<int>(val);
1546     const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1547     const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1548     int visible = count > 0;
1549     /* for entries with zero count, list only those that belong to current type of message */
1550
1551     if (!visible && valid_id && dump_stat->owner_mask)
1552         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1553
1554     if (visible)
1555         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1556                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1557 }
1558
1559 static void
1560 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1561 {
1562     if (count)
1563         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1564                           idx, (int) val, count,
1565                           xpercent(count, dump_stat->destroyedCount));
1566 }
1567
1568 static void
1569 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1570 {
1571     assert(hs);
1572     assert(e);
1573
1574     dump_stat = hs;
1575     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1576     storeAppendPrintf(e, "\nField type distribution\n");
1577     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1578                       "id", "name", "count", "#/header");
1579     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1580     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1581     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1582                       "id", "name", "count", "#/cc_field");
1583     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1584     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1585     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1586                       "id", "name", "count", "#/sc_field");
1587     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1588     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1589     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1590                       "id", "#flds", "count", "%total");
1591     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1592     storeAppendPrintf(e, "\n");
1593     dump_stat = NULL;
1594 }
1595
1596 void
1597 httpHeaderStoreReport(StoreEntry * e)
1598 {
1599     int i;
1600     assert(e);
1601
1602     HttpHeaderStats[0].parsedCount =
1603         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1604     HttpHeaderStats[0].ccParsedCount =
1605         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1606     HttpHeaderStats[0].destroyedCount =
1607         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1608     HttpHeaderStats[0].busyDestroyedCount =
1609         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1610
1611     for (i = 1; i < HttpHeaderStatCount; ++i) {
1612         httpHeaderStatDump(HttpHeaderStats + i, e);
1613     }
1614
1615     /* field stats for all messages */
1616     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1617
1618     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1619                       "id", "name", "#alive", "%err", "%repeat");
1620
1621     // scan heaaderTable and output
1622     for (auto h : WholeEnum<Http::HdrType>()) {
1623         auto stats = headerStatsTable[h];
1624         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1625                           Http::HeaderLookupTable.lookup(h).id,
1626                           Http::HeaderLookupTable.lookup(h).name,
1627                           stats.aliveCount,
1628                           xpercent(stats.errCount, stats.parsCount),
1629                           xpercent(stats.repCount, stats.seenCount));
1630     }
1631
1632     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1633                       HttpHeaderStats[hoRequest].parsedCount,
1634                       HttpHeaderStats[hoReply].parsedCount,
1635                       HttpHeaderStats[0].parsedCount);
1636     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1637 }
1638
1639 int
1640 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1641 {
1642     int result = 0;
1643     const char *pos = NULL;
1644     const char *item;
1645     int ilen;
1646     int mlen = strlen(member);
1647
1648     assert(any_registered_header(id));
1649
1650     String header (getStrOrList(id));
1651
1652     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1653         if (strncasecmp(item, member, mlen) == 0
1654                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1655             result = 1;
1656             break;
1657         }
1658     }
1659
1660     return result;
1661 }
1662
1663 int
1664 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1665 {
1666     int result = 0;
1667     const char *pos = NULL;
1668     const char *item;
1669     int ilen;
1670     int mlen = strlen(member);
1671
1672     assert(name);
1673
1674     String header (getByName(name));
1675
1676     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1677         if (strncasecmp(item, member, mlen) == 0
1678                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1679             result = 1;
1680             break;
1681         }
1682     }
1683
1684     return result;
1685 }
1686
1687 void
1688 HttpHeader::removeHopByHopEntries()
1689 {
1690     removeConnectionHeaderEntries();
1691
1692     const HttpHeaderEntry *e;
1693     HttpHeaderPos pos = HttpHeaderInitPos;
1694     int headers_deleted = 0;
1695     while ((e = getEntry(&pos))) {
1696         Http::HdrType id = e->id;
1697         if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1698             delAt(pos, headers_deleted);
1699             CBIT_CLR(mask, id);
1700         }
1701     }
1702 }
1703
1704 void
1705 HttpHeader::removeConnectionHeaderEntries()
1706 {
1707     if (has(Http::HdrType::CONNECTION)) {
1708         /* anything that matches Connection list member will be deleted */
1709         String strConnection;
1710
1711         (void) getList(Http::HdrType::CONNECTION, &strConnection);
1712         const HttpHeaderEntry *e;
1713         HttpHeaderPos pos = HttpHeaderInitPos;
1714         /*
1715          * think: on-average-best nesting of the two loops (hdrEntry
1716          * and strListItem) @?@
1717          */
1718         /*
1719          * maybe we should delete standard stuff ("keep-alive","close")
1720          * from strConnection first?
1721          */
1722
1723         int headers_deleted = 0;
1724         while ((e = getEntry(&pos))) {
1725             if (strListIsMember(&strConnection, e->name, ','))
1726                 delAt(pos, headers_deleted);
1727         }
1728         if (headers_deleted)
1729             refreshMask();
1730     }
1731 }
1732