src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/EnumIterator.h"
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "http/ContentLengthInterpreter.h"
  16 #include "HttpHdrCc.h"
  17 #include "HttpHdrContRange.h"
  18 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
  19 #include "HttpHeader.h"
  20 #include "HttpHeaderFieldInfo.h"
  21 #include "HttpHeaderStat.h"
  22 #include "HttpHeaderTools.h"
  23 #include "MemBuf.h"
  24 #include "mgr/Registration.h"
  25 #include "profiler/Profiler.h"
  26 #include "rfc1123.h"
  27 #include "SquidConfig.h"
  28 #include "StatHist.h"
  29 #include "Store.h"
  30 #include "StrList.h"
  31 #include "TimeOrTag.h"
  32 #include "util.h"
  33
  34 #include <algorithm>
  35
  36 /* XXX: the whole set of API managing the entries vector should be rethought
  37  *      after the parse4r-ng effort is complete.
  38  */
  39
  40 /*
  41  * On naming conventions:
  42  *
  43  * HTTP/1.1 defines message-header as
  44  *
  45  * message-header = field-name ":" [ field-value ] CRLF
  46  * field-name     = token
  47  * field-value    = *( field-content | LWS )
  48  *
  49  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  50  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  51  *
  52  * HttpHeader is an object that represents all message-headers in a message.
  53  * HttpHeader does not manage start-line.
  54  *
  55  * HttpHeader is implemented as a collection of header "entries".
  56  * An entry is a (field_id, field_name, field_value) triplet.
  57  */
  58
  59 /*
  60  * local constants and vars
  61  */
  62
  63 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
  64 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
  65
  66 /* request-only headers. Used for cachemgr */
  67 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
  68
  69 /* reply-only headers. Used for cachemgr */
  70 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
  71
  72 /* header accounting */
  73 // NP: keep in sync with enum http_hdr_owner_type
  74 static HttpHeaderStat HttpHeaderStats[] = {
  75     HttpHeaderStat(/*hoNone*/ "all", NULL),
  76 #if USE_HTCP
  77     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
  78 #endif
  79     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
  80     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
  81 #if USE_OPENSSL
  82     /* hoErrorDetail */
  83 #endif
  84     /* hoEnd */
  85 };
  86 static int HttpHeaderStatCount = countof(HttpHeaderStats);
  87
  88 static int HeaderEntryParsedCount = 0;
  89
  90 /*
  91  * forward declarations and local routines
  92  */
  93
  94 class StoreEntry;
  95
  96 // update parse statistics for header id; if error is true also account
  97 // for errors and write to debug log what happened
  98 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
  99 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 100 /** store report about current header usage and other stats */
 101 static void httpHeaderStoreReport(StoreEntry * e);
 102
 103 /*
 104  * Module initialization routines
 105  */
 106
 107 static void
 108 httpHeaderRegisterWithCacheManager(void)
 109 {
 110     Mgr::RegisterAction("http_headers",
 111                         "HTTP Header Statistics",
 112                         httpHeaderStoreReport, 0, 1);
 113 }
 114
 115 void
 116 httpHeaderInitModule(void)
 117 {
 118     /* check that we have enough space for masks */
 119     assert(8 * sizeof(HttpHeaderMask) >= Http::HdrType::enumEnd_);
 120
 121     // masks are needed for stats page still
 122     for (auto h : WholeEnum<Http::HdrType>()) {
 123         if (Http::HeaderLookupTable.lookup(h).request)
 124             CBIT_SET(RequestHeadersMask,h);
 125         if (Http::HeaderLookupTable.lookup(h).reply)
 126             CBIT_SET(ReplyHeadersMask,h);
 127     }
 128
 129     /* header stats initialized by class constructor */
 130     assert(HttpHeaderStatCount == hoReply + 1);
 131
 132     /* init dependent modules */
 133     httpHdrCcInitModule();
 134     httpHdrScInitModule();
 135
 136     httpHeaderRegisterWithCacheManager();
 137 }
 138
 139 /*
 140  * HttpHeader Implementation
 141  */
 142
 143 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
 144 {
 145     httpHeaderMaskInit(&mask, 0);
 146 }
 147
 148 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
 149 {
 150     assert(anOwner > hoNone && anOwner < hoEnd);
 151     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 152     httpHeaderMaskInit(&mask, 0);
 153 }
 154
 155 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
 156 {
 157     httpHeaderMaskInit(&mask, 0);
 158     update(&other); // will update the mask as well
 159 }
 160
 161 HttpHeader::~HttpHeader()
 162 {
 163     clean();
 164 }
 165
 166 HttpHeader &
 167 HttpHeader::operator =(const HttpHeader &other)
 168 {
 169     if (this != &other) {
 170         // we do not really care, but the caller probably does
 171         assert(owner == other.owner);
 172         clean();
 173         update(&other); // will update the mask as well
 174         len = other.len;
 175         conflictingContentLength_ = other.conflictingContentLength_;
 176     }
 177     return *this;
 178 }
 179
 180 void
 181 HttpHeader::clean()
 182 {
 183
 184     assert(owner > hoNone && owner < hoEnd);
 185     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 186
 187     PROF_start(HttpHeaderClean);
 188
 189     if (owner <= hoReply) {
 190         /*
 191          * An unfortunate bug.  The entries array is initialized
 192          * such that count is set to zero.  httpHeaderClean() seems to
 193          * be called both when 'hdr' is created, and destroyed.  Thus,
 194          * we accumulate a large number of zero counts for 'hdr' before
 195          * it is ever used.  Can't think of a good way to fix it, except
 196          * adding a state variable that indicates whether or not 'hdr'
 197          * has been used.  As a hack, just never count zero-sized header
 198          * arrays.
 199          */
 200         if (!entries.empty())
 201             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 202
 203         ++ HttpHeaderStats[owner].destroyedCount;
 204
 205         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 206     } // if (owner <= hoReply)
 207
 208     for (HttpHeaderEntry *e : entries) {
 209         if (e == nullptr)
 210             continue;
 211         if (!Http::any_valid_header(e->id)) {
 212             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 213         } else {
 214             if (owner <= hoReply)
 215                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 216             delete e;
 217         }
 218     }
 219
 220     entries.clear();
 221     httpHeaderMaskInit(&mask, 0);
 222     len = 0;
 223     conflictingContentLength_ = false;
 224     PROF_stop(HttpHeaderClean);
 225 }
 226
 227 /* append entries (also see httpHeaderUpdate) */
 228 void
 229 HttpHeader::append(const HttpHeader * src)
 230 {
 231     assert(src);
 232     assert(src != this);
 233     debugs(55, 7, "appending hdr: " << this << " += " << src);
 234
 235     for (auto e : src->entries) {
 236         if (e)
 237             addEntry(e->clone());
 238     }
 239 }
 240
 241 /// check whether the fresh header has any new/changed updatable fields
 242 bool
 243 HttpHeader::needUpdate(HttpHeader const *fresh) const
 244 {
 245     for (const auto e: fresh->entries) {
 246         if (!e || skipUpdateHeader(e->id))
 247             continue;
 248         String value;
 249         const char *name = e->name.termedBuf();
 250         if (!getByNameIfPresent(name, strlen(name), value) ||
 251                 (value != fresh->getByName(name)))
 252             return true;
 253     }
 254     return false;
 255 }
 256
 257 void
 258 HttpHeader::updateWarnings()
 259 {
 260     int count = 0;
 261     HttpHeaderPos pos = HttpHeaderInitPos;
 262
 263     // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
 264     while (HttpHeaderEntry *e = getEntry(&pos)) {
 265         if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
 266             delAt(pos, count);
 267     }
 268 }
 269
 270 bool
 271 HttpHeader::skipUpdateHeader(const Http::HdrType id) const
 272 {
 273     // RFC 7234, section 4.3.4: use fields other from Warning for update
 274     return id == Http::HdrType::WARNING;
 275 }
 276
 277 bool
 278 HttpHeader::update(HttpHeader const *fresh)
 279 {
 280     assert(fresh);
 281     assert(this != fresh);
 282
 283     // Optimization: Finding whether a header field changed is expensive
 284     // and probably not worth it except for collapsed revalidation needs.
 285     if (Config.onoff.collapsed_forwarding && !needUpdate(fresh))
 286         return false;
 287
 288     updateWarnings();
 289
 290     const HttpHeaderEntry *e;
 291     HttpHeaderPos pos = HttpHeaderInitPos;
 292
 293     while ((e = fresh->getEntry(&pos))) {
 294         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 295
 296         if (skipUpdateHeader(e->id))
 297             continue;
 298
 299         if (e->id != Http::HdrType::OTHER)
 300             delById(e->id);
 301         else
 302             delByName(e->name.termedBuf());
 303     }
 304
 305     pos = HttpHeaderInitPos;
 306     while ((e = fresh->getEntry(&pos))) {
 307         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 308
 309         if (skipUpdateHeader(e->id))
 310             continue;
 311
 312         debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
 313
 314         addEntry(e->clone());
 315     }
 316     return true;
 317 }
 318
 319 int
 320 HttpHeader::parse(const char *header_start, size_t hdrLen)
 321 {
 322     const char *field_ptr = header_start;
 323     const char *header_end = header_start + hdrLen; // XXX: remove
 324     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 325
 326     PROF_start(HttpHeaderParse);
 327
 328     assert(header_start && header_end);
 329     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 330     ++ HttpHeaderStats[owner].parsedCount;
 331
 332     char *nulpos;
 333     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 334         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 335                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 336         PROF_stop(HttpHeaderParse);
 337         clean();
 338         return 0;
 339     }
 340
 341     Http::ContentLengthInterpreter clen(warnOnError);
 342     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 343      * continuation lines start with a (single) space or tab */
 344     while (field_ptr < header_end) {
 345         const char *field_start = field_ptr;
 346         const char *field_end;
 347
 348         do {
 349             const char *this_line = field_ptr;
 350             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 351
 352             if (!field_ptr) {
 353                 // missing <LF>
 354                 PROF_stop(HttpHeaderParse);
 355                 clean();
 356                 return 0;
 357             }
 358
 359             field_end = field_ptr;
 360
 361             ++field_ptr;    /* Move to next line */
 362
 363             if (field_end > this_line && field_end[-1] == '\r') {
 364                 --field_end;    /* Ignore CR LF */
 365
 366                 if (owner == hoRequest && field_end > this_line) {
 367                     bool cr_only = true;
 368                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 369                         if (*p != '\r')
 370                             cr_only = false;
 371                     }
 372                     if (cr_only) {
 373                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 374                                "header field to prevent request smuggling attacks: {" <<
 375                                getStringPrefix(header_start, hdrLen) << "}");
 376                         PROF_stop(HttpHeaderParse);
 377                         clean();
 378                         return 0;
 379                     }
 380                 }
 381             }
 382
 383             /* Barf on stray CR characters */
 384             if (memchr(this_line, '\r', field_end - this_line)) {
 385                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 386                        getStringPrefix(field_start, field_end-field_start) << "}");
 387
 388                 if (Config.onoff.relaxed_header_parser) {
 389                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 390
 391                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 392                         *p = ' ';
 393                         ++p;
 394                     }
 395                 } else {
 396                     PROF_stop(HttpHeaderParse);
 397                     clean();
 398                     return 0;
 399                 }
 400             }
 401
 402             if (this_line + 1 == field_end && this_line > field_start) {
 403                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 404                        getStringPrefix(header_start, hdrLen) << "}");
 405                 PROF_stop(HttpHeaderParse);
 406                 clean();
 407                 return 0;
 408             }
 409         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 410
 411         if (field_start == field_end) {
 412             if (field_ptr < header_end) {
 413                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 414                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 415                 PROF_stop(HttpHeaderParse);
 416                 clean();
 417                 return 0;
 418             }
 419
 420             break;      /* terminating blank line */
 421         }
 422
 423         HttpHeaderEntry *e;
 424         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 425             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 426                    getStringPrefix(field_start, field_end-field_start) << "}");
 427             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 428
 429             if (Config.onoff.relaxed_header_parser)
 430                 continue;
 431
 432             PROF_stop(HttpHeaderParse);
 433             clean();
 434             return 0;
 435         }
 436
 437         if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
 438             delete e;
 439
 440             if (Config.onoff.relaxed_header_parser)
 441                 continue; // clen has printed any necessary warnings
 442
 443             PROF_stop(HttpHeaderParse);
 444             clean();
 445             return 0;
 446         }
 447
 448         if (e->id == Http::HdrType::OTHER && stringHasWhitespace(e->name.termedBuf())) {
 449             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 450                    getStringPrefix(field_start, field_end-field_start) << "}");
 451
 452             if (!Config.onoff.relaxed_header_parser) {
 453                 delete e;
 454                 PROF_stop(HttpHeaderParse);
 455                 clean();
 456                 return 0;
 457             }
 458         }
 459
 460         addEntry(e);
 461     }
 462
 463     if (clen.headerWideProblem) {
 464         debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
 465                " Content-Length field values in" <<
 466                Raw("header", header_start, hdrLen));
 467     }
 468
 469     if (chunked()) {
 470         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 471         // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
 472         delById(Http::HdrType::CONTENT_LENGTH);
 473         // and clen state becomes irrelevant
 474     } else if (clen.sawBad) {
 475         // ensure our callers do not accidentally see bad Content-Length values
 476         delById(Http::HdrType::CONTENT_LENGTH);
 477         conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
 478     } else if (clen.needsSanitizing) {
 479         // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
 480         // ensure our callers see a clean Content-Length value or none at all
 481         delById(Http::HdrType::CONTENT_LENGTH);
 482         if (clen.sawGood) {
 483             putInt64(Http::HdrType::CONTENT_LENGTH, clen.value);
 484             debugs(55, 5, "sanitized Content-Length to be " << clen.value);
 485         }
 486     }
 487
 488     PROF_stop(HttpHeaderParse);
 489     return 1;           /* even if no fields where found, it is a valid header */
 490 }
 491
 492 /* packs all the entries using supplied packer */
 493 void
 494 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 495 {
 496     HttpHeaderPos pos = HttpHeaderInitPos;
 497     const HttpHeaderEntry *e;
 498     assert(p);
 499     debugs(55, 7, this << " into " << p <<
 500            (mask_sensitive_info ? " while masking" : ""));
 501     /* pack all entries one by one */
 502     while ((e = getEntry(&pos))) {
 503         if (!mask_sensitive_info) {
 504             e->packInto(p);
 505             continue;
 506         }
 507
 508         bool maskThisEntry = false;
 509         switch (e->id) {
 510         case Http::HdrType::AUTHORIZATION:
 511         case Http::HdrType::PROXY_AUTHORIZATION:
 512             maskThisEntry = true;
 513             break;
 514
 515         case Http::HdrType::FTP_ARGUMENTS:
 516             if (const HttpHeaderEntry *cmd = findEntry(Http::HdrType::FTP_COMMAND))
 517                 maskThisEntry = (cmd->value == "PASS");
 518             break;
 519
 520         default:
 521             break;
 522         }
 523         if (maskThisEntry) {
 524             p->append(e->name.rawBuf(), e->name.size());
 525             p->append(": ** NOT DISPLAYED **\r\n", 23);
 526         } else {
 527             e->packInto(p);
 528         }
 529
 530     }
 531     /* Pack in the "special" entries */
 532
 533     /* Cache-Control */
 534 }
 535
 536 /* returns next valid entry */
 537 HttpHeaderEntry *
 538 HttpHeader::getEntry(HttpHeaderPos * pos) const
 539 {
 540     assert(pos);
 541     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 542
 543     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 544         if (entries[*pos])
 545             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 546     }
 547
 548     return NULL;
 549 }
 550
 551 /*
 552  * returns a pointer to a specified entry if any
 553  * note that we return one entry so it does not make much sense to ask for
 554  * "list" headers
 555  */
 556 HttpHeaderEntry *
 557 HttpHeader::findEntry(Http::HdrType id) const
 558 {
 559     assert(any_registered_header(id));
 560     assert(!Http::HeaderLookupTable.lookup(id).list);
 561
 562     /* check mask first */
 563
 564     if (!CBIT_TEST(mask, id))
 565         return NULL;
 566
 567     /* looks like we must have it, do linear search */
 568     for (auto e : entries) {
 569         if (e && e->id == id)
 570             return e;
 571     }
 572
 573     /* hm.. we thought it was there, but it was not found */
 574     assert(false);
 575     return nullptr;        /* not reached */
 576 }
 577
 578 /*
 579  * same as httpHeaderFindEntry
 580  */
 581 HttpHeaderEntry *
 582 HttpHeader::findLastEntry(Http::HdrType id) const
 583 {
 584     assert(any_registered_header(id));
 585     assert(!Http::HeaderLookupTable.lookup(id).list);
 586
 587     /* check mask first */
 588     if (!CBIT_TEST(mask, id))
 589         return NULL;
 590
 591     for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
 592         if (*e && (*e)->id == id)
 593             return *e;
 594     }
 595
 596     /* hm.. we thought it was there, but it was not found */
 597     assert(false);
 598     return nullptr; /* not reached */
 599 }
 600
 601 /*
 602  * deletes all fields with a given name if any, returns #fields deleted;
 603  */
 604 int
 605 HttpHeader::delByName(const char *name)
 606 {
 607     int count = 0;
 608     HttpHeaderPos pos = HttpHeaderInitPos;
 609     HttpHeaderEntry *e;
 610     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 611     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 612
 613     while ((e = getEntry(&pos))) {
 614         if (!e->name.caseCmp(name))
 615             delAt(pos, count);
 616         else
 617             CBIT_SET(mask, e->id);
 618     }
 619
 620     return count;
 621 }
 622
 623 /* deletes all entries with a given id, returns the #entries deleted */
 624 int
 625 HttpHeader::delById(Http::HdrType id)
 626 {
 627     debugs(55, 8, this << " del-by-id " << id);
 628     assert(any_registered_header(id));
 629
 630     if (!CBIT_TEST(mask, id))
 631         return 0;
 632
 633     int count = 0;
 634
 635     HttpHeaderPos pos = HttpHeaderInitPos;
 636     while (HttpHeaderEntry *e = getEntry(&pos)) {
 637         if (e->id == id)
 638             delAt(pos, count); // deletes e
 639     }
 640
 641     CBIT_CLR(mask, id);
 642     assert(count);
 643     return count;
 644 }
 645
 646 /*
 647  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 648  * possible to iterate(search) and delete fields at the same time
 649  * NOTE: Does not update the header mask. Caller must follow up with
 650  * a call to refreshMask() if headers_deleted was incremented.
 651  */
 652 void
 653 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 654 {
 655     HttpHeaderEntry *e;
 656     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 657     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 658     entries[pos] = NULL;
 659     /* decrement header length, allow for ": " and crlf */
 660     len -= e->name.size() + 2 + e->value.size() + 2;
 661     assert(len >= 0);
 662     delete e;
 663     ++headers_deleted;
 664 }
 665
 666 /*
 667  * Compacts the header storage
 668  */
 669 void
 670 HttpHeader::compact()
 671 {
 672     // TODO: optimize removal, or possibly make it so that's not needed.
 673     entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
 674                    entries.end());
 675 }
 676
 677 /*
 678  * Refreshes the header mask. Required after delAt() calls.
 679  */
 680 void
 681 HttpHeader::refreshMask()
 682 {
 683     httpHeaderMaskInit(&mask, 0);
 684     debugs(55, 7, "refreshing the mask in hdr " << this);
 685     for (auto e : entries) {
 686         if (e)
 687             CBIT_SET(mask, e->id);
 688     }
 689 }
 690
 691 /* appends an entry;
 692  * does not call e->clone() so one should not reuse "*e"
 693  */
 694 void
 695 HttpHeader::addEntry(HttpHeaderEntry * e)
 696 {
 697     assert(e);
 698     assert(any_HdrType_enum_value(e->id));
 699     assert(e->name.size());
 700
 701     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 702
 703     if (e->id != Http::HdrType::BAD_HDR) {
 704         if (CBIT_TEST(mask, e->id)) {
 705             ++ headerStatsTable[e->id].repCount;
 706         } else {
 707             CBIT_SET(mask, e->id);
 708         }
 709     }
 710
 711     entries.push_back(e);
 712
 713     /* increment header length, allow for ": " and crlf */
 714     len += e->name.size() + 2 + e->value.size() + 2;
 715 }
 716
 717 /* inserts an entry;
 718  * does not call e->clone() so one should not reuse "*e"
 719  */
 720 void
 721 HttpHeader::insertEntry(HttpHeaderEntry * e)
 722 {
 723     assert(e);
 724     assert(any_valid_header(e->id));
 725
 726     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 727
 728     // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
 729     if (CBIT_TEST(mask, e->id)) {
 730         ++ headerStatsTable[e->id].repCount;
 731     } else {
 732         CBIT_SET(mask, e->id);
 733     }
 734
 735     entries.insert(entries.begin(),e);
 736
 737     /* increment header length, allow for ": " and crlf */
 738     len += e->name.size() + 2 + e->value.size() + 2;
 739 }
 740
 741 bool
 742 HttpHeader::getList(Http::HdrType id, String *s) const
 743 {
 744     debugs(55, 9, this << " joining for id " << id);
 745     /* only fields from ListHeaders array can be "listed" */
 746     assert(Http::HeaderLookupTable.lookup(id).list);
 747
 748     if (!CBIT_TEST(mask, id))
 749         return false;
 750
 751     for (auto e: entries) {
 752         if (e && e->id == id)
 753             strListAdd(s, e->value.termedBuf(), ',');
 754     }
 755
 756     /*
 757      * note: we might get an empty (size==0) string if there was an "empty"
 758      * header. This results in an empty length String, which may have a NULL
 759      * buffer.
 760      */
 761     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 762     if (!s->size())
 763         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 764     else
 765         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 766
 767     return true;
 768 }
 769
 770 /* return a list of entries with the same id separated by ',' and ws */
 771 String
 772 HttpHeader::getList(Http::HdrType id) const
 773 {
 774     HttpHeaderEntry *e;
 775     HttpHeaderPos pos = HttpHeaderInitPos;
 776     debugs(55, 9, this << "joining for id " << id);
 777     /* only fields from ListHeaders array can be "listed" */
 778     assert(Http::HeaderLookupTable.lookup(id).list);
 779
 780     if (!CBIT_TEST(mask, id))
 781         return String();
 782
 783     String s;
 784
 785     while ((e = getEntry(&pos))) {
 786         if (e->id == id)
 787             strListAdd(&s, e->value.termedBuf(), ',');
 788     }
 789
 790     /*
 791      * note: we might get an empty (size==0) string if there was an "empty"
 792      * header. This results in an empty length String, which may have a NULL
 793      * buffer.
 794      */
 795     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 796     if (!s.size())
 797         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 798     else
 799         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 800
 801     return s;
 802 }
 803
 804 /* return a string or list of entries with the same id separated by ',' and ws */
 805 String
 806 HttpHeader::getStrOrList(Http::HdrType id) const
 807 {
 808     HttpHeaderEntry *e;
 809
 810     if (Http::HeaderLookupTable.lookup(id).list)
 811         return getList(id);
 812
 813     if ((e = findEntry(id)))
 814         return e->value;
 815
 816     return String();
 817 }
 818
 819 /*
 820  * Returns the value of the specified header and/or an undefined String.
 821  */
 822 String
 823 HttpHeader::getByName(const char *name) const
 824 {
 825     String result;
 826     // ignore presence: return undefined string if an empty header is present
 827     (void)getByNameIfPresent(name, strlen(name), result);
 828     return result;
 829 }
 830
 831 String
 832 HttpHeader::getByName(const SBuf &name) const
 833 {
 834     String result;
 835     // ignore presence: return undefined string if an empty header is present
 836     (void)getByNameIfPresent(name, result);
 837     return result;
 838 }
 839
 840 String
 841 HttpHeader::getById(Http::HdrType id) const
 842 {
 843     String result;
 844     (void)getByIdIfPresent(id,result);
 845     return result;
 846 }
 847
 848 bool
 849 HttpHeader::getByNameIfPresent(const SBuf &s, String &result) const
 850 {
 851     return getByNameIfPresent(s.rawContent(), s.length(), result);
 852 }
 853
 854 bool
 855 HttpHeader::getByIdIfPresent(Http::HdrType id, String &result) const
 856 {
 857     if (id == Http::HdrType::BAD_HDR)
 858         return false;
 859     if (!has(id))
 860         return false;
 861     result = getStrOrList(id);
 862     return true;
 863 }
 864
 865 bool
 866 HttpHeader::getByNameIfPresent(const char *name, int namelen, String &result) const
 867 {
 868     Http::HdrType id;
 869     HttpHeaderPos pos = HttpHeaderInitPos;
 870     HttpHeaderEntry *e;
 871
 872     assert(name);
 873
 874     /* First try the quick path */
 875     id = Http::HeaderLookupTable.lookup(name,namelen).id;
 876
 877     if (id != Http::HdrType::BAD_HDR) {
 878         if (getByIdIfPresent(id, result))
 879             return true;
 880     }
 881
 882     /* Sorry, an unknown header name. Do linear search */
 883     bool found = false;
 884     while ((e = getEntry(&pos))) {
 885         if (e->id == Http::HdrType::OTHER && e->name.caseCmp(name, namelen) == 0) {
 886             found = true;
 887             strListAdd(&result, e->value.termedBuf(), ',');
 888         }
 889     }
 890
 891     return found;
 892 }
 893
 894 /*
 895  * Returns a the value of the specified list member, if any.
 896  */
 897 String
 898 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
 899 {
 900     String header;
 901     const char *pos = NULL;
 902     const char *item;
 903     int ilen;
 904     int mlen = strlen(member);
 905
 906     assert(name);
 907
 908     header = getByName(name);
 909
 910     String result;
 911
 912     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 913         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 914             result.append(item + mlen + 1, ilen - mlen - 1);
 915             break;
 916         }
 917     }
 918
 919     return result;
 920 }
 921
 922 /*
 923  * returns a the value of the specified list member, if any.
 924  */
 925 String
 926 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
 927 {
 928     String header;
 929     const char *pos = NULL;
 930     const char *item;
 931     int ilen;
 932     int mlen = strlen(member);
 933
 934     assert(any_registered_header(id));
 935
 936     header = getStrOrList(id);
 937     String result;
 938
 939     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 940         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 941             result.append(item + mlen + 1, ilen - mlen - 1);
 942             break;
 943         }
 944     }
 945
 946     header.clean();
 947     return result;
 948 }
 949
 950 /* test if a field is present */
 951 int
 952 HttpHeader::has(Http::HdrType id) const
 953 {
 954     assert(any_registered_header(id));
 955     debugs(55, 9, this << " lookup for " << id);
 956     return CBIT_TEST(mask, id);
 957 }
 958
 959 void
 960 HttpHeader::putInt(Http::HdrType id, int number)
 961 {
 962     assert(any_registered_header(id));
 963     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
 964     assert(number >= 0);
 965     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
 966 }
 967
 968 void
 969 HttpHeader::putInt64(Http::HdrType id, int64_t number)
 970 {
 971     assert(any_registered_header(id));
 972     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
 973     assert(number >= 0);
 974     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
 975 }
 976
 977 void
 978 HttpHeader::putTime(Http::HdrType id, time_t htime)
 979 {
 980     assert(any_registered_header(id));
 981     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
 982     assert(htime >= 0);
 983     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
 984 }
 985
 986 void
 987 HttpHeader::putStr(Http::HdrType id, const char *str)
 988 {
 989     assert(any_registered_header(id));
 990     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
 991     assert(str);
 992     addEntry(new HttpHeaderEntry(id, NULL, str));
 993 }
 994
 995 void
 996 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
 997 {
 998     assert(auth_scheme && realm);
 999     httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1000 }
1001
1002 void
1003 HttpHeader::putCc(const HttpHdrCc * cc)
1004 {
1005     assert(cc);
1006     /* remove old directives if any */
1007     delById(Http::HdrType::CACHE_CONTROL);
1008     /* pack into mb */
1009     MemBuf mb;
1010     mb.init();
1011     cc->packInto(&mb);
1012     /* put */
1013     addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL, NULL, mb.buf));
1014     /* cleanup */
1015     mb.clean();
1016 }
1017
1018 void
1019 HttpHeader::putContRange(const HttpHdrContRange * cr)
1020 {
1021     assert(cr);
1022     /* remove old directives if any */
1023     delById(Http::HdrType::CONTENT_RANGE);
1024     /* pack into mb */
1025     MemBuf mb;
1026     mb.init();
1027     httpHdrContRangePackInto(cr, &mb);
1028     /* put */
1029     addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE, NULL, mb.buf));
1030     /* cleanup */
1031     mb.clean();
1032 }
1033
1034 void
1035 HttpHeader::putRange(const HttpHdrRange * range)
1036 {
1037     assert(range);
1038     /* remove old directives if any */
1039     delById(Http::HdrType::RANGE);
1040     /* pack into mb */
1041     MemBuf mb;
1042     mb.init();
1043     range->packInto(&mb);
1044     /* put */
1045     addEntry(new HttpHeaderEntry(Http::HdrType::RANGE, NULL, mb.buf));
1046     /* cleanup */
1047     mb.clean();
1048 }
1049
1050 void
1051 HttpHeader::putSc(HttpHdrSc *sc)
1052 {
1053     assert(sc);
1054     /* remove old directives if any */
1055     delById(Http::HdrType::SURROGATE_CONTROL);
1056     /* pack into mb */
1057     MemBuf mb;
1058     mb.init();
1059     sc->packInto(&mb);
1060     /* put */
1061     addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL, NULL, mb.buf));
1062     /* cleanup */
1063     mb.clean();
1064 }
1065
1066 void
1067 HttpHeader::putWarning(const int code, const char *const text)
1068 {
1069     char buf[512];
1070     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1071     putStr(Http::HdrType::WARNING, buf);
1072 }
1073
1074 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1075 void
1076 HttpHeader::putExt(const char *name, const char *value)
1077 {
1078     assert(name && value);
1079     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1080     addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, name, value));
1081 }
1082
1083 int
1084 HttpHeader::getInt(Http::HdrType id) const
1085 {
1086     assert(any_registered_header(id));
1087     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1088     HttpHeaderEntry *e;
1089
1090     if ((e = findEntry(id)))
1091         return e->getInt();
1092
1093     return -1;
1094 }
1095
1096 int64_t
1097 HttpHeader::getInt64(Http::HdrType id) const
1098 {
1099     assert(any_registered_header(id));
1100     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1101     HttpHeaderEntry *e;
1102
1103     if ((e = findEntry(id)))
1104         return e->getInt64();
1105
1106     return -1;
1107 }
1108
1109 time_t
1110 HttpHeader::getTime(Http::HdrType id) const
1111 {
1112     HttpHeaderEntry *e;
1113     time_t value = -1;
1114     assert(any_registered_header(id));
1115     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1116
1117     if ((e = findEntry(id))) {
1118         value = parse_rfc1123(e->value.termedBuf());
1119         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1120     }
1121
1122     return value;
1123 }
1124
1125 /* sync with httpHeaderGetLastStr */
1126 const char *
1127 HttpHeader::getStr(Http::HdrType id) const
1128 {
1129     HttpHeaderEntry *e;
1130     assert(any_registered_header(id));
1131     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1132
1133     if ((e = findEntry(id))) {
1134         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1135         return e->value.termedBuf();
1136     }
1137
1138     return NULL;
1139 }
1140
1141 /* unusual */
1142 const char *
1143 HttpHeader::getLastStr(Http::HdrType id) const
1144 {
1145     HttpHeaderEntry *e;
1146     assert(any_registered_header(id));
1147     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1148
1149     if ((e = findLastEntry(id))) {
1150         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1151         return e->value.termedBuf();
1152     }
1153
1154     return NULL;
1155 }
1156
1157 HttpHdrCc *
1158 HttpHeader::getCc() const
1159 {
1160     if (!CBIT_TEST(mask, Http::HdrType::CACHE_CONTROL))
1161         return NULL;
1162     PROF_start(HttpHeader_getCc);
1163
1164     String s;
1165     getList(Http::HdrType::CACHE_CONTROL, &s);
1166
1167     HttpHdrCc *cc=new HttpHdrCc();
1168
1169     if (!cc->parse(s)) {
1170         delete cc;
1171         cc = NULL;
1172     }
1173
1174     ++ HttpHeaderStats[owner].ccParsedCount;
1175
1176     if (cc)
1177         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1178
1179     httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL, s, !cc);
1180
1181     PROF_stop(HttpHeader_getCc);
1182
1183     return cc;
1184 }
1185
1186 HttpHdrRange *
1187 HttpHeader::getRange() const
1188 {
1189     HttpHdrRange *r = NULL;
1190     HttpHeaderEntry *e;
1191     /* some clients will send "Request-Range" _and_ *matching* "Range"
1192      * who knows, some clients might send Request-Range only;
1193      * this "if" should work correctly in both cases;
1194      * hopefully no clients send mismatched headers! */
1195
1196     if ((e = findEntry(Http::HdrType::RANGE)) ||
1197             (e = findEntry(Http::HdrType::REQUEST_RANGE))) {
1198         r = HttpHdrRange::ParseCreate(&e->value);
1199         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1200     }
1201
1202     return r;
1203 }
1204
1205 HttpHdrSc *
1206 HttpHeader::getSc() const
1207 {
1208     if (!CBIT_TEST(mask, Http::HdrType::SURROGATE_CONTROL))
1209         return NULL;
1210
1211     String s;
1212
1213     (void) getList(Http::HdrType::SURROGATE_CONTROL, &s);
1214
1215     HttpHdrSc *sc = httpHdrScParseCreate(s);
1216
1217     ++ HttpHeaderStats[owner].ccParsedCount;
1218
1219     if (sc)
1220         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1221
1222     httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL, s, !sc);
1223
1224     return sc;
1225 }
1226
1227 HttpHdrContRange *
1228 HttpHeader::getContRange() const
1229 {
1230     HttpHdrContRange *cr = NULL;
1231     HttpHeaderEntry *e;
1232
1233     if ((e = findEntry(Http::HdrType::CONTENT_RANGE))) {
1234         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1235         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1236     }
1237
1238     return cr;
1239 }
1240
1241 const char *
1242 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1243 {
1244     const char *field;
1245     int l;
1246     assert(auth_scheme);
1247     field = getStr(id);
1248
1249     if (!field)         /* no authorization field */
1250         return NULL;
1251
1252     l = strlen(auth_scheme);
1253
1254     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1255         return NULL;
1256
1257     field += l;
1258
1259     if (!xisspace(*field))  /* wrong scheme */
1260         return NULL;
1261
1262     /* skip white space */
1263     for (; field && xisspace(*field); ++field);
1264
1265     if (!*field)        /* no authorization cookie */
1266         return NULL;
1267
1268     static char decodedAuthToken[8192];
1269     struct base64_decode_ctx ctx;
1270     base64_decode_init(&ctx);
1271     size_t decodedLen = 0;
1272     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1273             !base64_decode_final(&ctx)) {
1274         return NULL;
1275     }
1276     decodedAuthToken[decodedLen] = '\0';
1277     return decodedAuthToken;
1278 }
1279
1280 ETag
1281 HttpHeader::getETag(Http::HdrType id) const
1282 {
1283     ETag etag = {NULL, -1};
1284     HttpHeaderEntry *e;
1285     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag);     /* must be of an appropriate type */
1286
1287     if ((e = findEntry(id)))
1288         etagParseInit(&etag, e->value.termedBuf());
1289
1290     return etag;
1291 }
1292
1293 TimeOrTag
1294 HttpHeader::getTimeOrTag(Http::HdrType id) const
1295 {
1296     TimeOrTag tot;
1297     HttpHeaderEntry *e;
1298     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag);    /* must be of an appropriate type */
1299     memset(&tot, 0, sizeof(tot));
1300
1301     if ((e = findEntry(id))) {
1302         const char *str = e->value.termedBuf();
1303         /* try as an ETag */
1304
1305         if (etagParseInit(&tot.tag, str)) {
1306             tot.valid = tot.tag.str != NULL;
1307             tot.time = -1;
1308         } else {
1309             /* or maybe it is time? */
1310             tot.time = parse_rfc1123(str);
1311             tot.valid = tot.time >= 0;
1312             tot.tag.str = NULL;
1313         }
1314     }
1315
1316     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1317     return tot;
1318 }
1319
1320 /*
1321  * HttpHeaderEntry
1322  */
1323
1324 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const char *aName, const char *aValue)
1325 {
1326     assert(any_HdrType_enum_value(anId));
1327     id = anId;
1328
1329     if (id != Http::HdrType::OTHER)
1330         name = Http::HeaderLookupTable.lookup(id).name;
1331     else
1332         name = aName;
1333
1334     value = aValue;
1335
1336     if (id != Http::HdrType::BAD_HDR)
1337         ++ headerStatsTable[id].aliveCount;
1338
1339     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1340 }
1341
1342 HttpHeaderEntry::~HttpHeaderEntry()
1343 {
1344     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1345
1346     if (id != Http::HdrType::BAD_HDR) {
1347         assert(headerStatsTable[id].aliveCount);
1348         -- headerStatsTable[id].aliveCount;
1349         id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1350     }
1351
1352 }
1353
1354 /* parses and inits header entry, returns true/false */
1355 HttpHeaderEntry *
1356 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1357 {
1358     /* note: name_start == field_start */
1359     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1360     int name_len = name_end ? name_end - field_start :0;
1361     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1362     /* note: value_end == field_end */
1363
1364     ++ HeaderEntryParsedCount;
1365
1366     /* do we have a valid field name within this field? */
1367
1368     if (!name_len || name_end > field_end)
1369         return NULL;
1370
1371     if (name_len > 65534) {
1372         /* String must be LESS THAN 64K and it adds a terminating NULL */
1373         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1374         return NULL;
1375     }
1376
1377     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1378         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1379                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1380
1381         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1382             --name_len;
1383
1384         if (!name_len)
1385             return NULL;
1386     }
1387
1388     /* now we know we can parse it */
1389
1390     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1391
1392     /* is it a "known" field? */
1393     Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1394     debugs(55, 9, "got hdr-id=" << id);
1395
1396     String name;
1397
1398     String value;
1399
1400     if (id == Http::HdrType::BAD_HDR)
1401         id = Http::HdrType::OTHER;
1402
1403     /* set field name */
1404     if (id == Http::HdrType::OTHER)
1405         name.limitInit(field_start, name_len);
1406     else
1407         name = Http::HeaderLookupTable.lookup(id).name;
1408
1409     /* trim field value */
1410     while (value_start < field_end && xisspace(*value_start))
1411         ++value_start;
1412
1413     while (value_start < field_end && xisspace(field_end[-1]))
1414         --field_end;
1415
1416     if (field_end - value_start > 65534) {
1417         /* String must be LESS THAN 64K and it adds a terminating NULL */
1418         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1419
1420         if (id == Http::HdrType::OTHER)
1421             name.clean();
1422
1423         return NULL;
1424     }
1425
1426     /* set field value */
1427     value.limitInit(value_start, field_end - value_start);
1428
1429     if (id != Http::HdrType::BAD_HDR)
1430         ++ headerStatsTable[id].seenCount;
1431
1432     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1433
1434     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1435 }
1436
1437 HttpHeaderEntry *
1438 HttpHeaderEntry::clone() const
1439 {
1440     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1441 }
1442
1443 void
1444 HttpHeaderEntry::packInto(Packable * p) const
1445 {
1446     assert(p);
1447     p->append(name.rawBuf(), name.size());
1448     p->append(": ", 2);
1449     p->append(value.rawBuf(), value.size());
1450     p->append("\r\n", 2);
1451 }
1452
1453 int
1454 HttpHeaderEntry::getInt() const
1455 {
1456     int val = -1;
1457     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1458     httpHeaderNoteParsedEntry(id, value, ok == 0);
1459     /* XXX: Should we check ok - ie
1460      * return ok ? -1 : value;
1461      */
1462     return val;
1463 }
1464
1465 int64_t
1466 HttpHeaderEntry::getInt64() const
1467 {
1468     int64_t val = -1;
1469     const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1470     httpHeaderNoteParsedEntry(id, value, ok);
1471     return val; // remains -1 if !ok (XXX: bad method API)
1472 }
1473
1474 static void
1475 httpHeaderNoteParsedEntry(Http::HdrType id, String const &context, bool error)
1476 {
1477     if (id != Http::HdrType::BAD_HDR)
1478         ++ headerStatsTable[id].parsCount;
1479
1480     if (error) {
1481         if (id != Http::HdrType::BAD_HDR)
1482             ++ headerStatsTable[id].errCount;
1483         debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1484     }
1485 }
1486
1487 /*
1488  * Reports
1489  */
1490
1491 /* tmp variable used to pass stat info to dumpers */
1492 extern const HttpHeaderStat *dump_stat;     /* argh! */
1493 const HttpHeaderStat *dump_stat = NULL;
1494
1495 void
1496 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1497 {
1498     const int id = static_cast<int>(val);
1499     const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1500     const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1501     int visible = count > 0;
1502     /* for entries with zero count, list only those that belong to current type of message */
1503
1504     if (!visible && valid_id && dump_stat->owner_mask)
1505         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1506
1507     if (visible)
1508         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1509                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1510 }
1511
1512 static void
1513 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1514 {
1515     if (count)
1516         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1517                           idx, (int) val, count,
1518                           xpercent(count, dump_stat->destroyedCount));
1519 }
1520
1521 static void
1522 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1523 {
1524     assert(hs);
1525     assert(e);
1526
1527     dump_stat = hs;
1528     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1529     storeAppendPrintf(e, "\nField type distribution\n");
1530     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1531                       "id", "name", "count", "#/header");
1532     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1533     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1534     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1535                       "id", "name", "count", "#/cc_field");
1536     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1537     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1538     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1539                       "id", "name", "count", "#/sc_field");
1540     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1541     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1542     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1543                       "id", "#flds", "count", "%total");
1544     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1545     storeAppendPrintf(e, "\n");
1546     dump_stat = NULL;
1547 }
1548
1549 void
1550 httpHeaderStoreReport(StoreEntry * e)
1551 {
1552     int i;
1553     assert(e);
1554
1555     HttpHeaderStats[0].parsedCount =
1556         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1557     HttpHeaderStats[0].ccParsedCount =
1558         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1559     HttpHeaderStats[0].destroyedCount =
1560         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1561     HttpHeaderStats[0].busyDestroyedCount =
1562         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1563
1564     for (i = 1; i < HttpHeaderStatCount; ++i) {
1565         httpHeaderStatDump(HttpHeaderStats + i, e);
1566     }
1567
1568     /* field stats for all messages */
1569     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1570
1571     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1572                       "id", "name", "#alive", "%err", "%repeat");
1573
1574     // scan heaaderTable and output
1575     for (auto h : WholeEnum<Http::HdrType>()) {
1576         auto stats = headerStatsTable[h];
1577         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1578                           Http::HeaderLookupTable.lookup(h).id,
1579                           Http::HeaderLookupTable.lookup(h).name,
1580                           stats.aliveCount,
1581                           xpercent(stats.errCount, stats.parsCount),
1582                           xpercent(stats.repCount, stats.seenCount));
1583     }
1584
1585     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1586                       HttpHeaderStats[hoRequest].parsedCount,
1587                       HttpHeaderStats[hoReply].parsedCount,
1588                       HttpHeaderStats[0].parsedCount);
1589     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1590 }
1591
1592 int
1593 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1594 {
1595     int result = 0;
1596     const char *pos = NULL;
1597     const char *item;
1598     int ilen;
1599     int mlen = strlen(member);
1600
1601     assert(any_registered_header(id));
1602
1603     String header (getStrOrList(id));
1604
1605     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1606         if (strncasecmp(item, member, mlen) == 0
1607                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1608             result = 1;
1609             break;
1610         }
1611     }
1612
1613     return result;
1614 }
1615
1616 int
1617 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1618 {
1619     int result = 0;
1620     const char *pos = NULL;
1621     const char *item;
1622     int ilen;
1623     int mlen = strlen(member);
1624
1625     assert(name);
1626
1627     String header (getByName(name));
1628
1629     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1630         if (strncasecmp(item, member, mlen) == 0
1631                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1632             result = 1;
1633             break;
1634         }
1635     }
1636
1637     return result;
1638 }
1639
1640 void
1641 HttpHeader::removeHopByHopEntries()
1642 {
1643     removeConnectionHeaderEntries();
1644
1645     const HttpHeaderEntry *e;
1646     HttpHeaderPos pos = HttpHeaderInitPos;
1647     int headers_deleted = 0;
1648     while ((e = getEntry(&pos))) {
1649         Http::HdrType id = e->id;
1650         if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1651             delAt(pos, headers_deleted);
1652             CBIT_CLR(mask, id);
1653         }
1654     }
1655 }
1656
1657 void
1658 HttpHeader::removeConnectionHeaderEntries()
1659 {
1660     if (has(Http::HdrType::CONNECTION)) {
1661         /* anything that matches Connection list member will be deleted */
1662         String strConnection;
1663
1664         (void) getList(Http::HdrType::CONNECTION, &strConnection);
1665         const HttpHeaderEntry *e;
1666         HttpHeaderPos pos = HttpHeaderInitPos;
1667         /*
1668          * think: on-average-best nesting of the two loops (hdrEntry
1669          * and strListItem) @?@
1670          */
1671         /*
1672          * maybe we should delete standard stuff ("keep-alive","close")
1673          * from strConnection first?
1674          */
1675
1676         int headers_deleted = 0;
1677         while ((e = getEntry(&pos))) {
1678             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1679                 delAt(pos, headers_deleted);
1680         }
1681         if (headers_deleted)
1682             refreshMask();
1683     }
1684 }
1685