src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/EnumIterator.h"
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "http/ContentLengthInterpreter.h"
  16 #include "HttpHdrCc.h"
  17 #include "HttpHdrContRange.h"
  18 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
  19 #include "HttpHeader.h"
  20 #include "HttpHeaderFieldInfo.h"
  21 #include "HttpHeaderStat.h"
  22 #include "HttpHeaderTools.h"
  23 #include "MemBuf.h"
  24 #include "mgr/Registration.h"
  25 #include "mime_header.h"
  26 #include "profiler/Profiler.h"
  27 #include "rfc1123.h"
  28 #include "SquidConfig.h"
  29 #include "StatHist.h"
  30 #include "Store.h"
  31 #include "StrList.h"
  32 #include "TimeOrTag.h"
  33 #include "util.h"
  34
  35 #include <algorithm>
  36
  37 /* XXX: the whole set of API managing the entries vector should be rethought
  38  *      after the parse4r-ng effort is complete.
  39  */
  40
  41 /*
  42  * On naming conventions:
  43  *
  44  * HTTP/1.1 defines message-header as
  45  *
  46  * message-header = field-name ":" [ field-value ] CRLF
  47  * field-name     = token
  48  * field-value    = *( field-content | LWS )
  49  *
  50  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  51  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  52  *
  53  * HttpHeader is an object that represents all message-headers in a message.
  54  * HttpHeader does not manage start-line.
  55  *
  56  * HttpHeader is implemented as a collection of header "entries".
  57  * An entry is a (field_id, field_name, field_value) triplet.
  58  */
  59
  60 /*
  61  * local constants and vars
  62  */
  63
  64 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
  65 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
  66
  67 /* request-only headers. Used for cachemgr */
  68 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
  69
  70 /* reply-only headers. Used for cachemgr */
  71 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
  72
  73 /* header accounting */
  74 // NP: keep in sync with enum http_hdr_owner_type
  75 static HttpHeaderStat HttpHeaderStats[] = {
  76     HttpHeaderStat(/*hoNone*/ "all", NULL),
  77 #if USE_HTCP
  78     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
  79 #endif
  80     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
  81     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
  82 #if USE_OPENSSL
  83     /* hoErrorDetail */
  84 #endif
  85     /* hoEnd */
  86 };
  87 static int HttpHeaderStatCount = countof(HttpHeaderStats);
  88
  89 static int HeaderEntryParsedCount = 0;
  90
  91 /*
  92  * forward declarations and local routines
  93  */
  94
  95 class StoreEntry;
  96
  97 // update parse statistics for header id; if error is true also account
  98 // for errors and write to debug log what happened
  99 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
 100 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 101 /** store report about current header usage and other stats */
 102 static void httpHeaderStoreReport(StoreEntry * e);
 103
 104 /*
 105  * Module initialization routines
 106  */
 107
 108 static void
 109 httpHeaderRegisterWithCacheManager(void)
 110 {
 111     Mgr::RegisterAction("http_headers",
 112                         "HTTP Header Statistics",
 113                         httpHeaderStoreReport, 0, 1);
 114 }
 115
 116 void
 117 httpHeaderInitModule(void)
 118 {
 119     /* check that we have enough space for masks */
 120     assert(8 * sizeof(HttpHeaderMask) >= Http::HdrType::enumEnd_);
 121
 122     // masks are needed for stats page still
 123     for (auto h : WholeEnum<Http::HdrType>()) {
 124         if (Http::HeaderLookupTable.lookup(h).request)
 125             CBIT_SET(RequestHeadersMask,h);
 126         if (Http::HeaderLookupTable.lookup(h).reply)
 127             CBIT_SET(ReplyHeadersMask,h);
 128     }
 129
 130     /* header stats initialized by class constructor */
 131     assert(HttpHeaderStatCount == hoReply + 1);
 132
 133     /* init dependent modules */
 134     httpHdrCcInitModule();
 135     httpHdrScInitModule();
 136
 137     httpHeaderRegisterWithCacheManager();
 138 }
 139
 140 /*
 141  * HttpHeader Implementation
 142  */
 143
 144 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
 145 {
 146     httpHeaderMaskInit(&mask, 0);
 147 }
 148
 149 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
 150 {
 151     assert(anOwner > hoNone && anOwner < hoEnd);
 152     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 153     httpHeaderMaskInit(&mask, 0);
 154 }
 155
 156 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
 157 {
 158     httpHeaderMaskInit(&mask, 0);
 159     update(&other); // will update the mask as well
 160 }
 161
 162 HttpHeader::~HttpHeader()
 163 {
 164     clean();
 165 }
 166
 167 HttpHeader &
 168 HttpHeader::operator =(const HttpHeader &other)
 169 {
 170     if (this != &other) {
 171         // we do not really care, but the caller probably does
 172         assert(owner == other.owner);
 173         clean();
 174         update(&other); // will update the mask as well
 175         len = other.len;
 176         conflictingContentLength_ = other.conflictingContentLength_;
 177     }
 178     return *this;
 179 }
 180
 181 void
 182 HttpHeader::clean()
 183 {
 184
 185     assert(owner > hoNone && owner < hoEnd);
 186     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 187
 188     PROF_start(HttpHeaderClean);
 189
 190     if (owner <= hoReply) {
 191         /*
 192          * An unfortunate bug.  The entries array is initialized
 193          * such that count is set to zero.  httpHeaderClean() seems to
 194          * be called both when 'hdr' is created, and destroyed.  Thus,
 195          * we accumulate a large number of zero counts for 'hdr' before
 196          * it is ever used.  Can't think of a good way to fix it, except
 197          * adding a state variable that indicates whether or not 'hdr'
 198          * has been used.  As a hack, just never count zero-sized header
 199          * arrays.
 200          */
 201         if (!entries.empty())
 202             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 203
 204         ++ HttpHeaderStats[owner].destroyedCount;
 205
 206         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 207     } // if (owner <= hoReply)
 208
 209     for (HttpHeaderEntry *e : entries) {
 210         if (e == nullptr)
 211             continue;
 212         if (!Http::any_valid_header(e->id)) {
 213             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 214         } else {
 215             if (owner <= hoReply)
 216                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 217             delete e;
 218         }
 219     }
 220
 221     entries.clear();
 222     httpHeaderMaskInit(&mask, 0);
 223     len = 0;
 224     conflictingContentLength_ = false;
 225     PROF_stop(HttpHeaderClean);
 226 }
 227
 228 /* append entries (also see httpHeaderUpdate) */
 229 void
 230 HttpHeader::append(const HttpHeader * src)
 231 {
 232     assert(src);
 233     assert(src != this);
 234     debugs(55, 7, "appending hdr: " << this << " += " << src);
 235
 236     for (auto e : src->entries) {
 237         if (e)
 238             addEntry(e->clone());
 239     }
 240 }
 241
 242 /// check whether the fresh header has any new/changed updatable fields
 243 bool
 244 HttpHeader::needUpdate(HttpHeader const *fresh) const
 245 {
 246     for (const auto e: fresh->entries) {
 247         if (!e || skipUpdateHeader(e->id))
 248             continue;
 249         String value;
 250         const char *name = e->name.termedBuf();
 251         if (!getByNameIfPresent(name, strlen(name), value) ||
 252                 (value != fresh->getByName(name)))
 253             return true;
 254     }
 255     return false;
 256 }
 257
 258 void
 259 HttpHeader::updateWarnings()
 260 {
 261     int count = 0;
 262     HttpHeaderPos pos = HttpHeaderInitPos;
 263
 264     // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
 265     while (HttpHeaderEntry *e = getEntry(&pos)) {
 266         if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
 267             delAt(pos, count);
 268     }
 269 }
 270
 271 bool
 272 HttpHeader::skipUpdateHeader(const Http::HdrType id) const
 273 {
 274     // RFC 7234, section 4.3.4: use fields other from Warning for update
 275     return id == Http::HdrType::WARNING;
 276 }
 277
 278 bool
 279 HttpHeader::update(HttpHeader const *fresh)
 280 {
 281     assert(fresh);
 282     assert(this != fresh);
 283
 284     // Optimization: Finding whether a header field changed is expensive
 285     // and probably not worth it except for collapsed revalidation needs.
 286     if (Config.onoff.collapsed_forwarding && !needUpdate(fresh))
 287         return false;
 288
 289     updateWarnings();
 290
 291     const HttpHeaderEntry *e;
 292     HttpHeaderPos pos = HttpHeaderInitPos;
 293
 294     while ((e = fresh->getEntry(&pos))) {
 295         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 296
 297         if (skipUpdateHeader(e->id))
 298             continue;
 299
 300         if (e->id != Http::HdrType::OTHER)
 301             delById(e->id);
 302         else
 303             delByName(e->name.termedBuf());
 304     }
 305
 306     pos = HttpHeaderInitPos;
 307     while ((e = fresh->getEntry(&pos))) {
 308         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 309
 310         if (skipUpdateHeader(e->id))
 311             continue;
 312
 313         debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
 314
 315         addEntry(e->clone());
 316     }
 317     return true;
 318 }
 319
 320 bool
 321 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
 322 {
 323     /*
 324      * parse_start points to the first line of HTTP message *headers*,
 325      * not including the request or status lines
 326      */
 327     const size_t end = headersEnd(*parse_start, l);
 328
 329     if (end) {
 330         *blk_start = *parse_start;
 331         *blk_end = *parse_start + end - 1;
 332         assert(**blk_end == '\n');
 333         // Point blk_end to the first character after the last header field.
 334         // In other words, blk_end should point to the CR?LF header terminator.
 335         if (end > 1 && *(*blk_end - 1) == '\r')
 336             --(*blk_end);
 337         *parse_start += end;
 338     }
 339     return end;
 340 }
 341
 342 int
 343 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz)
 344 {
 345     const char *parse_start = buf;
 346     const char *blk_start, *blk_end;
 347     hdr_sz = 0;
 348
 349     if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
 350         // XXX: do not parse non-isolated headers even if the connection is closed.
 351         // Treat unterminated headers as "partial headers" framing errors.
 352         if (!atEnd)
 353             return 0;
 354         blk_start = parse_start;
 355         blk_end = blk_start + strlen(blk_start);
 356     }
 357
 358     if (parse(blk_start, blk_end - blk_start)) {
 359         hdr_sz = parse_start - buf;
 360         return 1;
 361     }
 362     return -1;
 363 }
 364
 365 int
 366 HttpHeader::parse(const char *header_start, size_t hdrLen)
 367 {
 368     const char *field_ptr = header_start;
 369     const char *header_end = header_start + hdrLen; // XXX: remove
 370     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 371
 372     PROF_start(HttpHeaderParse);
 373
 374     assert(header_start && header_end);
 375     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 376     ++ HttpHeaderStats[owner].parsedCount;
 377
 378     char *nulpos;
 379     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 380         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 381                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 382         PROF_stop(HttpHeaderParse);
 383         clean();
 384         return 0;
 385     }
 386
 387     Http::ContentLengthInterpreter clen(warnOnError);
 388     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 389      * continuation lines start with a (single) space or tab */
 390     while (field_ptr < header_end) {
 391         const char *field_start = field_ptr;
 392         const char *field_end;
 393
 394         do {
 395             const char *this_line = field_ptr;
 396             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 397
 398             if (!field_ptr) {
 399                 // missing <LF>
 400                 PROF_stop(HttpHeaderParse);
 401                 clean();
 402                 return 0;
 403             }
 404
 405             field_end = field_ptr;
 406
 407             ++field_ptr;    /* Move to next line */
 408
 409             if (field_end > this_line && field_end[-1] == '\r') {
 410                 --field_end;    /* Ignore CR LF */
 411
 412                 if (owner == hoRequest && field_end > this_line) {
 413                     bool cr_only = true;
 414                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 415                         if (*p != '\r')
 416                             cr_only = false;
 417                     }
 418                     if (cr_only) {
 419                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 420                                "header field to prevent request smuggling attacks: {" <<
 421                                getStringPrefix(header_start, hdrLen) << "}");
 422                         PROF_stop(HttpHeaderParse);
 423                         clean();
 424                         return 0;
 425                     }
 426                 }
 427             }
 428
 429             /* Barf on stray CR characters */
 430             if (memchr(this_line, '\r', field_end - this_line)) {
 431                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 432                        getStringPrefix(field_start, field_end-field_start) << "}");
 433
 434                 if (Config.onoff.relaxed_header_parser) {
 435                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 436
 437                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 438                         *p = ' ';
 439                         ++p;
 440                     }
 441                 } else {
 442                     PROF_stop(HttpHeaderParse);
 443                     clean();
 444                     return 0;
 445                 }
 446             }
 447
 448             if (this_line + 1 == field_end && this_line > field_start) {
 449                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 450                        getStringPrefix(header_start, hdrLen) << "}");
 451                 PROF_stop(HttpHeaderParse);
 452                 clean();
 453                 return 0;
 454             }
 455         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 456
 457         if (field_start == field_end) {
 458             if (field_ptr < header_end) {
 459                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 460                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 461                 PROF_stop(HttpHeaderParse);
 462                 clean();
 463                 return 0;
 464             }
 465
 466             break;      /* terminating blank line */
 467         }
 468
 469         HttpHeaderEntry *e;
 470         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 471             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 472                    getStringPrefix(field_start, field_end-field_start) << "}");
 473             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 474
 475             if (Config.onoff.relaxed_header_parser)
 476                 continue;
 477
 478             PROF_stop(HttpHeaderParse);
 479             clean();
 480             return 0;
 481         }
 482
 483         if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
 484             delete e;
 485
 486             if (Config.onoff.relaxed_header_parser)
 487                 continue; // clen has printed any necessary warnings
 488
 489             PROF_stop(HttpHeaderParse);
 490             clean();
 491             return 0;
 492         }
 493
 494         if (e->id == Http::HdrType::OTHER && stringHasWhitespace(e->name.termedBuf())) {
 495             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 496                    getStringPrefix(field_start, field_end-field_start) << "}");
 497
 498             if (!Config.onoff.relaxed_header_parser) {
 499                 delete e;
 500                 PROF_stop(HttpHeaderParse);
 501                 clean();
 502                 return 0;
 503             }
 504         }
 505
 506         addEntry(e);
 507     }
 508
 509     if (clen.headerWideProblem) {
 510         debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
 511                " Content-Length field values in" <<
 512                Raw("header", header_start, hdrLen));
 513     }
 514
 515     if (chunked()) {
 516         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 517         // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
 518         delById(Http::HdrType::CONTENT_LENGTH);
 519         // and clen state becomes irrelevant
 520     } else if (clen.sawBad) {
 521         // ensure our callers do not accidentally see bad Content-Length values
 522         delById(Http::HdrType::CONTENT_LENGTH);
 523         conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
 524     } else if (clen.needsSanitizing) {
 525         // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
 526         // ensure our callers see a clean Content-Length value or none at all
 527         delById(Http::HdrType::CONTENT_LENGTH);
 528         if (clen.sawGood) {
 529             putInt64(Http::HdrType::CONTENT_LENGTH, clen.value);
 530             debugs(55, 5, "sanitized Content-Length to be " << clen.value);
 531         }
 532     }
 533
 534     PROF_stop(HttpHeaderParse);
 535     return 1;           /* even if no fields where found, it is a valid header */
 536 }
 537
 538 /* packs all the entries using supplied packer */
 539 void
 540 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 541 {
 542     HttpHeaderPos pos = HttpHeaderInitPos;
 543     const HttpHeaderEntry *e;
 544     assert(p);
 545     debugs(55, 7, this << " into " << p <<
 546            (mask_sensitive_info ? " while masking" : ""));
 547     /* pack all entries one by one */
 548     while ((e = getEntry(&pos))) {
 549         if (!mask_sensitive_info) {
 550             e->packInto(p);
 551             continue;
 552         }
 553
 554         bool maskThisEntry = false;
 555         switch (e->id) {
 556         case Http::HdrType::AUTHORIZATION:
 557         case Http::HdrType::PROXY_AUTHORIZATION:
 558             maskThisEntry = true;
 559             break;
 560
 561         case Http::HdrType::FTP_ARGUMENTS:
 562             if (const HttpHeaderEntry *cmd = findEntry(Http::HdrType::FTP_COMMAND))
 563                 maskThisEntry = (cmd->value == "PASS");
 564             break;
 565
 566         default:
 567             break;
 568         }
 569         if (maskThisEntry) {
 570             p->append(e->name.rawBuf(), e->name.size());
 571             p->append(": ** NOT DISPLAYED **\r\n", 23);
 572         } else {
 573             e->packInto(p);
 574         }
 575
 576     }
 577     /* Pack in the "special" entries */
 578
 579     /* Cache-Control */
 580 }
 581
 582 /* returns next valid entry */
 583 HttpHeaderEntry *
 584 HttpHeader::getEntry(HttpHeaderPos * pos) const
 585 {
 586     assert(pos);
 587     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 588
 589     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 590         if (entries[*pos])
 591             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 592     }
 593
 594     return NULL;
 595 }
 596
 597 /*
 598  * returns a pointer to a specified entry if any
 599  * note that we return one entry so it does not make much sense to ask for
 600  * "list" headers
 601  */
 602 HttpHeaderEntry *
 603 HttpHeader::findEntry(Http::HdrType id) const
 604 {
 605     assert(any_registered_header(id));
 606     assert(!Http::HeaderLookupTable.lookup(id).list);
 607
 608     /* check mask first */
 609
 610     if (!CBIT_TEST(mask, id))
 611         return NULL;
 612
 613     /* looks like we must have it, do linear search */
 614     for (auto e : entries) {
 615         if (e && e->id == id)
 616             return e;
 617     }
 618
 619     /* hm.. we thought it was there, but it was not found */
 620     assert(false);
 621     return nullptr;        /* not reached */
 622 }
 623
 624 /*
 625  * same as httpHeaderFindEntry
 626  */
 627 HttpHeaderEntry *
 628 HttpHeader::findLastEntry(Http::HdrType id) const
 629 {
 630     assert(any_registered_header(id));
 631     assert(!Http::HeaderLookupTable.lookup(id).list);
 632
 633     /* check mask first */
 634     if (!CBIT_TEST(mask, id))
 635         return NULL;
 636
 637     for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
 638         if (*e && (*e)->id == id)
 639             return *e;
 640     }
 641
 642     /* hm.. we thought it was there, but it was not found */
 643     assert(false);
 644     return nullptr; /* not reached */
 645 }
 646
 647 /*
 648  * deletes all fields with a given name if any, returns #fields deleted;
 649  */
 650 int
 651 HttpHeader::delByName(const char *name)
 652 {
 653     int count = 0;
 654     HttpHeaderPos pos = HttpHeaderInitPos;
 655     HttpHeaderEntry *e;
 656     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 657     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 658
 659     while ((e = getEntry(&pos))) {
 660         if (!e->name.caseCmp(name))
 661             delAt(pos, count);
 662         else
 663             CBIT_SET(mask, e->id);
 664     }
 665
 666     return count;
 667 }
 668
 669 /* deletes all entries with a given id, returns the #entries deleted */
 670 int
 671 HttpHeader::delById(Http::HdrType id)
 672 {
 673     debugs(55, 8, this << " del-by-id " << id);
 674     assert(any_registered_header(id));
 675
 676     if (!CBIT_TEST(mask, id))
 677         return 0;
 678
 679     int count = 0;
 680
 681     HttpHeaderPos pos = HttpHeaderInitPos;
 682     while (HttpHeaderEntry *e = getEntry(&pos)) {
 683         if (e->id == id)
 684             delAt(pos, count); // deletes e
 685     }
 686
 687     CBIT_CLR(mask, id);
 688     assert(count);
 689     return count;
 690 }
 691
 692 /*
 693  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 694  * possible to iterate(search) and delete fields at the same time
 695  * NOTE: Does not update the header mask. Caller must follow up with
 696  * a call to refreshMask() if headers_deleted was incremented.
 697  */
 698 void
 699 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 700 {
 701     HttpHeaderEntry *e;
 702     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 703     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 704     entries[pos] = NULL;
 705     /* decrement header length, allow for ": " and crlf */
 706     len -= e->name.size() + 2 + e->value.size() + 2;
 707     assert(len >= 0);
 708     delete e;
 709     ++headers_deleted;
 710 }
 711
 712 /*
 713  * Compacts the header storage
 714  */
 715 void
 716 HttpHeader::compact()
 717 {
 718     // TODO: optimize removal, or possibly make it so that's not needed.
 719     entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
 720                    entries.end());
 721 }
 722
 723 /*
 724  * Refreshes the header mask. Required after delAt() calls.
 725  */
 726 void
 727 HttpHeader::refreshMask()
 728 {
 729     httpHeaderMaskInit(&mask, 0);
 730     debugs(55, 7, "refreshing the mask in hdr " << this);
 731     for (auto e : entries) {
 732         if (e)
 733             CBIT_SET(mask, e->id);
 734     }
 735 }
 736
 737 /* appends an entry;
 738  * does not call e->clone() so one should not reuse "*e"
 739  */
 740 void
 741 HttpHeader::addEntry(HttpHeaderEntry * e)
 742 {
 743     assert(e);
 744     assert(any_HdrType_enum_value(e->id));
 745     assert(e->name.size());
 746
 747     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 748
 749     if (e->id != Http::HdrType::BAD_HDR) {
 750         if (CBIT_TEST(mask, e->id)) {
 751             ++ headerStatsTable[e->id].repCount;
 752         } else {
 753             CBIT_SET(mask, e->id);
 754         }
 755     }
 756
 757     entries.push_back(e);
 758
 759     /* increment header length, allow for ": " and crlf */
 760     len += e->name.size() + 2 + e->value.size() + 2;
 761 }
 762
 763 /* inserts an entry;
 764  * does not call e->clone() so one should not reuse "*e"
 765  */
 766 void
 767 HttpHeader::insertEntry(HttpHeaderEntry * e)
 768 {
 769     assert(e);
 770     assert(any_valid_header(e->id));
 771
 772     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 773
 774     // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
 775     if (CBIT_TEST(mask, e->id)) {
 776         ++ headerStatsTable[e->id].repCount;
 777     } else {
 778         CBIT_SET(mask, e->id);
 779     }
 780
 781     entries.insert(entries.begin(),e);
 782
 783     /* increment header length, allow for ": " and crlf */
 784     len += e->name.size() + 2 + e->value.size() + 2;
 785 }
 786
 787 bool
 788 HttpHeader::getList(Http::HdrType id, String *s) const
 789 {
 790     debugs(55, 9, this << " joining for id " << id);
 791     /* only fields from ListHeaders array can be "listed" */
 792     assert(Http::HeaderLookupTable.lookup(id).list);
 793
 794     if (!CBIT_TEST(mask, id))
 795         return false;
 796
 797     for (auto e: entries) {
 798         if (e && e->id == id)
 799             strListAdd(s, e->value.termedBuf(), ',');
 800     }
 801
 802     /*
 803      * note: we might get an empty (size==0) string if there was an "empty"
 804      * header. This results in an empty length String, which may have a NULL
 805      * buffer.
 806      */
 807     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 808     if (!s->size())
 809         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 810     else
 811         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 812
 813     return true;
 814 }
 815
 816 /* return a list of entries with the same id separated by ',' and ws */
 817 String
 818 HttpHeader::getList(Http::HdrType id) const
 819 {
 820     HttpHeaderEntry *e;
 821     HttpHeaderPos pos = HttpHeaderInitPos;
 822     debugs(55, 9, this << "joining for id " << id);
 823     /* only fields from ListHeaders array can be "listed" */
 824     assert(Http::HeaderLookupTable.lookup(id).list);
 825
 826     if (!CBIT_TEST(mask, id))
 827         return String();
 828
 829     String s;
 830
 831     while ((e = getEntry(&pos))) {
 832         if (e->id == id)
 833             strListAdd(&s, e->value.termedBuf(), ',');
 834     }
 835
 836     /*
 837      * note: we might get an empty (size==0) string if there was an "empty"
 838      * header. This results in an empty length String, which may have a NULL
 839      * buffer.
 840      */
 841     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 842     if (!s.size())
 843         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 844     else
 845         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 846
 847     return s;
 848 }
 849
 850 /* return a string or list of entries with the same id separated by ',' and ws */
 851 String
 852 HttpHeader::getStrOrList(Http::HdrType id) const
 853 {
 854     HttpHeaderEntry *e;
 855
 856     if (Http::HeaderLookupTable.lookup(id).list)
 857         return getList(id);
 858
 859     if ((e = findEntry(id)))
 860         return e->value;
 861
 862     return String();
 863 }
 864
 865 /*
 866  * Returns the value of the specified header and/or an undefined String.
 867  */
 868 String
 869 HttpHeader::getByName(const char *name) const
 870 {
 871     String result;
 872     // ignore presence: return undefined string if an empty header is present
 873     (void)getByNameIfPresent(name, strlen(name), result);
 874     return result;
 875 }
 876
 877 String
 878 HttpHeader::getByName(const SBuf &name) const
 879 {
 880     String result;
 881     // ignore presence: return undefined string if an empty header is present
 882     (void)getByNameIfPresent(name, result);
 883     return result;
 884 }
 885
 886 String
 887 HttpHeader::getById(Http::HdrType id) const
 888 {
 889     String result;
 890     (void)getByIdIfPresent(id,result);
 891     return result;
 892 }
 893
 894 bool
 895 HttpHeader::getByNameIfPresent(const SBuf &s, String &result) const
 896 {
 897     return getByNameIfPresent(s.rawContent(), s.length(), result);
 898 }
 899
 900 bool
 901 HttpHeader::getByIdIfPresent(Http::HdrType id, String &result) const
 902 {
 903     if (id == Http::HdrType::BAD_HDR)
 904         return false;
 905     if (!has(id))
 906         return false;
 907     result = getStrOrList(id);
 908     return true;
 909 }
 910
 911 bool
 912 HttpHeader::getByNameIfPresent(const char *name, int namelen, String &result) const
 913 {
 914     Http::HdrType id;
 915     HttpHeaderPos pos = HttpHeaderInitPos;
 916     HttpHeaderEntry *e;
 917
 918     assert(name);
 919
 920     /* First try the quick path */
 921     id = Http::HeaderLookupTable.lookup(name,namelen).id;
 922
 923     if (id != Http::HdrType::BAD_HDR) {
 924         if (getByIdIfPresent(id, result))
 925             return true;
 926     }
 927
 928     /* Sorry, an unknown header name. Do linear search */
 929     bool found = false;
 930     while ((e = getEntry(&pos))) {
 931         if (e->id == Http::HdrType::OTHER && e->name.size() == static_cast<String::size_type>(namelen) && e->name.caseCmp(name, namelen) == 0) {
 932             found = true;
 933             strListAdd(&result, e->value.termedBuf(), ',');
 934         }
 935     }
 936
 937     return found;
 938 }
 939
 940 /*
 941  * Returns a the value of the specified list member, if any.
 942  */
 943 String
 944 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
 945 {
 946     String header;
 947     const char *pos = NULL;
 948     const char *item;
 949     int ilen;
 950     int mlen = strlen(member);
 951
 952     assert(name);
 953
 954     header = getByName(name);
 955
 956     String result;
 957
 958     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 959         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 960             result.append(item + mlen + 1, ilen - mlen - 1);
 961             break;
 962         }
 963     }
 964
 965     return result;
 966 }
 967
 968 /*
 969  * returns a the value of the specified list member, if any.
 970  */
 971 String
 972 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
 973 {
 974     String header;
 975     const char *pos = NULL;
 976     const char *item;
 977     int ilen;
 978     int mlen = strlen(member);
 979
 980     assert(any_registered_header(id));
 981
 982     header = getStrOrList(id);
 983     String result;
 984
 985     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 986         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 987             result.append(item + mlen + 1, ilen - mlen - 1);
 988             break;
 989         }
 990     }
 991
 992     header.clean();
 993     return result;
 994 }
 995
 996 /* test if a field is present */
 997 int
 998 HttpHeader::has(Http::HdrType id) const
 999 {
1000     assert(any_registered_header(id));
1001     debugs(55, 9, this << " lookup for " << id);
1002     return CBIT_TEST(mask, id);
1003 }
1004
1005 void
1006 HttpHeader::putInt(Http::HdrType id, int number)
1007 {
1008     assert(any_registered_header(id));
1009     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1010     assert(number >= 0);
1011     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1012 }
1013
1014 void
1015 HttpHeader::putInt64(Http::HdrType id, int64_t number)
1016 {
1017     assert(any_registered_header(id));
1018     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1019     assert(number >= 0);
1020     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1021 }
1022
1023 void
1024 HttpHeader::putTime(Http::HdrType id, time_t htime)
1025 {
1026     assert(any_registered_header(id));
1027     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1028     assert(htime >= 0);
1029     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1030 }
1031
1032 void
1033 HttpHeader::putStr(Http::HdrType id, const char *str)
1034 {
1035     assert(any_registered_header(id));
1036     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1037     assert(str);
1038     addEntry(new HttpHeaderEntry(id, NULL, str));
1039 }
1040
1041 void
1042 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1043 {
1044     assert(auth_scheme && realm);
1045     httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1046 }
1047
1048 void
1049 HttpHeader::putCc(const HttpHdrCc * cc)
1050 {
1051     assert(cc);
1052     /* remove old directives if any */
1053     delById(Http::HdrType::CACHE_CONTROL);
1054     /* pack into mb */
1055     MemBuf mb;
1056     mb.init();
1057     cc->packInto(&mb);
1058     /* put */
1059     addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL, NULL, mb.buf));
1060     /* cleanup */
1061     mb.clean();
1062 }
1063
1064 void
1065 HttpHeader::putContRange(const HttpHdrContRange * cr)
1066 {
1067     assert(cr);
1068     /* remove old directives if any */
1069     delById(Http::HdrType::CONTENT_RANGE);
1070     /* pack into mb */
1071     MemBuf mb;
1072     mb.init();
1073     httpHdrContRangePackInto(cr, &mb);
1074     /* put */
1075     addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE, NULL, mb.buf));
1076     /* cleanup */
1077     mb.clean();
1078 }
1079
1080 void
1081 HttpHeader::putRange(const HttpHdrRange * range)
1082 {
1083     assert(range);
1084     /* remove old directives if any */
1085     delById(Http::HdrType::RANGE);
1086     /* pack into mb */
1087     MemBuf mb;
1088     mb.init();
1089     range->packInto(&mb);
1090     /* put */
1091     addEntry(new HttpHeaderEntry(Http::HdrType::RANGE, NULL, mb.buf));
1092     /* cleanup */
1093     mb.clean();
1094 }
1095
1096 void
1097 HttpHeader::putSc(HttpHdrSc *sc)
1098 {
1099     assert(sc);
1100     /* remove old directives if any */
1101     delById(Http::HdrType::SURROGATE_CONTROL);
1102     /* pack into mb */
1103     MemBuf mb;
1104     mb.init();
1105     sc->packInto(&mb);
1106     /* put */
1107     addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL, NULL, mb.buf));
1108     /* cleanup */
1109     mb.clean();
1110 }
1111
1112 void
1113 HttpHeader::putWarning(const int code, const char *const text)
1114 {
1115     char buf[512];
1116     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1117     putStr(Http::HdrType::WARNING, buf);
1118 }
1119
1120 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1121 void
1122 HttpHeader::putExt(const char *name, const char *value)
1123 {
1124     assert(name && value);
1125     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1126     addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, name, value));
1127 }
1128
1129 int
1130 HttpHeader::getInt(Http::HdrType id) const
1131 {
1132     assert(any_registered_header(id));
1133     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1134     HttpHeaderEntry *e;
1135
1136     if ((e = findEntry(id)))
1137         return e->getInt();
1138
1139     return -1;
1140 }
1141
1142 int64_t
1143 HttpHeader::getInt64(Http::HdrType id) const
1144 {
1145     assert(any_registered_header(id));
1146     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1147     HttpHeaderEntry *e;
1148
1149     if ((e = findEntry(id)))
1150         return e->getInt64();
1151
1152     return -1;
1153 }
1154
1155 time_t
1156 HttpHeader::getTime(Http::HdrType id) const
1157 {
1158     HttpHeaderEntry *e;
1159     time_t value = -1;
1160     assert(any_registered_header(id));
1161     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1162
1163     if ((e = findEntry(id))) {
1164         value = parse_rfc1123(e->value.termedBuf());
1165         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1166     }
1167
1168     return value;
1169 }
1170
1171 /* sync with httpHeaderGetLastStr */
1172 const char *
1173 HttpHeader::getStr(Http::HdrType id) const
1174 {
1175     HttpHeaderEntry *e;
1176     assert(any_registered_header(id));
1177     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1178
1179     if ((e = findEntry(id))) {
1180         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1181         return e->value.termedBuf();
1182     }
1183
1184     return NULL;
1185 }
1186
1187 /* unusual */
1188 const char *
1189 HttpHeader::getLastStr(Http::HdrType id) const
1190 {
1191     HttpHeaderEntry *e;
1192     assert(any_registered_header(id));
1193     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1194
1195     if ((e = findLastEntry(id))) {
1196         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1197         return e->value.termedBuf();
1198     }
1199
1200     return NULL;
1201 }
1202
1203 HttpHdrCc *
1204 HttpHeader::getCc() const
1205 {
1206     if (!CBIT_TEST(mask, Http::HdrType::CACHE_CONTROL))
1207         return NULL;
1208     PROF_start(HttpHeader_getCc);
1209
1210     String s;
1211     getList(Http::HdrType::CACHE_CONTROL, &s);
1212
1213     HttpHdrCc *cc=new HttpHdrCc();
1214
1215     if (!cc->parse(s)) {
1216         delete cc;
1217         cc = NULL;
1218     }
1219
1220     ++ HttpHeaderStats[owner].ccParsedCount;
1221
1222     if (cc)
1223         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1224
1225     httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL, s, !cc);
1226
1227     PROF_stop(HttpHeader_getCc);
1228
1229     return cc;
1230 }
1231
1232 HttpHdrRange *
1233 HttpHeader::getRange() const
1234 {
1235     HttpHdrRange *r = NULL;
1236     HttpHeaderEntry *e;
1237     /* some clients will send "Request-Range" _and_ *matching* "Range"
1238      * who knows, some clients might send Request-Range only;
1239      * this "if" should work correctly in both cases;
1240      * hopefully no clients send mismatched headers! */
1241
1242     if ((e = findEntry(Http::HdrType::RANGE)) ||
1243             (e = findEntry(Http::HdrType::REQUEST_RANGE))) {
1244         r = HttpHdrRange::ParseCreate(&e->value);
1245         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1246     }
1247
1248     return r;
1249 }
1250
1251 HttpHdrSc *
1252 HttpHeader::getSc() const
1253 {
1254     if (!CBIT_TEST(mask, Http::HdrType::SURROGATE_CONTROL))
1255         return NULL;
1256
1257     String s;
1258
1259     (void) getList(Http::HdrType::SURROGATE_CONTROL, &s);
1260
1261     HttpHdrSc *sc = httpHdrScParseCreate(s);
1262
1263     ++ HttpHeaderStats[owner].ccParsedCount;
1264
1265     if (sc)
1266         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1267
1268     httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL, s, !sc);
1269
1270     return sc;
1271 }
1272
1273 HttpHdrContRange *
1274 HttpHeader::getContRange() const
1275 {
1276     HttpHdrContRange *cr = NULL;
1277     HttpHeaderEntry *e;
1278
1279     if ((e = findEntry(Http::HdrType::CONTENT_RANGE))) {
1280         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1281         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1282     }
1283
1284     return cr;
1285 }
1286
1287 const char *
1288 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1289 {
1290     const char *field;
1291     int l;
1292     assert(auth_scheme);
1293     field = getStr(id);
1294
1295     if (!field)         /* no authorization field */
1296         return NULL;
1297
1298     l = strlen(auth_scheme);
1299
1300     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1301         return NULL;
1302
1303     field += l;
1304
1305     if (!xisspace(*field))  /* wrong scheme */
1306         return NULL;
1307
1308     /* skip white space */
1309     for (; field && xisspace(*field); ++field);
1310
1311     if (!*field)        /* no authorization cookie */
1312         return NULL;
1313
1314     static char decodedAuthToken[8192];
1315     struct base64_decode_ctx ctx;
1316     base64_decode_init(&ctx);
1317     size_t decodedLen = 0;
1318     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1319             !base64_decode_final(&ctx)) {
1320         return NULL;
1321     }
1322     decodedAuthToken[decodedLen] = '\0';
1323     return decodedAuthToken;
1324 }
1325
1326 ETag
1327 HttpHeader::getETag(Http::HdrType id) const
1328 {
1329     ETag etag = {NULL, -1};
1330     HttpHeaderEntry *e;
1331     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag);     /* must be of an appropriate type */
1332
1333     if ((e = findEntry(id)))
1334         etagParseInit(&etag, e->value.termedBuf());
1335
1336     return etag;
1337 }
1338
1339 TimeOrTag
1340 HttpHeader::getTimeOrTag(Http::HdrType id) const
1341 {
1342     TimeOrTag tot;
1343     HttpHeaderEntry *e;
1344     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag);    /* must be of an appropriate type */
1345     memset(&tot, 0, sizeof(tot));
1346
1347     if ((e = findEntry(id))) {
1348         const char *str = e->value.termedBuf();
1349         /* try as an ETag */
1350
1351         if (etagParseInit(&tot.tag, str)) {
1352             tot.valid = tot.tag.str != NULL;
1353             tot.time = -1;
1354         } else {
1355             /* or maybe it is time? */
1356             tot.time = parse_rfc1123(str);
1357             tot.valid = tot.time >= 0;
1358             tot.tag.str = NULL;
1359         }
1360     }
1361
1362     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1363     return tot;
1364 }
1365
1366 /*
1367  * HttpHeaderEntry
1368  */
1369
1370 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const char *aName, const char *aValue)
1371 {
1372     assert(any_HdrType_enum_value(anId));
1373     id = anId;
1374
1375     if (id != Http::HdrType::OTHER)
1376         name = Http::HeaderLookupTable.lookup(id).name;
1377     else
1378         name = aName;
1379
1380     value = aValue;
1381
1382     if (id != Http::HdrType::BAD_HDR)
1383         ++ headerStatsTable[id].aliveCount;
1384
1385     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1386 }
1387
1388 HttpHeaderEntry::~HttpHeaderEntry()
1389 {
1390     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1391
1392     if (id != Http::HdrType::BAD_HDR) {
1393         assert(headerStatsTable[id].aliveCount);
1394         -- headerStatsTable[id].aliveCount;
1395         id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1396     }
1397
1398 }
1399
1400 /* parses and inits header entry, returns true/false */
1401 HttpHeaderEntry *
1402 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1403 {
1404     /* note: name_start == field_start */
1405     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1406     int name_len = name_end ? name_end - field_start :0;
1407     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1408     /* note: value_end == field_end */
1409
1410     ++ HeaderEntryParsedCount;
1411
1412     /* do we have a valid field name within this field? */
1413
1414     if (!name_len || name_end > field_end)
1415         return NULL;
1416
1417     if (name_len > 65534) {
1418         /* String must be LESS THAN 64K and it adds a terminating NULL */
1419         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1420         return NULL;
1421     }
1422
1423     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1424         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1425                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1426
1427         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1428             --name_len;
1429
1430         if (!name_len)
1431             return NULL;
1432     }
1433
1434     /* now we know we can parse it */
1435
1436     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1437
1438     /* is it a "known" field? */
1439     Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1440     debugs(55, 9, "got hdr-id=" << id);
1441
1442     String name;
1443
1444     String value;
1445
1446     if (id == Http::HdrType::BAD_HDR)
1447         id = Http::HdrType::OTHER;
1448
1449     /* set field name */
1450     if (id == Http::HdrType::OTHER)
1451         name.limitInit(field_start, name_len);
1452     else
1453         name = Http::HeaderLookupTable.lookup(id).name;
1454
1455     /* trim field value */
1456     while (value_start < field_end && xisspace(*value_start))
1457         ++value_start;
1458
1459     while (value_start < field_end && xisspace(field_end[-1]))
1460         --field_end;
1461
1462     if (field_end - value_start > 65534) {
1463         /* String must be LESS THAN 64K and it adds a terminating NULL */
1464         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1465
1466         if (id == Http::HdrType::OTHER)
1467             name.clean();
1468
1469         return NULL;
1470     }
1471
1472     /* set field value */
1473     value.limitInit(value_start, field_end - value_start);
1474
1475     if (id != Http::HdrType::BAD_HDR)
1476         ++ headerStatsTable[id].seenCount;
1477
1478     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1479
1480     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1481 }
1482
1483 HttpHeaderEntry *
1484 HttpHeaderEntry::clone() const
1485 {
1486     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1487 }
1488
1489 void
1490 HttpHeaderEntry::packInto(Packable * p) const
1491 {
1492     assert(p);
1493     p->append(name.rawBuf(), name.size());
1494     p->append(": ", 2);
1495     p->append(value.rawBuf(), value.size());
1496     p->append("\r\n", 2);
1497 }
1498
1499 int
1500 HttpHeaderEntry::getInt() const
1501 {
1502     int val = -1;
1503     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1504     httpHeaderNoteParsedEntry(id, value, ok == 0);
1505     /* XXX: Should we check ok - ie
1506      * return ok ? -1 : value;
1507      */
1508     return val;
1509 }
1510
1511 int64_t
1512 HttpHeaderEntry::getInt64() const
1513 {
1514     int64_t val = -1;
1515     const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1516     httpHeaderNoteParsedEntry(id, value, !ok);
1517     return val; // remains -1 if !ok (XXX: bad method API)
1518 }
1519
1520 static void
1521 httpHeaderNoteParsedEntry(Http::HdrType id, String const &context, bool error)
1522 {
1523     if (id != Http::HdrType::BAD_HDR)
1524         ++ headerStatsTable[id].parsCount;
1525
1526     if (error) {
1527         if (id != Http::HdrType::BAD_HDR)
1528             ++ headerStatsTable[id].errCount;
1529         debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1530     }
1531 }
1532
1533 /*
1534  * Reports
1535  */
1536
1537 /* tmp variable used to pass stat info to dumpers */
1538 extern const HttpHeaderStat *dump_stat;     /* argh! */
1539 const HttpHeaderStat *dump_stat = NULL;
1540
1541 void
1542 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1543 {
1544     const int id = static_cast<int>(val);
1545     const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1546     const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1547     int visible = count > 0;
1548     /* for entries with zero count, list only those that belong to current type of message */
1549
1550     if (!visible && valid_id && dump_stat->owner_mask)
1551         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1552
1553     if (visible)
1554         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1555                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1556 }
1557
1558 static void
1559 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1560 {
1561     if (count)
1562         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1563                           idx, (int) val, count,
1564                           xpercent(count, dump_stat->destroyedCount));
1565 }
1566
1567 static void
1568 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1569 {
1570     assert(hs);
1571     assert(e);
1572
1573     dump_stat = hs;
1574     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1575     storeAppendPrintf(e, "\nField type distribution\n");
1576     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1577                       "id", "name", "count", "#/header");
1578     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1579     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1580     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1581                       "id", "name", "count", "#/cc_field");
1582     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1583     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1584     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1585                       "id", "name", "count", "#/sc_field");
1586     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1587     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1588     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1589                       "id", "#flds", "count", "%total");
1590     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1591     storeAppendPrintf(e, "\n");
1592     dump_stat = NULL;
1593 }
1594
1595 void
1596 httpHeaderStoreReport(StoreEntry * e)
1597 {
1598     int i;
1599     assert(e);
1600
1601     HttpHeaderStats[0].parsedCount =
1602         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1603     HttpHeaderStats[0].ccParsedCount =
1604         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1605     HttpHeaderStats[0].destroyedCount =
1606         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1607     HttpHeaderStats[0].busyDestroyedCount =
1608         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1609
1610     for (i = 1; i < HttpHeaderStatCount; ++i) {
1611         httpHeaderStatDump(HttpHeaderStats + i, e);
1612     }
1613
1614     /* field stats for all messages */
1615     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1616
1617     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1618                       "id", "name", "#alive", "%err", "%repeat");
1619
1620     // scan heaaderTable and output
1621     for (auto h : WholeEnum<Http::HdrType>()) {
1622         auto stats = headerStatsTable[h];
1623         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1624                           Http::HeaderLookupTable.lookup(h).id,
1625                           Http::HeaderLookupTable.lookup(h).name,
1626                           stats.aliveCount,
1627                           xpercent(stats.errCount, stats.parsCount),
1628                           xpercent(stats.repCount, stats.seenCount));
1629     }
1630
1631     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1632                       HttpHeaderStats[hoRequest].parsedCount,
1633                       HttpHeaderStats[hoReply].parsedCount,
1634                       HttpHeaderStats[0].parsedCount);
1635     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1636 }
1637
1638 int
1639 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1640 {
1641     int result = 0;
1642     const char *pos = NULL;
1643     const char *item;
1644     int ilen;
1645     int mlen = strlen(member);
1646
1647     assert(any_registered_header(id));
1648
1649     String header (getStrOrList(id));
1650
1651     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1652         if (strncasecmp(item, member, mlen) == 0
1653                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1654             result = 1;
1655             break;
1656         }
1657     }
1658
1659     return result;
1660 }
1661
1662 int
1663 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1664 {
1665     int result = 0;
1666     const char *pos = NULL;
1667     const char *item;
1668     int ilen;
1669     int mlen = strlen(member);
1670
1671     assert(name);
1672
1673     String header (getByName(name));
1674
1675     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1676         if (strncasecmp(item, member, mlen) == 0
1677                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1678             result = 1;
1679             break;
1680         }
1681     }
1682
1683     return result;
1684 }
1685
1686 void
1687 HttpHeader::removeHopByHopEntries()
1688 {
1689     removeConnectionHeaderEntries();
1690
1691     const HttpHeaderEntry *e;
1692     HttpHeaderPos pos = HttpHeaderInitPos;
1693     int headers_deleted = 0;
1694     while ((e = getEntry(&pos))) {
1695         Http::HdrType id = e->id;
1696         if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1697             delAt(pos, headers_deleted);
1698             CBIT_CLR(mask, id);
1699         }
1700     }
1701 }
1702
1703 void
1704 HttpHeader::removeConnectionHeaderEntries()
1705 {
1706     if (has(Http::HdrType::CONNECTION)) {
1707         /* anything that matches Connection list member will be deleted */
1708         String strConnection;
1709
1710         (void) getList(Http::HdrType::CONNECTION, &strConnection);
1711         const HttpHeaderEntry *e;
1712         HttpHeaderPos pos = HttpHeaderInitPos;
1713         /*
1714          * think: on-average-best nesting of the two loops (hdrEntry
1715          * and strListItem) @?@
1716          */
1717         /*
1718          * maybe we should delete standard stuff ("keep-alive","close")
1719          * from strConnection first?
1720          */
1721
1722         int headers_deleted = 0;
1723         while ((e = getEntry(&pos))) {
1724             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1725                 delAt(pos, headers_deleted);
1726         }
1727         if (headers_deleted)
1728             refreshMask();
1729     }
1730 }
1731