src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/EnumIterator.h"
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "http/ContentLengthInterpreter.h"
  16 #include "HttpHdrCc.h"
  17 #include "HttpHdrContRange.h"
  18 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
  19 #include "HttpHeader.h"
  20 #include "HttpHeaderFieldInfo.h"
  21 #include "HttpHeaderStat.h"
  22 #include "HttpHeaderTools.h"
  23 #include "MemBuf.h"
  24 #include "mgr/Registration.h"
  25 #include "mime_header.h"
  26 #include "profiler/Profiler.h"
  27 #include "rfc1123.h"
  28 #include "SquidConfig.h"
  29 #include "StatHist.h"
  30 #include "Store.h"
  31 #include "StrList.h"
  32 #include "TimeOrTag.h"
  33 #include "util.h"
  34
  35 #include <algorithm>
  36
  37 /* XXX: the whole set of API managing the entries vector should be rethought
  38  *      after the parse4r-ng effort is complete.
  39  */
  40
  41 /*
  42  * On naming conventions:
  43  *
  44  * HTTP/1.1 defines message-header as
  45  *
  46  * message-header = field-name ":" [ field-value ] CRLF
  47  * field-name     = token
  48  * field-value    = *( field-content | LWS )
  49  *
  50  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  51  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  52  *
  53  * HttpHeader is an object that represents all message-headers in a message.
  54  * HttpHeader does not manage start-line.
  55  *
  56  * HttpHeader is implemented as a collection of header "entries".
  57  * An entry is a (field_id, field_name, field_value) triplet.
  58  */
  59
  60 /*
  61  * local constants and vars
  62  */
  63
  64 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
  65 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::enumEnd_);
  66
  67 /* request-only headers. Used for cachemgr */
  68 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
  69
  70 /* reply-only headers. Used for cachemgr */
  71 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
  72
  73 /* header accounting */
  74 // NP: keep in sync with enum http_hdr_owner_type
  75 static HttpHeaderStat HttpHeaderStats[] = {
  76     HttpHeaderStat(/*hoNone*/ "all", NULL),
  77 #if USE_HTCP
  78     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
  79 #endif
  80     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
  81     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
  82 #if USE_OPENSSL
  83     /* hoErrorDetail */
  84 #endif
  85     /* hoEnd */
  86 };
  87 static int HttpHeaderStatCount = countof(HttpHeaderStats);
  88
  89 static int HeaderEntryParsedCount = 0;
  90
  91 /*
  92  * forward declarations and local routines
  93  */
  94
  95 class StoreEntry;
  96
  97 // update parse statistics for header id; if error is true also account
  98 // for errors and write to debug log what happened
  99 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, bool error);
 100 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 101 /** store report about current header usage and other stats */
 102 static void httpHeaderStoreReport(StoreEntry * e);
 103
 104 /*
 105  * Module initialization routines
 106  */
 107
 108 static void
 109 httpHeaderRegisterWithCacheManager(void)
 110 {
 111     Mgr::RegisterAction("http_headers",
 112                         "HTTP Header Statistics",
 113                         httpHeaderStoreReport, 0, 1);
 114 }
 115
 116 void
 117 httpHeaderInitModule(void)
 118 {
 119     /* check that we have enough space for masks */
 120     assert(8 * sizeof(HttpHeaderMask) >= Http::HdrType::enumEnd_);
 121
 122     // masks are needed for stats page still
 123     for (auto h : WholeEnum<Http::HdrType>()) {
 124         if (Http::HeaderLookupTable.lookup(h).request)
 125             CBIT_SET(RequestHeadersMask,h);
 126         if (Http::HeaderLookupTable.lookup(h).reply)
 127             CBIT_SET(ReplyHeadersMask,h);
 128     }
 129
 130     /* header stats initialized by class constructor */
 131     assert(HttpHeaderStatCount == hoReply + 1);
 132
 133     /* init dependent modules */
 134     httpHdrCcInitModule();
 135     httpHdrScInitModule();
 136
 137     httpHeaderRegisterWithCacheManager();
 138 }
 139
 140 /*
 141  * HttpHeader Implementation
 142  */
 143
 144 HttpHeader::HttpHeader() : owner (hoNone), len (0), conflictingContentLength_(false)
 145 {
 146     httpHeaderMaskInit(&mask, 0);
 147 }
 148
 149 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0), conflictingContentLength_(false)
 150 {
 151     assert(anOwner > hoNone && anOwner < hoEnd);
 152     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 153     httpHeaderMaskInit(&mask, 0);
 154 }
 155
 156 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len), conflictingContentLength_(false)
 157 {
 158     httpHeaderMaskInit(&mask, 0);
 159     update(&other); // will update the mask as well
 160 }
 161
 162 HttpHeader::~HttpHeader()
 163 {
 164     clean();
 165 }
 166
 167 HttpHeader &
 168 HttpHeader::operator =(const HttpHeader &other)
 169 {
 170     if (this != &other) {
 171         // we do not really care, but the caller probably does
 172         assert(owner == other.owner);
 173         clean();
 174         update(&other); // will update the mask as well
 175         len = other.len;
 176         conflictingContentLength_ = other.conflictingContentLength_;
 177     }
 178     return *this;
 179 }
 180
 181 void
 182 HttpHeader::clean()
 183 {
 184
 185     assert(owner > hoNone && owner < hoEnd);
 186     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 187
 188     PROF_start(HttpHeaderClean);
 189
 190     if (owner <= hoReply) {
 191         /*
 192          * An unfortunate bug.  The entries array is initialized
 193          * such that count is set to zero.  httpHeaderClean() seems to
 194          * be called both when 'hdr' is created, and destroyed.  Thus,
 195          * we accumulate a large number of zero counts for 'hdr' before
 196          * it is ever used.  Can't think of a good way to fix it, except
 197          * adding a state variable that indicates whether or not 'hdr'
 198          * has been used.  As a hack, just never count zero-sized header
 199          * arrays.
 200          */
 201         if (!entries.empty())
 202             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 203
 204         ++ HttpHeaderStats[owner].destroyedCount;
 205
 206         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 207     } // if (owner <= hoReply)
 208
 209     for (HttpHeaderEntry *e : entries) {
 210         if (e == nullptr)
 211             continue;
 212         if (!Http::any_valid_header(e->id)) {
 213             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 214         } else {
 215             if (owner <= hoReply)
 216                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 217             delete e;
 218         }
 219     }
 220
 221     entries.clear();
 222     httpHeaderMaskInit(&mask, 0);
 223     len = 0;
 224     conflictingContentLength_ = false;
 225     PROF_stop(HttpHeaderClean);
 226 }
 227
 228 /* append entries (also see httpHeaderUpdate) */
 229 void
 230 HttpHeader::append(const HttpHeader * src)
 231 {
 232     assert(src);
 233     assert(src != this);
 234     debugs(55, 7, "appending hdr: " << this << " += " << src);
 235
 236     for (auto e : src->entries) {
 237         if (e)
 238             addEntry(e->clone());
 239     }
 240 }
 241
 242 /// check whether the fresh header has any new/changed updatable fields
 243 bool
 244 HttpHeader::needUpdate(HttpHeader const *fresh) const
 245 {
 246     for (const auto e: fresh->entries) {
 247         if (!e || skipUpdateHeader(e->id))
 248             continue;
 249         String value;
 250         const char *name = e->name.termedBuf();
 251         if (!hasNamed(name, strlen(name), &value) ||
 252                 (value != fresh->getByName(name)))
 253             return true;
 254     }
 255     return false;
 256 }
 257
 258 void
 259 HttpHeader::updateWarnings()
 260 {
 261     int count = 0;
 262     HttpHeaderPos pos = HttpHeaderInitPos;
 263
 264     // RFC 7234, section 4.3.4: delete 1xx warnings and retain 2xx warnings
 265     while (HttpHeaderEntry *e = getEntry(&pos)) {
 266         if (e->id == Http::HdrType::WARNING && (e->getInt()/100 == 1) )
 267             delAt(pos, count);
 268     }
 269 }
 270
 271 bool
 272 HttpHeader::skipUpdateHeader(const Http::HdrType id) const
 273 {
 274     // RFC 7234, section 4.3.4: use fields other from Warning for update
 275     return id == Http::HdrType::WARNING;
 276 }
 277
 278 bool
 279 HttpHeader::update(HttpHeader const *fresh)
 280 {
 281     assert(fresh);
 282     assert(this != fresh);
 283
 284     // Optimization: Finding whether a header field changed is expensive
 285     // and probably not worth it except for collapsed revalidation needs.
 286     if (Config.onoff.collapsed_forwarding && !needUpdate(fresh))
 287         return false;
 288
 289     updateWarnings();
 290
 291     const HttpHeaderEntry *e;
 292     HttpHeaderPos pos = HttpHeaderInitPos;
 293
 294     while ((e = fresh->getEntry(&pos))) {
 295         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 296
 297         if (skipUpdateHeader(e->id))
 298             continue;
 299
 300         if (e->id != Http::HdrType::OTHER)
 301             delById(e->id);
 302         else
 303             delByName(e->name.termedBuf());
 304     }
 305
 306     pos = HttpHeaderInitPos;
 307     while ((e = fresh->getEntry(&pos))) {
 308         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 309
 310         if (skipUpdateHeader(e->id))
 311             continue;
 312
 313         debugs(55, 7, "Updating header '" << Http::HeaderLookupTable.lookup(e->id).name << "' in cached entry");
 314
 315         addEntry(e->clone());
 316     }
 317     return true;
 318 }
 319
 320 bool
 321 HttpHeader::Isolate(const char **parse_start, size_t l, const char **blk_start, const char **blk_end)
 322 {
 323     /*
 324      * parse_start points to the first line of HTTP message *headers*,
 325      * not including the request or status lines
 326      */
 327     const size_t end = headersEnd(*parse_start, l);
 328
 329     if (end) {
 330         *blk_start = *parse_start;
 331         *blk_end = *parse_start + end - 1;
 332         assert(**blk_end == '\n');
 333         // Point blk_end to the first character after the last header field.
 334         // In other words, blk_end should point to the CR?LF header terminator.
 335         if (end > 1 && *(*blk_end - 1) == '\r')
 336             --(*blk_end);
 337         *parse_start += end;
 338     }
 339     return end;
 340 }
 341
 342 int
 343 HttpHeader::parse(const char *buf, size_t buf_len, bool atEnd, size_t &hdr_sz)
 344 {
 345     const char *parse_start = buf;
 346     const char *blk_start, *blk_end;
 347     hdr_sz = 0;
 348
 349     if (!Isolate(&parse_start, buf_len, &blk_start, &blk_end)) {
 350         // XXX: do not parse non-isolated headers even if the connection is closed.
 351         // Treat unterminated headers as "partial headers" framing errors.
 352         if (!atEnd)
 353             return 0;
 354         blk_start = parse_start;
 355         blk_end = blk_start + strlen(blk_start);
 356     }
 357
 358     if (parse(blk_start, blk_end - blk_start)) {
 359         hdr_sz = parse_start - buf;
 360         return 1;
 361     }
 362     return -1;
 363 }
 364
 365 int
 366 HttpHeader::parse(const char *header_start, size_t hdrLen)
 367 {
 368     const char *field_ptr = header_start;
 369     const char *header_end = header_start + hdrLen; // XXX: remove
 370     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 371
 372     PROF_start(HttpHeaderParse);
 373
 374     assert(header_start && header_end);
 375     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 376     ++ HttpHeaderStats[owner].parsedCount;
 377
 378     char *nulpos;
 379     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 380         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 381                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 382         PROF_stop(HttpHeaderParse);
 383         clean();
 384         return 0;
 385     }
 386
 387     Http::ContentLengthInterpreter clen(warnOnError);
 388     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 389      * continuation lines start with a (single) space or tab */
 390     while (field_ptr < header_end) {
 391         const char *field_start = field_ptr;
 392         const char *field_end;
 393
 394         do {
 395             const char *this_line = field_ptr;
 396             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 397
 398             if (!field_ptr) {
 399                 // missing <LF>
 400                 PROF_stop(HttpHeaderParse);
 401                 clean();
 402                 return 0;
 403             }
 404
 405             field_end = field_ptr;
 406
 407             ++field_ptr;    /* Move to next line */
 408
 409             if (field_end > this_line && field_end[-1] == '\r') {
 410                 --field_end;    /* Ignore CR LF */
 411
 412                 if (owner == hoRequest && field_end > this_line) {
 413                     bool cr_only = true;
 414                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 415                         if (*p != '\r')
 416                             cr_only = false;
 417                     }
 418                     if (cr_only) {
 419                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 420                                "header field to prevent request smuggling attacks: {" <<
 421                                getStringPrefix(header_start, hdrLen) << "}");
 422                         PROF_stop(HttpHeaderParse);
 423                         clean();
 424                         return 0;
 425                     }
 426                 }
 427             }
 428
 429             /* Barf on stray CR characters */
 430             if (memchr(this_line, '\r', field_end - this_line)) {
 431                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 432                        getStringPrefix(field_start, field_end-field_start) << "}");
 433
 434                 if (Config.onoff.relaxed_header_parser) {
 435                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 436
 437                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 438                         *p = ' ';
 439                         ++p;
 440                     }
 441                 } else {
 442                     PROF_stop(HttpHeaderParse);
 443                     clean();
 444                     return 0;
 445                 }
 446             }
 447
 448             if (this_line + 1 == field_end && this_line > field_start) {
 449                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 450                        getStringPrefix(header_start, hdrLen) << "}");
 451                 PROF_stop(HttpHeaderParse);
 452                 clean();
 453                 return 0;
 454             }
 455         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 456
 457         if (field_start == field_end) {
 458             if (field_ptr < header_end) {
 459                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 460                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 461                 PROF_stop(HttpHeaderParse);
 462                 clean();
 463                 return 0;
 464             }
 465
 466             break;      /* terminating blank line */
 467         }
 468
 469         HttpHeaderEntry *e;
 470         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 471             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 472                    getStringPrefix(field_start, field_end-field_start) << "}");
 473             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 474
 475             if (Config.onoff.relaxed_header_parser)
 476                 continue;
 477
 478             PROF_stop(HttpHeaderParse);
 479             clean();
 480             return 0;
 481         }
 482
 483         if (e->id == Http::HdrType::CONTENT_LENGTH && !clen.checkField(e->value)) {
 484             delete e;
 485
 486             if (Config.onoff.relaxed_header_parser)
 487                 continue; // clen has printed any necessary warnings
 488
 489             PROF_stop(HttpHeaderParse);
 490             clean();
 491             return 0;
 492         }
 493
 494         if (e->id == Http::HdrType::OTHER && stringHasWhitespace(e->name.termedBuf())) {
 495             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 496                    getStringPrefix(field_start, field_end-field_start) << "}");
 497
 498             if (!Config.onoff.relaxed_header_parser) {
 499                 delete e;
 500                 PROF_stop(HttpHeaderParse);
 501                 clean();
 502                 return 0;
 503             }
 504         }
 505
 506         addEntry(e);
 507     }
 508
 509     if (clen.headerWideProblem) {
 510         debugs(55, warnOnError, "WARNING: " << clen.headerWideProblem <<
 511                " Content-Length field values in" <<
 512                Raw("header", header_start, hdrLen));
 513     }
 514
 515     if (chunked()) {
 516         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 517         // RFC 7230 section 3.3.3 #3: Transfer-Encoding overwrites Content-Length
 518         delById(Http::HdrType::CONTENT_LENGTH);
 519         // and clen state becomes irrelevant
 520     } else if (clen.sawBad) {
 521         // ensure our callers do not accidentally see bad Content-Length values
 522         delById(Http::HdrType::CONTENT_LENGTH);
 523         conflictingContentLength_ = true; // TODO: Rename to badContentLength_.
 524     } else if (clen.needsSanitizing) {
 525         // RFC 7230 section 3.3.2: MUST either reject or ... [sanitize];
 526         // ensure our callers see a clean Content-Length value or none at all
 527         delById(Http::HdrType::CONTENT_LENGTH);
 528         if (clen.sawGood) {
 529             putInt64(Http::HdrType::CONTENT_LENGTH, clen.value);
 530             debugs(55, 5, "sanitized Content-Length to be " << clen.value);
 531         }
 532     }
 533
 534     PROF_stop(HttpHeaderParse);
 535     return 1;           /* even if no fields where found, it is a valid header */
 536 }
 537
 538 /* packs all the entries using supplied packer */
 539 void
 540 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 541 {
 542     HttpHeaderPos pos = HttpHeaderInitPos;
 543     const HttpHeaderEntry *e;
 544     assert(p);
 545     debugs(55, 7, this << " into " << p <<
 546            (mask_sensitive_info ? " while masking" : ""));
 547     /* pack all entries one by one */
 548     while ((e = getEntry(&pos))) {
 549         if (!mask_sensitive_info) {
 550             e->packInto(p);
 551             continue;
 552         }
 553
 554         bool maskThisEntry = false;
 555         switch (e->id) {
 556         case Http::HdrType::AUTHORIZATION:
 557         case Http::HdrType::PROXY_AUTHORIZATION:
 558             maskThisEntry = true;
 559             break;
 560
 561         case Http::HdrType::FTP_ARGUMENTS:
 562             if (const HttpHeaderEntry *cmd = findEntry(Http::HdrType::FTP_COMMAND))
 563                 maskThisEntry = (cmd->value == "PASS");
 564             break;
 565
 566         default:
 567             break;
 568         }
 569         if (maskThisEntry) {
 570             p->append(e->name.rawBuf(), e->name.size());
 571             p->append(": ** NOT DISPLAYED **\r\n", 23);
 572         } else {
 573             e->packInto(p);
 574         }
 575
 576     }
 577     /* Pack in the "special" entries */
 578
 579     /* Cache-Control */
 580 }
 581
 582 /* returns next valid entry */
 583 HttpHeaderEntry *
 584 HttpHeader::getEntry(HttpHeaderPos * pos) const
 585 {
 586     assert(pos);
 587     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 588
 589     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 590         if (entries[*pos])
 591             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 592     }
 593
 594     return NULL;
 595 }
 596
 597 /*
 598  * returns a pointer to a specified entry if any
 599  * note that we return one entry so it does not make much sense to ask for
 600  * "list" headers
 601  */
 602 HttpHeaderEntry *
 603 HttpHeader::findEntry(Http::HdrType id) const
 604 {
 605     assert(any_registered_header(id));
 606     assert(!Http::HeaderLookupTable.lookup(id).list);
 607
 608     /* check mask first */
 609
 610     if (!CBIT_TEST(mask, id))
 611         return NULL;
 612
 613     /* looks like we must have it, do linear search */
 614     for (auto e : entries) {
 615         if (e && e->id == id)
 616             return e;
 617     }
 618
 619     /* hm.. we thought it was there, but it was not found */
 620     assert(false);
 621     return nullptr;        /* not reached */
 622 }
 623
 624 /*
 625  * same as httpHeaderFindEntry
 626  */
 627 HttpHeaderEntry *
 628 HttpHeader::findLastEntry(Http::HdrType id) const
 629 {
 630     assert(any_registered_header(id));
 631     assert(!Http::HeaderLookupTable.lookup(id).list);
 632
 633     /* check mask first */
 634     if (!CBIT_TEST(mask, id))
 635         return NULL;
 636
 637     for (auto e = entries.rbegin(); e != entries.rend(); ++e) {
 638         if (*e && (*e)->id == id)
 639             return *e;
 640     }
 641
 642     /* hm.. we thought it was there, but it was not found */
 643     assert(false);
 644     return nullptr; /* not reached */
 645 }
 646
 647 /*
 648  * deletes all fields with a given name if any, returns #fields deleted;
 649  */
 650 int
 651 HttpHeader::delByName(const char *name)
 652 {
 653     int count = 0;
 654     HttpHeaderPos pos = HttpHeaderInitPos;
 655     HttpHeaderEntry *e;
 656     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 657     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 658
 659     while ((e = getEntry(&pos))) {
 660         if (!e->name.caseCmp(name))
 661             delAt(pos, count);
 662         else
 663             CBIT_SET(mask, e->id);
 664     }
 665
 666     return count;
 667 }
 668
 669 /* deletes all entries with a given id, returns the #entries deleted */
 670 int
 671 HttpHeader::delById(Http::HdrType id)
 672 {
 673     debugs(55, 8, this << " del-by-id " << id);
 674     assert(any_registered_header(id));
 675
 676     if (!CBIT_TEST(mask, id))
 677         return 0;
 678
 679     int count = 0;
 680
 681     HttpHeaderPos pos = HttpHeaderInitPos;
 682     while (HttpHeaderEntry *e = getEntry(&pos)) {
 683         if (e->id == id)
 684             delAt(pos, count); // deletes e
 685     }
 686
 687     CBIT_CLR(mask, id);
 688     assert(count);
 689     return count;
 690 }
 691
 692 /*
 693  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 694  * possible to iterate(search) and delete fields at the same time
 695  * NOTE: Does not update the header mask. Caller must follow up with
 696  * a call to refreshMask() if headers_deleted was incremented.
 697  */
 698 void
 699 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 700 {
 701     HttpHeaderEntry *e;
 702     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 703     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 704     entries[pos] = NULL;
 705     /* decrement header length, allow for ": " and crlf */
 706     len -= e->name.size() + 2 + e->value.size() + 2;
 707     assert(len >= 0);
 708     delete e;
 709     ++headers_deleted;
 710 }
 711
 712 /*
 713  * Compacts the header storage
 714  */
 715 void
 716 HttpHeader::compact()
 717 {
 718     // TODO: optimize removal, or possibly make it so that's not needed.
 719     entries.erase( std::remove(entries.begin(), entries.end(), nullptr),
 720                    entries.end());
 721 }
 722
 723 /*
 724  * Refreshes the header mask. Required after delAt() calls.
 725  */
 726 void
 727 HttpHeader::refreshMask()
 728 {
 729     httpHeaderMaskInit(&mask, 0);
 730     debugs(55, 7, "refreshing the mask in hdr " << this);
 731     for (auto e : entries) {
 732         if (e)
 733             CBIT_SET(mask, e->id);
 734     }
 735 }
 736
 737 /* appends an entry;
 738  * does not call e->clone() so one should not reuse "*e"
 739  */
 740 void
 741 HttpHeader::addEntry(HttpHeaderEntry * e)
 742 {
 743     assert(e);
 744     assert(any_HdrType_enum_value(e->id));
 745     assert(e->name.size());
 746
 747     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 748
 749     if (e->id != Http::HdrType::BAD_HDR) {
 750         if (CBIT_TEST(mask, e->id)) {
 751             ++ headerStatsTable[e->id].repCount;
 752         } else {
 753             CBIT_SET(mask, e->id);
 754         }
 755     }
 756
 757     entries.push_back(e);
 758
 759     /* increment header length, allow for ": " and crlf */
 760     len += e->name.size() + 2 + e->value.size() + 2;
 761 }
 762
 763 /* inserts an entry;
 764  * does not call e->clone() so one should not reuse "*e"
 765  */
 766 void
 767 HttpHeader::insertEntry(HttpHeaderEntry * e)
 768 {
 769     assert(e);
 770     assert(any_valid_header(e->id));
 771
 772     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 773
 774     // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
 775     if (CBIT_TEST(mask, e->id)) {
 776         ++ headerStatsTable[e->id].repCount;
 777     } else {
 778         CBIT_SET(mask, e->id);
 779     }
 780
 781     entries.insert(entries.begin(),e);
 782
 783     /* increment header length, allow for ": " and crlf */
 784     len += e->name.size() + 2 + e->value.size() + 2;
 785 }
 786
 787 bool
 788 HttpHeader::getList(Http::HdrType id, String *s) const
 789 {
 790     debugs(55, 9, this << " joining for id " << id);
 791     /* only fields from ListHeaders array can be "listed" */
 792     assert(Http::HeaderLookupTable.lookup(id).list);
 793
 794     if (!CBIT_TEST(mask, id))
 795         return false;
 796
 797     for (auto e: entries) {
 798         if (e && e->id == id)
 799             strListAdd(s, e->value.termedBuf(), ',');
 800     }
 801
 802     /*
 803      * note: we might get an empty (size==0) string if there was an "empty"
 804      * header. This results in an empty length String, which may have a NULL
 805      * buffer.
 806      */
 807     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 808     if (!s->size())
 809         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 810     else
 811         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 812
 813     return true;
 814 }
 815
 816 /* return a list of entries with the same id separated by ',' and ws */
 817 String
 818 HttpHeader::getList(Http::HdrType id) const
 819 {
 820     HttpHeaderEntry *e;
 821     HttpHeaderPos pos = HttpHeaderInitPos;
 822     debugs(55, 9, this << "joining for id " << id);
 823     /* only fields from ListHeaders array can be "listed" */
 824     assert(Http::HeaderLookupTable.lookup(id).list);
 825
 826     if (!CBIT_TEST(mask, id))
 827         return String();
 828
 829     String s;
 830
 831     while ((e = getEntry(&pos))) {
 832         if (e->id == id)
 833             strListAdd(&s, e->value.termedBuf(), ',');
 834     }
 835
 836     /*
 837      * note: we might get an empty (size==0) string if there was an "empty"
 838      * header. This results in an empty length String, which may have a NULL
 839      * buffer.
 840      */
 841     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 842     if (!s.size())
 843         debugs(55, 3, "empty list header: " << Http::HeaderLookupTable.lookup(id).name << "(" << id << ")");
 844     else
 845         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 846
 847     return s;
 848 }
 849
 850 /* return a string or list of entries with the same id separated by ',' and ws */
 851 String
 852 HttpHeader::getStrOrList(Http::HdrType id) const
 853 {
 854     HttpHeaderEntry *e;
 855
 856     if (Http::HeaderLookupTable.lookup(id).list)
 857         return getList(id);
 858
 859     if ((e = findEntry(id)))
 860         return e->value;
 861
 862     return String();
 863 }
 864
 865 /*
 866  * Returns the value of the specified header and/or an undefined String.
 867  */
 868 String
 869 HttpHeader::getByName(const char *name) const
 870 {
 871     String result;
 872     // ignore presence: return undefined string if an empty header is present
 873     (void)hasNamed(name, strlen(name), &result);
 874     return result;
 875 }
 876
 877 String
 878 HttpHeader::getByName(const SBuf &name) const
 879 {
 880     String result;
 881     // ignore presence: return undefined string if an empty header is present
 882     (void)hasNamed(name, &result);
 883     return result;
 884 }
 885
 886 String
 887 HttpHeader::getById(Http::HdrType id) const
 888 {
 889     String result;
 890     (void)getByIdIfPresent(id, &result);
 891     return result;
 892 }
 893
 894 bool
 895 HttpHeader::hasNamed(const SBuf &s, String *result) const
 896 {
 897     return hasNamed(s.rawContent(), s.length(), result);
 898 }
 899
 900 bool
 901 HttpHeader::getByIdIfPresent(Http::HdrType id, String *result) const
 902 {
 903     if (id == Http::HdrType::BAD_HDR)
 904         return false;
 905     if (!has(id))
 906         return false;
 907     if (result)
 908         *result = getStrOrList(id);
 909     return true;
 910 }
 911
 912 bool
 913 HttpHeader::hasNamed(const char *name, int namelen, String *result) const
 914 {
 915     Http::HdrType id;
 916     HttpHeaderPos pos = HttpHeaderInitPos;
 917     HttpHeaderEntry *e;
 918
 919     assert(name);
 920
 921     /* First try the quick path */
 922     id = Http::HeaderLookupTable.lookup(name,namelen).id;
 923
 924     if (id != Http::HdrType::BAD_HDR) {
 925         if (getByIdIfPresent(id, result))
 926             return true;
 927     }
 928
 929     /* Sorry, an unknown header name. Do linear search */
 930     bool found = false;
 931     while ((e = getEntry(&pos))) {
 932         if (e->id == Http::HdrType::OTHER && e->name.size() == static_cast<String::size_type>(namelen) && e->name.caseCmp(name, namelen) == 0) {
 933             found = true;
 934             if (!result)
 935                 break;
 936             strListAdd(result, e->value.termedBuf(), ',');
 937         }
 938     }
 939
 940     return found;
 941 }
 942
 943 /*
 944  * Returns a the value of the specified list member, if any.
 945  */
 946 String
 947 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
 948 {
 949     String header;
 950     const char *pos = NULL;
 951     const char *item;
 952     int ilen;
 953     int mlen = strlen(member);
 954
 955     assert(name);
 956
 957     header = getByName(name);
 958
 959     String result;
 960
 961     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 962         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 963             result.append(item + mlen + 1, ilen - mlen - 1);
 964             break;
 965         }
 966     }
 967
 968     return result;
 969 }
 970
 971 /*
 972  * returns a the value of the specified list member, if any.
 973  */
 974 String
 975 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
 976 {
 977     String header;
 978     const char *pos = NULL;
 979     const char *item;
 980     int ilen;
 981     int mlen = strlen(member);
 982
 983     assert(any_registered_header(id));
 984
 985     header = getStrOrList(id);
 986     String result;
 987
 988     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 989         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
 990             result.append(item + mlen + 1, ilen - mlen - 1);
 991             break;
 992         }
 993     }
 994
 995     header.clean();
 996     return result;
 997 }
 998
 999 /* test if a field is present */
1000 int
1001 HttpHeader::has(Http::HdrType id) const
1002 {
1003     assert(any_registered_header(id));
1004     debugs(55, 9, this << " lookup for " << id);
1005     return CBIT_TEST(mask, id);
1006 }
1007
1008 void
1009 HttpHeader::putInt(Http::HdrType id, int number)
1010 {
1011     assert(any_registered_header(id));
1012     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1013     assert(number >= 0);
1014     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1015 }
1016
1017 void
1018 HttpHeader::putInt64(Http::HdrType id, int64_t number)
1019 {
1020     assert(any_registered_header(id));
1021     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1022     assert(number >= 0);
1023     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1024 }
1025
1026 void
1027 HttpHeader::putTime(Http::HdrType id, time_t htime)
1028 {
1029     assert(any_registered_header(id));
1030     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1031     assert(htime >= 0);
1032     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1033 }
1034
1035 void
1036 HttpHeader::putStr(Http::HdrType id, const char *str)
1037 {
1038     assert(any_registered_header(id));
1039     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1040     assert(str);
1041     addEntry(new HttpHeaderEntry(id, NULL, str));
1042 }
1043
1044 void
1045 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1046 {
1047     assert(auth_scheme && realm);
1048     httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1049 }
1050
1051 void
1052 HttpHeader::putCc(const HttpHdrCc * cc)
1053 {
1054     assert(cc);
1055     /* remove old directives if any */
1056     delById(Http::HdrType::CACHE_CONTROL);
1057     /* pack into mb */
1058     MemBuf mb;
1059     mb.init();
1060     cc->packInto(&mb);
1061     /* put */
1062     addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL, NULL, mb.buf));
1063     /* cleanup */
1064     mb.clean();
1065 }
1066
1067 void
1068 HttpHeader::putContRange(const HttpHdrContRange * cr)
1069 {
1070     assert(cr);
1071     /* remove old directives if any */
1072     delById(Http::HdrType::CONTENT_RANGE);
1073     /* pack into mb */
1074     MemBuf mb;
1075     mb.init();
1076     httpHdrContRangePackInto(cr, &mb);
1077     /* put */
1078     addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE, NULL, mb.buf));
1079     /* cleanup */
1080     mb.clean();
1081 }
1082
1083 void
1084 HttpHeader::putRange(const HttpHdrRange * range)
1085 {
1086     assert(range);
1087     /* remove old directives if any */
1088     delById(Http::HdrType::RANGE);
1089     /* pack into mb */
1090     MemBuf mb;
1091     mb.init();
1092     range->packInto(&mb);
1093     /* put */
1094     addEntry(new HttpHeaderEntry(Http::HdrType::RANGE, NULL, mb.buf));
1095     /* cleanup */
1096     mb.clean();
1097 }
1098
1099 void
1100 HttpHeader::putSc(HttpHdrSc *sc)
1101 {
1102     assert(sc);
1103     /* remove old directives if any */
1104     delById(Http::HdrType::SURROGATE_CONTROL);
1105     /* pack into mb */
1106     MemBuf mb;
1107     mb.init();
1108     sc->packInto(&mb);
1109     /* put */
1110     addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL, NULL, mb.buf));
1111     /* cleanup */
1112     mb.clean();
1113 }
1114
1115 void
1116 HttpHeader::putWarning(const int code, const char *const text)
1117 {
1118     char buf[512];
1119     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1120     putStr(Http::HdrType::WARNING, buf);
1121 }
1122
1123 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1124 void
1125 HttpHeader::putExt(const char *name, const char *value)
1126 {
1127     assert(name && value);
1128     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1129     addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, name, value));
1130 }
1131
1132 int
1133 HttpHeader::getInt(Http::HdrType id) const
1134 {
1135     assert(any_registered_header(id));
1136     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1137     HttpHeaderEntry *e;
1138
1139     if ((e = findEntry(id)))
1140         return e->getInt();
1141
1142     return -1;
1143 }
1144
1145 int64_t
1146 HttpHeader::getInt64(Http::HdrType id) const
1147 {
1148     assert(any_registered_header(id));
1149     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1150     HttpHeaderEntry *e;
1151
1152     if ((e = findEntry(id)))
1153         return e->getInt64();
1154
1155     return -1;
1156 }
1157
1158 time_t
1159 HttpHeader::getTime(Http::HdrType id) const
1160 {
1161     HttpHeaderEntry *e;
1162     time_t value = -1;
1163     assert(any_registered_header(id));
1164     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1165
1166     if ((e = findEntry(id))) {
1167         value = parse_rfc1123(e->value.termedBuf());
1168         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1169     }
1170
1171     return value;
1172 }
1173
1174 /* sync with httpHeaderGetLastStr */
1175 const char *
1176 HttpHeader::getStr(Http::HdrType id) const
1177 {
1178     HttpHeaderEntry *e;
1179     assert(any_registered_header(id));
1180     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1181
1182     if ((e = findEntry(id))) {
1183         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1184         return e->value.termedBuf();
1185     }
1186
1187     return NULL;
1188 }
1189
1190 /* unusual */
1191 const char *
1192 HttpHeader::getLastStr(Http::HdrType id) const
1193 {
1194     HttpHeaderEntry *e;
1195     assert(any_registered_header(id));
1196     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1197
1198     if ((e = findLastEntry(id))) {
1199         httpHeaderNoteParsedEntry(e->id, e->value, false);  /* no errors are possible */
1200         return e->value.termedBuf();
1201     }
1202
1203     return NULL;
1204 }
1205
1206 HttpHdrCc *
1207 HttpHeader::getCc() const
1208 {
1209     if (!CBIT_TEST(mask, Http::HdrType::CACHE_CONTROL))
1210         return NULL;
1211     PROF_start(HttpHeader_getCc);
1212
1213     String s;
1214     getList(Http::HdrType::CACHE_CONTROL, &s);
1215
1216     HttpHdrCc *cc=new HttpHdrCc();
1217
1218     if (!cc->parse(s)) {
1219         delete cc;
1220         cc = NULL;
1221     }
1222
1223     ++ HttpHeaderStats[owner].ccParsedCount;
1224
1225     if (cc)
1226         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1227
1228     httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL, s, !cc);
1229
1230     PROF_stop(HttpHeader_getCc);
1231
1232     return cc;
1233 }
1234
1235 HttpHdrRange *
1236 HttpHeader::getRange() const
1237 {
1238     HttpHdrRange *r = NULL;
1239     HttpHeaderEntry *e;
1240     /* some clients will send "Request-Range" _and_ *matching* "Range"
1241      * who knows, some clients might send Request-Range only;
1242      * this "if" should work correctly in both cases;
1243      * hopefully no clients send mismatched headers! */
1244
1245     if ((e = findEntry(Http::HdrType::RANGE)) ||
1246             (e = findEntry(Http::HdrType::REQUEST_RANGE))) {
1247         r = HttpHdrRange::ParseCreate(&e->value);
1248         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1249     }
1250
1251     return r;
1252 }
1253
1254 HttpHdrSc *
1255 HttpHeader::getSc() const
1256 {
1257     if (!CBIT_TEST(mask, Http::HdrType::SURROGATE_CONTROL))
1258         return NULL;
1259
1260     String s;
1261
1262     (void) getList(Http::HdrType::SURROGATE_CONTROL, &s);
1263
1264     HttpHdrSc *sc = httpHdrScParseCreate(s);
1265
1266     ++ HttpHeaderStats[owner].ccParsedCount;
1267
1268     if (sc)
1269         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1270
1271     httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL, s, !sc);
1272
1273     return sc;
1274 }
1275
1276 HttpHdrContRange *
1277 HttpHeader::getContRange() const
1278 {
1279     HttpHdrContRange *cr = NULL;
1280     HttpHeaderEntry *e;
1281
1282     if ((e = findEntry(Http::HdrType::CONTENT_RANGE))) {
1283         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1284         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1285     }
1286
1287     return cr;
1288 }
1289
1290 const char *
1291 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1292 {
1293     const char *field;
1294     int l;
1295     assert(auth_scheme);
1296     field = getStr(id);
1297
1298     if (!field)         /* no authorization field */
1299         return NULL;
1300
1301     l = strlen(auth_scheme);
1302
1303     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1304         return NULL;
1305
1306     field += l;
1307
1308     if (!xisspace(*field))  /* wrong scheme */
1309         return NULL;
1310
1311     /* skip white space */
1312     for (; field && xisspace(*field); ++field);
1313
1314     if (!*field)        /* no authorization cookie */
1315         return NULL;
1316
1317     static char decodedAuthToken[8192];
1318     struct base64_decode_ctx ctx;
1319     base64_decode_init(&ctx);
1320     size_t decodedLen = 0;
1321     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1322             !base64_decode_final(&ctx)) {
1323         return NULL;
1324     }
1325     decodedAuthToken[decodedLen] = '\0';
1326     return decodedAuthToken;
1327 }
1328
1329 ETag
1330 HttpHeader::getETag(Http::HdrType id) const
1331 {
1332     ETag etag = {NULL, -1};
1333     HttpHeaderEntry *e;
1334     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftETag);     /* must be of an appropriate type */
1335
1336     if ((e = findEntry(id)))
1337         etagParseInit(&etag, e->value.termedBuf());
1338
1339     return etag;
1340 }
1341
1342 TimeOrTag
1343 HttpHeader::getTimeOrTag(Http::HdrType id) const
1344 {
1345     TimeOrTag tot;
1346     HttpHeaderEntry *e;
1347     assert(Http::HeaderLookupTable.lookup(id).type == Http::HdrFieldType::ftDate_1123_or_ETag);    /* must be of an appropriate type */
1348     memset(&tot, 0, sizeof(tot));
1349
1350     if ((e = findEntry(id))) {
1351         const char *str = e->value.termedBuf();
1352         /* try as an ETag */
1353
1354         if (etagParseInit(&tot.tag, str)) {
1355             tot.valid = tot.tag.str != NULL;
1356             tot.time = -1;
1357         } else {
1358             /* or maybe it is time? */
1359             tot.time = parse_rfc1123(str);
1360             tot.valid = tot.time >= 0;
1361             tot.tag.str = NULL;
1362         }
1363     }
1364
1365     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1366     return tot;
1367 }
1368
1369 /*
1370  * HttpHeaderEntry
1371  */
1372
1373 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const char *aName, const char *aValue)
1374 {
1375     assert(any_HdrType_enum_value(anId));
1376     id = anId;
1377
1378     if (id != Http::HdrType::OTHER)
1379         name = Http::HeaderLookupTable.lookup(id).name;
1380     else
1381         name = aName;
1382
1383     value = aValue;
1384
1385     if (id != Http::HdrType::BAD_HDR)
1386         ++ headerStatsTable[id].aliveCount;
1387
1388     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1389 }
1390
1391 HttpHeaderEntry::~HttpHeaderEntry()
1392 {
1393     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1394
1395     if (id != Http::HdrType::BAD_HDR) {
1396         assert(headerStatsTable[id].aliveCount);
1397         -- headerStatsTable[id].aliveCount;
1398         id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1399     }
1400
1401 }
1402
1403 /* parses and inits header entry, returns true/false */
1404 HttpHeaderEntry *
1405 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1406 {
1407     /* note: name_start == field_start */
1408     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1409     int name_len = name_end ? name_end - field_start :0;
1410     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1411     /* note: value_end == field_end */
1412
1413     ++ HeaderEntryParsedCount;
1414
1415     /* do we have a valid field name within this field? */
1416
1417     if (!name_len || name_end > field_end)
1418         return NULL;
1419
1420     if (name_len > 65534) {
1421         /* String must be LESS THAN 64K and it adds a terminating NULL */
1422         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1423         return NULL;
1424     }
1425
1426     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1427         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1428                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1429
1430         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1431             --name_len;
1432
1433         if (!name_len)
1434             return NULL;
1435     }
1436
1437     /* now we know we can parse it */
1438
1439     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1440
1441     /* is it a "known" field? */
1442     Http::HdrType id = Http::HeaderLookupTable.lookup(field_start,name_len).id;
1443     debugs(55, 9, "got hdr-id=" << id);
1444
1445     String name;
1446
1447     String value;
1448
1449     if (id == Http::HdrType::BAD_HDR)
1450         id = Http::HdrType::OTHER;
1451
1452     /* set field name */
1453     if (id == Http::HdrType::OTHER)
1454         name.limitInit(field_start, name_len);
1455     else
1456         name = Http::HeaderLookupTable.lookup(id).name;
1457
1458     /* trim field value */
1459     while (value_start < field_end && xisspace(*value_start))
1460         ++value_start;
1461
1462     while (value_start < field_end && xisspace(field_end[-1]))
1463         --field_end;
1464
1465     if (field_end - value_start > 65534) {
1466         /* String must be LESS THAN 64K and it adds a terminating NULL */
1467         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1468
1469         if (id == Http::HdrType::OTHER)
1470             name.clean();
1471
1472         return NULL;
1473     }
1474
1475     /* set field value */
1476     value.limitInit(value_start, field_end - value_start);
1477
1478     if (id != Http::HdrType::BAD_HDR)
1479         ++ headerStatsTable[id].seenCount;
1480
1481     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1482
1483     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1484 }
1485
1486 HttpHeaderEntry *
1487 HttpHeaderEntry::clone() const
1488 {
1489     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1490 }
1491
1492 void
1493 HttpHeaderEntry::packInto(Packable * p) const
1494 {
1495     assert(p);
1496     p->append(name.rawBuf(), name.size());
1497     p->append(": ", 2);
1498     p->append(value.rawBuf(), value.size());
1499     p->append("\r\n", 2);
1500 }
1501
1502 int
1503 HttpHeaderEntry::getInt() const
1504 {
1505     int val = -1;
1506     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1507     httpHeaderNoteParsedEntry(id, value, ok == 0);
1508     /* XXX: Should we check ok - ie
1509      * return ok ? -1 : value;
1510      */
1511     return val;
1512 }
1513
1514 int64_t
1515 HttpHeaderEntry::getInt64() const
1516 {
1517     int64_t val = -1;
1518     const bool ok = httpHeaderParseOffset(value.termedBuf(), &val);
1519     httpHeaderNoteParsedEntry(id, value, !ok);
1520     return val; // remains -1 if !ok (XXX: bad method API)
1521 }
1522
1523 static void
1524 httpHeaderNoteParsedEntry(Http::HdrType id, String const &context, bool error)
1525 {
1526     if (id != Http::HdrType::BAD_HDR)
1527         ++ headerStatsTable[id].parsCount;
1528
1529     if (error) {
1530         if (id != Http::HdrType::BAD_HDR)
1531             ++ headerStatsTable[id].errCount;
1532         debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderLookupTable.lookup(id).name << ": " << context << "'");
1533     }
1534 }
1535
1536 /*
1537  * Reports
1538  */
1539
1540 /* tmp variable used to pass stat info to dumpers */
1541 extern const HttpHeaderStat *dump_stat;     /* argh! */
1542 const HttpHeaderStat *dump_stat = NULL;
1543
1544 void
1545 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1546 {
1547     const int id = static_cast<int>(val);
1548     const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1549     const char *name = valid_id ? Http::HeaderLookupTable.lookup(static_cast<Http::HdrType>(id)).name : "INVALID";
1550     int visible = count > 0;
1551     /* for entries with zero count, list only those that belong to current type of message */
1552
1553     if (!visible && valid_id && dump_stat->owner_mask)
1554         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1555
1556     if (visible)
1557         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1558                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1559 }
1560
1561 static void
1562 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1563 {
1564     if (count)
1565         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1566                           idx, (int) val, count,
1567                           xpercent(count, dump_stat->destroyedCount));
1568 }
1569
1570 static void
1571 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1572 {
1573     assert(hs);
1574     assert(e);
1575
1576     dump_stat = hs;
1577     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1578     storeAppendPrintf(e, "\nField type distribution\n");
1579     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1580                       "id", "name", "count", "#/header");
1581     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1582     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1583     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1584                       "id", "name", "count", "#/cc_field");
1585     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1586     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1587     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1588                       "id", "name", "count", "#/sc_field");
1589     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1590     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1591     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1592                       "id", "#flds", "count", "%total");
1593     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1594     storeAppendPrintf(e, "\n");
1595     dump_stat = NULL;
1596 }
1597
1598 void
1599 httpHeaderStoreReport(StoreEntry * e)
1600 {
1601     int i;
1602     assert(e);
1603
1604     HttpHeaderStats[0].parsedCount =
1605         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1606     HttpHeaderStats[0].ccParsedCount =
1607         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1608     HttpHeaderStats[0].destroyedCount =
1609         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1610     HttpHeaderStats[0].busyDestroyedCount =
1611         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1612
1613     for (i = 1; i < HttpHeaderStatCount; ++i) {
1614         httpHeaderStatDump(HttpHeaderStats + i, e);
1615     }
1616
1617     /* field stats for all messages */
1618     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1619
1620     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1621                       "id", "name", "#alive", "%err", "%repeat");
1622
1623     // scan heaaderTable and output
1624     for (auto h : WholeEnum<Http::HdrType>()) {
1625         auto stats = headerStatsTable[h];
1626         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1627                           Http::HeaderLookupTable.lookup(h).id,
1628                           Http::HeaderLookupTable.lookup(h).name,
1629                           stats.aliveCount,
1630                           xpercent(stats.errCount, stats.parsCount),
1631                           xpercent(stats.repCount, stats.seenCount));
1632     }
1633
1634     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1635                       HttpHeaderStats[hoRequest].parsedCount,
1636                       HttpHeaderStats[hoReply].parsedCount,
1637                       HttpHeaderStats[0].parsedCount);
1638     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1639 }
1640
1641 int
1642 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1643 {
1644     int result = 0;
1645     const char *pos = NULL;
1646     const char *item;
1647     int ilen;
1648     int mlen = strlen(member);
1649
1650     assert(any_registered_header(id));
1651
1652     String header (getStrOrList(id));
1653
1654     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1655         if (strncasecmp(item, member, mlen) == 0
1656                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1657             result = 1;
1658             break;
1659         }
1660     }
1661
1662     return result;
1663 }
1664
1665 int
1666 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1667 {
1668     int result = 0;
1669     const char *pos = NULL;
1670     const char *item;
1671     int ilen;
1672     int mlen = strlen(member);
1673
1674     assert(name);
1675
1676     String header (getByName(name));
1677
1678     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1679         if (strncasecmp(item, member, mlen) == 0
1680                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1681             result = 1;
1682             break;
1683         }
1684     }
1685
1686     return result;
1687 }
1688
1689 void
1690 HttpHeader::removeHopByHopEntries()
1691 {
1692     removeConnectionHeaderEntries();
1693
1694     const HttpHeaderEntry *e;
1695     HttpHeaderPos pos = HttpHeaderInitPos;
1696     int headers_deleted = 0;
1697     while ((e = getEntry(&pos))) {
1698         Http::HdrType id = e->id;
1699         if (Http::HeaderLookupTable.lookup(id).hopbyhop) {
1700             delAt(pos, headers_deleted);
1701             CBIT_CLR(mask, id);
1702         }
1703     }
1704 }
1705
1706 void
1707 HttpHeader::removeConnectionHeaderEntries()
1708 {
1709     if (has(Http::HdrType::CONNECTION)) {
1710         /* anything that matches Connection list member will be deleted */
1711         String strConnection;
1712
1713         (void) getList(Http::HdrType::CONNECTION, &strConnection);
1714         const HttpHeaderEntry *e;
1715         HttpHeaderPos pos = HttpHeaderInitPos;
1716         /*
1717          * think: on-average-best nesting of the two loops (hdrEntry
1718          * and strListItem) @?@
1719          */
1720         /*
1721          * maybe we should delete standard stuff ("keep-alive","close")
1722          * from strConnection first?
1723          */
1724
1725         int headers_deleted = 0;
1726         while ((e = getEntry(&pos))) {
1727             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1728                 delAt(pos, headers_deleted);
1729         }
1730         if (headers_deleted)
1731             refreshMask();
1732     }
1733 }
1734