src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 //#include "base/LookupTable.h" // pulled by HttpHdrCc.h
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "HttpHdrCc.h"
  16 #include "HttpHdrContRange.h"
  17 #include "HttpHdrScTarget.h" // also includes HttpHdrSc.h
  18 #include "HttpHeader.h"
  19 #include "HttpHeaderFieldInfo.h"
  20 #include "HttpHeaderStat.h"
  21 #include "HttpHeaderTools.h"
  22 #include "MemBuf.h"
  23 #include "mgr/Registration.h"
  24 #include "profiler/Profiler.h"
  25 #include "rfc1123.h"
  26 #include "SquidConfig.h"
  27 //#include "SquidString.h" // pulled by HttpHdrCc.h
  28 #include "StatHist.h"
  29 #include "Store.h"
  30 #include "StrList.h"
  31 #include "TimeOrTag.h"
  32 #include "util.h"
  33
  34 #include <algorithm>
  35
  36 /* XXX: the whole set of API managing the entries vector should be rethought
  37  *      after the parse4r-ng effort is complete.
  38  */
  39
  40 /*
  41  * On naming conventions:
  42  *
  43  * HTTP/1.1 defines message-header as
  44  *
  45  * message-header = field-name ":" [ field-value ] CRLF
  46  * field-name     = token
  47  * field-value    = *( field-content | LWS )
  48  *
  49  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  50  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  51  *
  52  * HttpHeader is an object that represents all message-headers in a message.
  53  * HttpHeader does not manage start-line.
  54  *
  55  * HttpHeader is implemented as a collection of header "entries".
  56  * An entry is a (field_id, field_name, field_value) triplet.
  57  */
  58
  59 /*
  60  * local constants and vars
  61  */
  62
  63 // statistics counters for headers. clients must not allow Http::HdrType::BAD_HDR to be counted
  64 std::vector<HttpHeaderFieldStat> headerStatsTable(Http::HdrType::ENUM_END);
  65
  66 /*
  67  * headers with field values defined as #(values) in HTTP/1.1
  68  * Headers that are currently not recognized, are commented out.
  69  */
  70 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
  71 static Http::HdrType ListHeadersArr[] = {
  72     Http::HdrType::ACCEPT,
  73     Http::HdrType::ACCEPT_CHARSET,
  74     Http::HdrType::ACCEPT_ENCODING,
  75     Http::HdrType::ACCEPT_LANGUAGE,
  76     Http::HdrType::ACCEPT_RANGES,
  77     Http::HdrType::ALLOW,
  78     Http::HdrType::CACHE_CONTROL,
  79     Http::HdrType::CONTENT_ENCODING,
  80     Http::HdrType::CONTENT_LANGUAGE,
  81     Http::HdrType::CONNECTION,
  82     Http::HdrType::EXPECT,
  83     Http::HdrType::IF_MATCH,
  84     Http::HdrType::IF_NONE_MATCH,
  85     Http::HdrType::KEY,
  86     Http::HdrType::LINK,
  87     Http::HdrType::PRAGMA,
  88     Http::HdrType::PROXY_CONNECTION,
  89     Http::HdrType::PROXY_SUPPORT,
  90     Http::HdrType::TRANSFER_ENCODING,
  91     Http::HdrType::UPGRADE,
  92     Http::HdrType::VARY,
  93     Http::HdrType::VIA,
  94     Http::HdrType::WARNING,
  95     Http::HdrType::WWW_AUTHENTICATE,
  96     Http::HdrType::AUTHENTICATION_INFO,
  97     Http::HdrType::PROXY_AUTHENTICATION_INFO,
  98     /* Http::HdrType::TE, Http::HdrType::TRAILER */
  99 #if X_ACCELERATOR_VARY
 100     Http::HdrType::HDR_X_ACCELERATOR_VARY,
 101 #endif
 102 #if USE_ADAPTATION
 103     Http::HdrType::X_NEXT_SERVICES,
 104 #endif
 105     Http::HdrType::SURROGATE_CAPABILITY,
 106     Http::HdrType::SURROGATE_CONTROL,
 107     Http::HdrType::FORWARDED,
 108     Http::HdrType::X_FORWARDED_FOR
 109 };
 110
 111 /* general-headers */
 112 static Http::HdrType GeneralHeadersArr[] = {
 113     Http::HdrType::CACHE_CONTROL,
 114     Http::HdrType::CONNECTION,
 115     Http::HdrType::DATE,
 116     Http::HdrType::FORWARDED,
 117     Http::HdrType::X_FORWARDED_FOR,
 118     Http::HdrType::MIME_VERSION,
 119     Http::HdrType::PRAGMA,
 120     Http::HdrType::PROXY_CONNECTION,
 121     Http::HdrType::TRANSFER_ENCODING,
 122     Http::HdrType::UPGRADE,
 123     /* Http::HdrType::TRAILER, */
 124     Http::HdrType::VIA,
 125 };
 126
 127 /* entity-headers */
 128 static Http::HdrType EntityHeadersArr[] = {
 129     Http::HdrType::ALLOW,
 130     Http::HdrType::CONTENT_BASE,
 131     Http::HdrType::CONTENT_ENCODING,
 132     Http::HdrType::CONTENT_LANGUAGE,
 133     Http::HdrType::CONTENT_LENGTH,
 134     Http::HdrType::CONTENT_LOCATION,
 135     Http::HdrType::CONTENT_MD5,
 136     Http::HdrType::CONTENT_RANGE,
 137     Http::HdrType::CONTENT_TYPE,
 138     Http::HdrType::ETAG,
 139     Http::HdrType::EXPIRES,
 140     Http::HdrType::LAST_MODIFIED,
 141     Http::HdrType::LINK,
 142     Http::HdrType::OTHER
 143 };
 144
 145 /* request-only headers */
 146 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
 147 static Http::HdrType RequestHeadersArr[] = {
 148     Http::HdrType::ACCEPT,
 149     Http::HdrType::ACCEPT_CHARSET,
 150     Http::HdrType::ACCEPT_ENCODING,
 151     Http::HdrType::ACCEPT_LANGUAGE,
 152     Http::HdrType::AUTHORIZATION,
 153     Http::HdrType::EXPECT,
 154     Http::HdrType::FROM,
 155     Http::HdrType::HOST,
 156     Http::HdrType::HTTP2_SETTINGS,
 157     Http::HdrType::IF_MATCH,
 158     Http::HdrType::IF_MODIFIED_SINCE,
 159     Http::HdrType::IF_NONE_MATCH,
 160     Http::HdrType::IF_RANGE,
 161     Http::HdrType::IF_UNMODIFIED_SINCE,
 162     Http::HdrType::MAX_FORWARDS,
 163     Http::HdrType::ORIGIN,
 164     Http::HdrType::PROXY_AUTHORIZATION,
 165     Http::HdrType::RANGE,
 166     Http::HdrType::REFERER,
 167     Http::HdrType::REQUEST_RANGE,
 168     Http::HdrType::TE,
 169     Http::HdrType::USER_AGENT,
 170     Http::HdrType::SURROGATE_CAPABILITY
 171 };
 172
 173 /* reply-only headers */
 174 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
 175 static Http::HdrType ReplyHeadersArr[] = {
 176     Http::HdrType::ACCEPT_ENCODING,
 177     Http::HdrType::ACCEPT_RANGES,
 178     Http::HdrType::AGE,
 179     Http::HdrType::KEY,
 180     Http::HdrType::LOCATION,
 181     Http::HdrType::PROXY_AUTHENTICATE,
 182     Http::HdrType::PUBLIC,
 183     Http::HdrType::RETRY_AFTER,
 184     Http::HdrType::SERVER,
 185     Http::HdrType::SET_COOKIE,
 186     Http::HdrType::SET_COOKIE2,
 187     Http::HdrType::VARY,
 188     Http::HdrType::WARNING,
 189     Http::HdrType::WWW_AUTHENTICATE,
 190     Http::HdrType::X_CACHE,
 191     Http::HdrType::X_CACHE_LOOKUP,
 192     Http::HdrType::X_REQUEST_URI,
 193 #if X_ACCELERATOR_VARY
 194     Http::HdrType::HDR_X_ACCELERATOR_VARY,
 195 #endif
 196 #if USE_ADAPTATION
 197     Http::HdrType::X_NEXT_SERVICES,
 198 #endif
 199     Http::HdrType::X_SQUID_ERROR,
 200     Http::HdrType::SURROGATE_CONTROL
 201 };
 202
 203 /* hop-by-hop headers */
 204 static HttpHeaderMask HopByHopHeadersMask;
 205 static Http::HdrType HopByHopHeadersArr[] = {
 206     Http::HdrType::ALTERNATE_PROTOCOL,
 207     Http::HdrType::CONNECTION,
 208     Http::HdrType::HTTP2_SETTINGS,
 209     Http::HdrType::KEEP_ALIVE,
 210     /*Http::HdrType::PROXY_AUTHENTICATE, // removal handled specially for peer login */
 211     Http::HdrType::PROXY_AUTHORIZATION,
 212     Http::HdrType::TE,
 213     Http::HdrType::TRAILER,
 214     Http::HdrType::TRANSFER_ENCODING,
 215     Http::HdrType::UPGRADE,
 216     Http::HdrType::PROXY_CONNECTION
 217 };
 218
 219 /* header accounting */
 220 // NP: keep in sync with enum http_hdr_owner_type
 221 static HttpHeaderStat HttpHeaderStats[] = {
 222     HttpHeaderStat(/*hoNone*/ "all", NULL),
 223 #if USE_HTCP
 224     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
 225 #endif
 226     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
 227     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
 228 #if USE_OPENSSL
 229     /* hoErrorDetail */
 230 #endif
 231     /* hoEnd */
 232 };
 233 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 234
 235 static int HeaderEntryParsedCount = 0;
 236
 237 /*
 238  * forward declarations and local routines
 239  */
 240
 241 class StoreEntry;
 242
 243 static void httpHeaderNoteParsedEntry(Http::HdrType id, String const &value, int error);
 244 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 245 /** store report about current header usage and other stats */
 246 static void httpHeaderStoreReport(StoreEntry * e);
 247
 248 /*
 249  * Module initialization routines
 250  */
 251
 252 static void
 253 httpHeaderRegisterWithCacheManager(void)
 254 {
 255     Mgr::RegisterAction("http_headers",
 256                         "HTTP Header Statistics",
 257                         httpHeaderStoreReport, 0, 1);
 258 }
 259
 260 void
 261 httpHeaderInitModule(void)
 262 {
 263     /* check that we have enough space for masks */
 264     assert(8 * sizeof(HttpHeaderMask) >= Http::HdrType::ENUM_END);
 265
 266     // check invariant: for each index in headerTable, (int)headerTable[index] = index
 267     for (int i = 0; Http::HeaderTable[i].name; ++i)
 268         assert(Http::HeaderTable[i].id == i);
 269
 270     /* create masks. XXX: migrate to std::vector<bool>? */
 271     httpHeaderMaskInit(&ListHeadersMask, 0);
 272     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 273
 274     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 275     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 276     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 277     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 278
 279     httpHeaderMaskInit(&RequestHeadersMask, 0);
 280     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 281     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 282     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 283
 284     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 285     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 286
 287     /* header stats initialized by class constructor */
 288     assert(HttpHeaderStatCount == hoReply + 1);
 289
 290     /* init dependent modules */
 291     httpHdrCcInitModule();
 292     httpHdrScInitModule();
 293
 294     httpHeaderRegisterWithCacheManager();
 295 }
 296
 297 /*
 298  * HttpHeader Implementation
 299  */
 300
 301 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 302 {
 303     httpHeaderMaskInit(&mask, 0);
 304 }
 305
 306 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 307 {
 308     assert(anOwner > hoNone && anOwner < hoEnd);
 309     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 310     httpHeaderMaskInit(&mask, 0);
 311 }
 312
 313 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 314 {
 315     httpHeaderMaskInit(&mask, 0);
 316     update(&other, NULL); // will update the mask as well
 317 }
 318
 319 HttpHeader::~HttpHeader()
 320 {
 321     clean();
 322 }
 323
 324 HttpHeader &
 325 HttpHeader::operator =(const HttpHeader &other)
 326 {
 327     if (this != &other) {
 328         // we do not really care, but the caller probably does
 329         assert(owner == other.owner);
 330         clean();
 331         update(&other, NULL); // will update the mask as well
 332         len = other.len;
 333     }
 334     return *this;
 335 }
 336
 337 void
 338 HttpHeader::clean()
 339 {
 340
 341     assert(owner > hoNone && owner < hoEnd);
 342     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 343
 344     PROF_start(HttpHeaderClean);
 345
 346     if (owner <= hoReply) {
 347         /*
 348          * An unfortunate bug.  The entries array is initialized
 349          * such that count is set to zero.  httpHeaderClean() seems to
 350          * be called both when 'hdr' is created, and destroyed.  Thus,
 351          * we accumulate a large number of zero counts for 'hdr' before
 352          * it is ever used.  Can't think of a good way to fix it, except
 353          * adding a state variable that indicates whether or not 'hdr'
 354          * has been used.  As a hack, just never count zero-sized header
 355          * arrays.
 356          */
 357         if (!entries.empty())
 358             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 359
 360         ++ HttpHeaderStats[owner].destroyedCount;
 361
 362         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 363     } // if (owner <= hoReply)
 364
 365     for(HttpHeaderEntry *e : entries) {
 366         if (e == nullptr)
 367             continue;
 368         if (!Http::any_valid_header(e->id)) {
 369             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 370         } else {
 371             if (owner <= hoReply)
 372                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 373             delete e;
 374         }
 375     }
 376
 377     entries.clear();
 378     httpHeaderMaskInit(&mask, 0);
 379     len = 0;
 380     PROF_stop(HttpHeaderClean);
 381 }
 382
 383 /* append entries (also see httpHeaderUpdate) */
 384 void
 385 HttpHeader::append(const HttpHeader * src)
 386 {
 387     const HttpHeaderEntry *e;
 388     HttpHeaderPos pos = HttpHeaderInitPos;
 389     assert(src);
 390     assert(src != this);
 391     debugs(55, 7, "appending hdr: " << this << " += " << src);
 392
 393     while ((e = src->getEntry(&pos))) {
 394         addEntry(e->clone());
 395     }
 396 }
 397
 398 /* use fresh entries to replace old ones */
 399 void
 400 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 401 {
 402     assert (old);
 403     old->update (fresh, denied_mask);
 404 }
 405
 406 void
 407 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 408 {
 409     const HttpHeaderEntry *e;
 410     HttpHeaderPos pos = HttpHeaderInitPos;
 411     assert(fresh);
 412     assert(this != fresh);
 413
 414     while ((e = fresh->getEntry(&pos))) {
 415         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 416
 417         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 418             continue;
 419
 420         if (e->id != Http::HdrType::OTHER)
 421             delById(e->id);
 422         else
 423             delByName(e->name.termedBuf());
 424     }
 425
 426     pos = HttpHeaderInitPos;
 427     while ((e = fresh->getEntry(&pos))) {
 428         /* deny bad guys (ok to check for Http::HdrType::OTHER) here */
 429
 430         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 431             continue;
 432
 433         debugs(55, 7, "Updating header '" << Http::HeaderTable[e->id].name << "' in cached entry");
 434
 435         addEntry(e->clone());
 436     }
 437 }
 438
 439 /* just handy in parsing: resets and returns false */
 440 int
 441 HttpHeader::reset()
 442 {
 443     clean();
 444     return 0;
 445 }
 446
 447 int
 448 HttpHeader::parse(const char *header_start, size_t hdrLen)
 449 {
 450     const char *field_ptr = header_start;
 451     const char *header_end = header_start + hdrLen; // XXX: remove
 452     HttpHeaderEntry *e, *e2;
 453     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 454
 455     PROF_start(HttpHeaderParse);
 456
 457     assert(header_start && header_end);
 458     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 459     ++ HttpHeaderStats[owner].parsedCount;
 460
 461     char *nulpos;
 462     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 463         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 464                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 465         PROF_stop(HttpHeaderParse);
 466         return reset();
 467     }
 468
 469     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 470      * continuation lines start with a (single) space or tab */
 471     while (field_ptr < header_end) {
 472         const char *field_start = field_ptr;
 473         const char *field_end;
 474
 475         do {
 476             const char *this_line = field_ptr;
 477             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 478
 479             if (!field_ptr) {
 480                 // missing <LF>
 481                 PROF_stop(HttpHeaderParse);
 482                 return reset();
 483             }
 484
 485             field_end = field_ptr;
 486
 487             ++field_ptr;    /* Move to next line */
 488
 489             if (field_end > this_line && field_end[-1] == '\r') {
 490                 --field_end;    /* Ignore CR LF */
 491
 492                 if (owner == hoRequest && field_end > this_line) {
 493                     bool cr_only = true;
 494                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 495                         if (*p != '\r')
 496                             cr_only = false;
 497                     }
 498                     if (cr_only) {
 499                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 500                                "header field to prevent request smuggling attacks: {" <<
 501                                getStringPrefix(header_start, hdrLen) << "}");
 502                         PROF_stop(HttpHeaderParse);
 503                         return reset();
 504                     }
 505                 }
 506             }
 507
 508             /* Barf on stray CR characters */
 509             if (memchr(this_line, '\r', field_end - this_line)) {
 510                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 511                        getStringPrefix(field_start, field_end-field_start) << "}");
 512
 513                 if (Config.onoff.relaxed_header_parser) {
 514                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 515
 516                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 517                         *p = ' ';
 518                         ++p;
 519                     }
 520                 } else {
 521                     PROF_stop(HttpHeaderParse);
 522                     return reset();
 523                 }
 524             }
 525
 526             if (this_line + 1 == field_end && this_line > field_start) {
 527                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 528                        getStringPrefix(header_start, hdrLen) << "}");
 529                 PROF_stop(HttpHeaderParse);
 530                 return reset();
 531             }
 532         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 533
 534         if (field_start == field_end) {
 535             if (field_ptr < header_end) {
 536                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 537                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 538                 PROF_stop(HttpHeaderParse);
 539                 return reset();
 540             }
 541
 542             break;      /* terminating blank line */
 543         }
 544
 545         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 546             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 547                    getStringPrefix(field_start, field_end-field_start) << "}");
 548             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 549
 550             if (Config.onoff.relaxed_header_parser)
 551                 continue;
 552
 553             PROF_stop(HttpHeaderParse);
 554             return reset();
 555         }
 556
 557         if (e->id == Http::HdrType::CONTENT_LENGTH && (e2 = findEntry(e->id)) != nullptr) {
 558             if (e->value != e2->value) {
 559                 int64_t l1, l2;
 560                 debugs(55, warnOnError, "WARNING: found two conflicting content-length headers in {" <<
 561                        getStringPrefix(header_start, hdrLen) << "}");
 562
 563                 if (!Config.onoff.relaxed_header_parser) {
 564                     delete e;
 565                     PROF_stop(HttpHeaderParse);
 566                     return reset();
 567                 }
 568
 569                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 570                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e->value << "'");
 571                     delete e;
 572                     continue;
 573                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 574                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e2->value << "'");
 575                     delById(e2->id);
 576                 } else if (l1 > l2) {
 577                     delById(e2->id);
 578                 } else {
 579                     delete e;
 580                     continue;
 581                 }
 582             } else {
 583                 debugs(55, warnOnError, "NOTICE: found double content-length header");
 584                 delete e;
 585
 586                 if (Config.onoff.relaxed_header_parser)
 587                     continue;
 588
 589                 PROF_stop(HttpHeaderParse);
 590                 return reset();
 591             }
 592         }
 593
 594         if (e->id == Http::HdrType::OTHER && stringHasWhitespace(e->name.termedBuf())) {
 595             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 596                    getStringPrefix(field_start, field_end-field_start) << "}");
 597
 598             if (!Config.onoff.relaxed_header_parser) {
 599                 delete e;
 600                 PROF_stop(HttpHeaderParse);
 601                 return reset();
 602             }
 603         }
 604
 605         addEntry(e);
 606     }
 607
 608     if (chunked()) {
 609         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 610         delById(Http::HdrType::CONTENT_LENGTH);
 611     }
 612
 613     PROF_stop(HttpHeaderParse);
 614     return 1;           /* even if no fields where found, it is a valid header */
 615 }
 616
 617 /* packs all the entries using supplied packer */
 618 void
 619 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 620 {
 621     HttpHeaderPos pos = HttpHeaderInitPos;
 622     const HttpHeaderEntry *e;
 623     assert(p);
 624     debugs(55, 7, this << " into " << p <<
 625            (mask_sensitive_info ? " while masking" : ""));
 626     /* pack all entries one by one */
 627     while ((e = getEntry(&pos))) {
 628         if (!mask_sensitive_info) {
 629             e->packInto(p);
 630             continue;
 631         }
 632
 633         bool maskThisEntry = false;
 634         switch (e->id) {
 635         case Http::HdrType::AUTHORIZATION:
 636         case Http::HdrType::PROXY_AUTHORIZATION:
 637             maskThisEntry = true;
 638             break;
 639
 640         case Http::HdrType::FTP_ARGUMENTS:
 641             if (const HttpHeaderEntry *cmd = findEntry(Http::HdrType::FTP_COMMAND))
 642                 maskThisEntry = (cmd->value == "PASS");
 643             break;
 644
 645         default:
 646             break;
 647         }
 648         if (maskThisEntry) {
 649             p->append(e->name.rawBuf(), e->name.size());
 650             p->append(": ** NOT DISPLAYED **\r\n", 23);
 651         } else {
 652             e->packInto(p);
 653         }
 654
 655     }
 656     /* Pack in the "special" entries */
 657
 658     /* Cache-Control */
 659 }
 660
 661 /* returns next valid entry */
 662 HttpHeaderEntry *
 663 HttpHeader::getEntry(HttpHeaderPos * pos) const
 664 {
 665     assert(pos);
 666     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 667
 668     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 669         if (entries[*pos])
 670             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 671     }
 672
 673     return NULL;
 674 }
 675
 676 /*
 677  * returns a pointer to a specified entry if any
 678  * note that we return one entry so it does not make much sense to ask for
 679  * "list" headers
 680  */
 681 HttpHeaderEntry *
 682 HttpHeader::findEntry(Http::HdrType id) const
 683 {
 684     HttpHeaderPos pos = HttpHeaderInitPos;
 685     HttpHeaderEntry *e;
 686     assert(any_valid_header(id));
 687     assert(!CBIT_TEST(ListHeadersMask, id));
 688
 689     /* check mask first */
 690
 691     if (!CBIT_TEST(mask, id))
 692         return NULL;
 693
 694     /* looks like we must have it, do linear search */
 695     while ((e = getEntry(&pos))) {
 696         if (e->id == id)
 697             return e;
 698     }
 699
 700     /* hm.. we thought it was there, but it was not found */
 701     assert(0);
 702
 703     return NULL;        /* not reached */
 704 }
 705
 706 /*
 707  * same as httpHeaderFindEntry
 708  */
 709 HttpHeaderEntry *
 710 HttpHeader::findLastEntry(Http::HdrType id) const
 711 {
 712     HttpHeaderPos pos = HttpHeaderInitPos;
 713     HttpHeaderEntry *e;
 714     HttpHeaderEntry *result = NULL;
 715     assert(any_valid_header(id));
 716     assert(!CBIT_TEST(ListHeadersMask, id));
 717
 718     /* check mask first */
 719
 720     if (!CBIT_TEST(mask, id))
 721         return NULL;
 722
 723     /* looks like we must have it, do linear search */
 724     while ((e = getEntry(&pos))) {
 725         if (e->id == id)
 726             result = e;
 727     }
 728
 729     assert(result);     /* must be there! */
 730     return result;
 731 }
 732
 733 /*
 734  * deletes all fields with a given name if any, returns #fields deleted;
 735  */
 736 int
 737 HttpHeader::delByName(const char *name)
 738 {
 739     int count = 0;
 740     HttpHeaderPos pos = HttpHeaderInitPos;
 741     HttpHeaderEntry *e;
 742     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 743     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 744
 745     while ((e = getEntry(&pos))) {
 746         if (!e->name.caseCmp(name))
 747             delAt(pos, count);
 748         else
 749             CBIT_SET(mask, e->id);
 750     }
 751
 752     return count;
 753 }
 754
 755 /* deletes all entries with a given id, returns the #entries deleted */
 756 int
 757 HttpHeader::delById(Http::HdrType id)
 758 {
 759     int count = 0;
 760     HttpHeaderPos pos = HttpHeaderInitPos;
 761     HttpHeaderEntry *e;
 762     debugs(55, 8, this << " del-by-id " << id);
 763     assert(any_valid_header(id));
 764     assert(id != Http::HdrType::OTHER);        /* does not make sense */
 765
 766     if (!CBIT_TEST(mask, id))
 767         return 0;
 768
 769     while ((e = getEntry(&pos))) {
 770         if (e->id == id)
 771             delAt(pos, count);
 772     }
 773
 774     CBIT_CLR(mask, id);
 775     assert(count);
 776     return count;
 777 }
 778
 779 /*
 780  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 781  * possible to iterate(search) and delete fields at the same time
 782  * NOTE: Does not update the header mask. Caller must follow up with
 783  * a call to refreshMask() if headers_deleted was incremented.
 784  */
 785 void
 786 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 787 {
 788     HttpHeaderEntry *e;
 789     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 790     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 791     entries[pos] = NULL;
 792     /* decrement header length, allow for ": " and crlf */
 793     len -= e->name.size() + 2 + e->value.size() + 2;
 794     assert(len >= 0);
 795     delete e;
 796     ++headers_deleted;
 797 }
 798
 799 /*
 800  * Compacts the header storage
 801  */
 802 void
 803 HttpHeader::compact()
 804 {
 805     // TODO: optimize removal, or possibly make it so that's not needed.
 806     std::vector<HttpHeaderEntry *>::iterator newend;
 807     newend = std::remove(entries.begin(), entries.end(), static_cast<HttpHeaderEntry *>(NULL));
 808     entries.resize(newend-entries.begin());
 809 }
 810
 811 /*
 812  * Refreshes the header mask. Required after delAt() calls.
 813  */
 814 void
 815 HttpHeader::refreshMask()
 816 {
 817     httpHeaderMaskInit(&mask, 0);
 818     debugs(55, 7, "refreshing the mask in hdr " << this);
 819     HttpHeaderPos pos = HttpHeaderInitPos;
 820     while (HttpHeaderEntry *e = getEntry(&pos)) {
 821         CBIT_SET(mask, e->id);
 822     }
 823 }
 824
 825 /* appends an entry;
 826  * does not call e->clone() so one should not reuse "*e"
 827  */
 828 void
 829 HttpHeader::addEntry(HttpHeaderEntry * e)
 830 {
 831     assert(e);
 832     assert(any_HdrType_enum_value(e->id));
 833     assert(e->name.size());
 834
 835     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 836
 837     if (e->id != Http::HdrType::BAD_HDR) {
 838         if (CBIT_TEST(mask, e->id)) {
 839             ++ headerStatsTable[e->id].repCount;
 840         } else {
 841             CBIT_SET(mask, e->id);
 842         }
 843     }
 844
 845     entries.push_back(e);
 846
 847     /* increment header length, allow for ": " and crlf */
 848     len += e->name.size() + 2 + e->value.size() + 2;
 849 }
 850
 851 /* inserts an entry;
 852  * does not call e->clone() so one should not reuse "*e"
 853  */
 854 void
 855 HttpHeader::insertEntry(HttpHeaderEntry * e)
 856 {
 857     assert(e);
 858     assert(any_valid_header(e->id));
 859
 860     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 861
 862     // Http::HdrType::BAD_HDR is filtered out by assert_any_valid_header
 863     if (CBIT_TEST(mask, e->id)) {
 864         ++ headerStatsTable[e->id].repCount;
 865     } else {
 866         CBIT_SET(mask, e->id);
 867     }
 868
 869     entries.insert(entries.begin(),e);
 870
 871     /* increment header length, allow for ": " and crlf */
 872     len += e->name.size() + 2 + e->value.size() + 2;
 873 }
 874
 875 bool
 876 HttpHeader::getList(Http::HdrType id, String *s) const
 877 {
 878     HttpHeaderEntry *e;
 879     HttpHeaderPos pos = HttpHeaderInitPos;
 880     debugs(55, 9, this << " joining for id " << id);
 881     /* only fields from ListHeaders array can be "listed" */
 882     assert(CBIT_TEST(ListHeadersMask, id));
 883
 884     if (!CBIT_TEST(mask, id))
 885         return false;
 886
 887     while ((e = getEntry(&pos))) {
 888         if (e->id == id)
 889             strListAdd(s, e->value.termedBuf(), ',');
 890     }
 891
 892     /*
 893      * note: we might get an empty (size==0) string if there was an "empty"
 894      * header. This results in an empty length String, which may have a NULL
 895      * buffer.
 896      */
 897     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 898     if (!s->size())
 899         debugs(55, 3, "empty list header: " << Http::HeaderTable[id].name << "(" << id << ")");
 900     else
 901         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 902
 903     return true;
 904 }
 905
 906 /* return a list of entries with the same id separated by ',' and ws */
 907 String
 908 HttpHeader::getList(Http::HdrType id) const
 909 {
 910     HttpHeaderEntry *e;
 911     HttpHeaderPos pos = HttpHeaderInitPos;
 912     debugs(55, 9, this << "joining for id " << id);
 913     /* only fields from ListHeaders array can be "listed" */
 914     assert(CBIT_TEST(ListHeadersMask, id));
 915
 916     if (!CBIT_TEST(mask, id))
 917         return String();
 918
 919     String s;
 920
 921     while ((e = getEntry(&pos))) {
 922         if (e->id == id)
 923             strListAdd(&s, e->value.termedBuf(), ',');
 924     }
 925
 926     /*
 927      * note: we might get an empty (size==0) string if there was an "empty"
 928      * header. This results in an empty length String, which may have a NULL
 929      * buffer.
 930      */
 931     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 932     if (!s.size())
 933         debugs(55, 3, "empty list header: " << Http::HeaderTable[id].name << "(" << id << ")");
 934     else
 935         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 936
 937     return s;
 938 }
 939
 940 /* return a string or list of entries with the same id separated by ',' and ws */
 941 String
 942 HttpHeader::getStrOrList(Http::HdrType id) const
 943 {
 944     HttpHeaderEntry *e;
 945
 946     if (CBIT_TEST(ListHeadersMask, id))
 947         return getList(id);
 948
 949     if ((e = findEntry(id)))
 950         return e->value;
 951
 952     return String();
 953 }
 954
 955 /*
 956  * Returns the value of the specified header and/or an undefined String.
 957  */
 958 String
 959 HttpHeader::getByName(const char *name) const
 960 {
 961     String result;
 962     // ignore presence: return undefined string if an empty header is present
 963     (void)getByNameIfPresent(name, result);
 964     return result;
 965 }
 966
 967 bool
 968 HttpHeader::getByNameIfPresent(const char *name, String &result) const
 969 {
 970     Http::HdrType id;
 971     HttpHeaderPos pos = HttpHeaderInitPos;
 972     HttpHeaderEntry *e;
 973
 974     assert(name);
 975
 976     /* First try the quick path */
 977     id = Http::HeaderLookupTable.lookup(SBuf(name));
 978
 979     if (id != Http::HdrType::BAD_HDR) {
 980         if (!has(id))
 981             return false;
 982         result = getStrOrList(id);
 983         return true;
 984     }
 985
 986     /* Sorry, an unknown header name. Do linear search */
 987     bool found = false;
 988     while ((e = getEntry(&pos))) {
 989         if (e->id == Http::HdrType::OTHER && e->name.caseCmp(name) == 0) {
 990             found = true;
 991             strListAdd(&result, e->value.termedBuf(), ',');
 992         }
 993     }
 994
 995     return found;
 996 }
 997
 998 /*
 999  * Returns a the value of the specified list member, if any.
1000  */
1001 String
1002 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1003 {
1004     String header;
1005     const char *pos = NULL;
1006     const char *item;
1007     int ilen;
1008     int mlen = strlen(member);
1009
1010     assert(name);
1011
1012     header = getByName(name);
1013
1014     String result;
1015
1016     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1017         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1018             result.append(item + mlen + 1, ilen - mlen - 1);
1019             break;
1020         }
1021     }
1022
1023     return result;
1024 }
1025
1026 /*
1027  * returns a the value of the specified list member, if any.
1028  */
1029 String
1030 HttpHeader::getListMember(Http::HdrType id, const char *member, const char separator) const
1031 {
1032     String header;
1033     const char *pos = NULL;
1034     const char *item;
1035     int ilen;
1036     int mlen = strlen(member);
1037
1038     assert(any_valid_header(id));
1039
1040     header = getStrOrList(id);
1041     String result;
1042
1043     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1044         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1045             result.append(item + mlen + 1, ilen - mlen - 1);
1046             break;
1047         }
1048     }
1049
1050     header.clean();
1051     return result;
1052 }
1053
1054 /* test if a field is present */
1055 int
1056 HttpHeader::has(Http::HdrType id) const
1057 {
1058     assert(any_valid_header(id));
1059     assert(id != Http::HdrType::OTHER);
1060     debugs(55, 9, this << " lookup for " << id);
1061     return CBIT_TEST(mask, id);
1062 }
1063
1064 void
1065 HttpHeader::putInt(Http::HdrType id, int number)
1066 {
1067     assert(any_valid_header(id));
1068     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1069     assert(number >= 0);
1070     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1071 }
1072
1073 void
1074 HttpHeader::putInt64(Http::HdrType id, int64_t number)
1075 {
1076     assert(any_valid_header(id));
1077     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1078     assert(number >= 0);
1079     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1080 }
1081
1082 void
1083 HttpHeader::putTime(Http::HdrType id, time_t htime)
1084 {
1085     assert(any_valid_header(id));
1086     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1087     assert(htime >= 0);
1088     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1089 }
1090
1091 void
1092 HttpHeader::insertTime(Http::HdrType id, time_t htime)
1093 {
1094     assert(any_valid_header(id));
1095     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1096     assert(htime >= 0);
1097     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1098 }
1099
1100 void
1101 HttpHeader::putStr(Http::HdrType id, const char *str)
1102 {
1103     assert(any_valid_header(id));
1104     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1105     assert(str);
1106     addEntry(new HttpHeaderEntry(id, NULL, str));
1107 }
1108
1109 void
1110 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1111 {
1112     assert(auth_scheme && realm);
1113     httpHeaderPutStrf(this, Http::HdrType::WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1114 }
1115
1116 void
1117 HttpHeader::putCc(const HttpHdrCc * cc)
1118 {
1119     assert(cc);
1120     /* remove old directives if any */
1121     delById(Http::HdrType::CACHE_CONTROL);
1122     /* pack into mb */
1123     MemBuf mb;
1124     mb.init();
1125     cc->packInto(&mb);
1126     /* put */
1127     addEntry(new HttpHeaderEntry(Http::HdrType::CACHE_CONTROL, NULL, mb.buf));
1128     /* cleanup */
1129     mb.clean();
1130 }
1131
1132 void
1133 HttpHeader::putContRange(const HttpHdrContRange * cr)
1134 {
1135     assert(cr);
1136     /* remove old directives if any */
1137     delById(Http::HdrType::CONTENT_RANGE);
1138     /* pack into mb */
1139     MemBuf mb;
1140     mb.init();
1141     httpHdrContRangePackInto(cr, &mb);
1142     /* put */
1143     addEntry(new HttpHeaderEntry(Http::HdrType::CONTENT_RANGE, NULL, mb.buf));
1144     /* cleanup */
1145     mb.clean();
1146 }
1147
1148 void
1149 HttpHeader::putRange(const HttpHdrRange * range)
1150 {
1151     assert(range);
1152     /* remove old directives if any */
1153     delById(Http::HdrType::RANGE);
1154     /* pack into mb */
1155     MemBuf mb;
1156     mb.init();
1157     range->packInto(&mb);
1158     /* put */
1159     addEntry(new HttpHeaderEntry(Http::HdrType::RANGE, NULL, mb.buf));
1160     /* cleanup */
1161     mb.clean();
1162 }
1163
1164 void
1165 HttpHeader::putSc(HttpHdrSc *sc)
1166 {
1167     assert(sc);
1168     /* remove old directives if any */
1169     delById(Http::HdrType::SURROGATE_CONTROL);
1170     /* pack into mb */
1171     MemBuf mb;
1172     mb.init();
1173     sc->packInto(&mb);
1174     /* put */
1175     addEntry(new HttpHeaderEntry(Http::HdrType::SURROGATE_CONTROL, NULL, mb.buf));
1176     /* cleanup */
1177     mb.clean();
1178 }
1179
1180 void
1181 HttpHeader::putWarning(const int code, const char *const text)
1182 {
1183     char buf[512];
1184     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1185     putStr(Http::HdrType::WARNING, buf);
1186 }
1187
1188 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1189 void
1190 HttpHeader::putExt(const char *name, const char *value)
1191 {
1192     assert(name && value);
1193     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1194     addEntry(new HttpHeaderEntry(Http::HdrType::OTHER, name, value));
1195 }
1196
1197 int
1198 HttpHeader::getInt(Http::HdrType id) const
1199 {
1200     assert(any_valid_header(id));
1201     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftInt);  /* must be of an appropriate type */
1202     HttpHeaderEntry *e;
1203
1204     if ((e = findEntry(id)))
1205         return e->getInt();
1206
1207     return -1;
1208 }
1209
1210 int64_t
1211 HttpHeader::getInt64(Http::HdrType id) const
1212 {
1213     assert(any_valid_header(id));
1214     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftInt64);    /* must be of an appropriate type */
1215     HttpHeaderEntry *e;
1216
1217     if ((e = findEntry(id)))
1218         return e->getInt64();
1219
1220     return -1;
1221 }
1222
1223 time_t
1224 HttpHeader::getTime(Http::HdrType id) const
1225 {
1226     HttpHeaderEntry *e;
1227     time_t value = -1;
1228     assert(any_valid_header(id));
1229     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftDate_1123);    /* must be of an appropriate type */
1230
1231     if ((e = findEntry(id))) {
1232         value = parse_rfc1123(e->value.termedBuf());
1233         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1234     }
1235
1236     return value;
1237 }
1238
1239 /* sync with httpHeaderGetLastStr */
1240 const char *
1241 HttpHeader::getStr(Http::HdrType id) const
1242 {
1243     HttpHeaderEntry *e;
1244     assert(any_valid_header(id));
1245     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1246
1247     if ((e = findEntry(id))) {
1248         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1249         return e->value.termedBuf();
1250     }
1251
1252     return NULL;
1253 }
1254
1255 /* unusual */
1256 const char *
1257 HttpHeader::getLastStr(Http::HdrType id) const
1258 {
1259     HttpHeaderEntry *e;
1260     assert(any_valid_header(id));
1261     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftStr);  /* must be of an appropriate type */
1262
1263     if ((e = findLastEntry(id))) {
1264         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1265         return e->value.termedBuf();
1266     }
1267
1268     return NULL;
1269 }
1270
1271 HttpHdrCc *
1272 HttpHeader::getCc() const
1273 {
1274     if (!CBIT_TEST(mask, Http::HdrType::CACHE_CONTROL))
1275         return NULL;
1276     PROF_start(HttpHeader_getCc);
1277
1278     String s;
1279     getList(Http::HdrType::CACHE_CONTROL, &s);
1280
1281     HttpHdrCc *cc=new HttpHdrCc();
1282
1283     if (!cc->parse(s)) {
1284         delete cc;
1285         cc = NULL;
1286     }
1287
1288     ++ HttpHeaderStats[owner].ccParsedCount;
1289
1290     if (cc)
1291         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1292
1293     httpHeaderNoteParsedEntry(Http::HdrType::CACHE_CONTROL, s, !cc);
1294
1295     PROF_stop(HttpHeader_getCc);
1296
1297     return cc;
1298 }
1299
1300 HttpHdrRange *
1301 HttpHeader::getRange() const
1302 {
1303     HttpHdrRange *r = NULL;
1304     HttpHeaderEntry *e;
1305     /* some clients will send "Request-Range" _and_ *matching* "Range"
1306      * who knows, some clients might send Request-Range only;
1307      * this "if" should work correctly in both cases;
1308      * hopefully no clients send mismatched headers! */
1309
1310     if ((e = findEntry(Http::HdrType::RANGE)) ||
1311             (e = findEntry(Http::HdrType::REQUEST_RANGE))) {
1312         r = HttpHdrRange::ParseCreate(&e->value);
1313         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1314     }
1315
1316     return r;
1317 }
1318
1319 HttpHdrSc *
1320 HttpHeader::getSc() const
1321 {
1322     if (!CBIT_TEST(mask, Http::HdrType::SURROGATE_CONTROL))
1323         return NULL;
1324
1325     String s;
1326
1327     (void) getList(Http::HdrType::SURROGATE_CONTROL, &s);
1328
1329     HttpHdrSc *sc = httpHdrScParseCreate(s);
1330
1331     ++ HttpHeaderStats[owner].ccParsedCount;
1332
1333     if (sc)
1334         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1335
1336     httpHeaderNoteParsedEntry(Http::HdrType::SURROGATE_CONTROL, s, !sc);
1337
1338     return sc;
1339 }
1340
1341 HttpHdrContRange *
1342 HttpHeader::getContRange() const
1343 {
1344     HttpHdrContRange *cr = NULL;
1345     HttpHeaderEntry *e;
1346
1347     if ((e = findEntry(Http::HdrType::CONTENT_RANGE))) {
1348         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1349         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1350     }
1351
1352     return cr;
1353 }
1354
1355 const char *
1356 HttpHeader::getAuth(Http::HdrType id, const char *auth_scheme) const
1357 {
1358     const char *field;
1359     int l;
1360     assert(auth_scheme);
1361     field = getStr(id);
1362
1363     if (!field)         /* no authorization field */
1364         return NULL;
1365
1366     l = strlen(auth_scheme);
1367
1368     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1369         return NULL;
1370
1371     field += l;
1372
1373     if (!xisspace(*field))  /* wrong scheme */
1374         return NULL;
1375
1376     /* skip white space */
1377     for (; field && xisspace(*field); ++field);
1378
1379     if (!*field)        /* no authorization cookie */
1380         return NULL;
1381
1382     static char decodedAuthToken[8192];
1383     struct base64_decode_ctx ctx;
1384     base64_decode_init(&ctx);
1385     size_t decodedLen = 0;
1386     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1387             !base64_decode_final(&ctx)) {
1388         return NULL;
1389     }
1390     decodedAuthToken[decodedLen] = '\0';
1391     return decodedAuthToken;
1392 }
1393
1394 ETag
1395 HttpHeader::getETag(Http::HdrType id) const
1396 {
1397     ETag etag = {NULL, -1};
1398     HttpHeaderEntry *e;
1399     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftETag);     /* must be of an appropriate type */
1400
1401     if ((e = findEntry(id)))
1402         etagParseInit(&etag, e->value.termedBuf());
1403
1404     return etag;
1405 }
1406
1407 TimeOrTag
1408 HttpHeader::getTimeOrTag(Http::HdrType id) const
1409 {
1410     TimeOrTag tot;
1411     HttpHeaderEntry *e;
1412     assert(Http::HeaderTable[id].type == Http::HdrFieldType::ftDate_1123_or_ETag);    /* must be of an appropriate type */
1413     memset(&tot, 0, sizeof(tot));
1414
1415     if ((e = findEntry(id))) {
1416         const char *str = e->value.termedBuf();
1417         /* try as an ETag */
1418
1419         if (etagParseInit(&tot.tag, str)) {
1420             tot.valid = tot.tag.str != NULL;
1421             tot.time = -1;
1422         } else {
1423             /* or maybe it is time? */
1424             tot.time = parse_rfc1123(str);
1425             tot.valid = tot.time >= 0;
1426             tot.tag.str = NULL;
1427         }
1428     }
1429
1430     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1431     return tot;
1432 }
1433
1434 /*
1435  * HttpHeaderEntry
1436  */
1437
1438 HttpHeaderEntry::HttpHeaderEntry(Http::HdrType anId, const char *aName, const char *aValue)
1439 {
1440     assert(any_HdrType_enum_value(anId));
1441     id = anId;
1442
1443     if (id != Http::HdrType::OTHER)
1444         name = Http::HeaderTable[id].name;
1445     else
1446         name = aName;
1447
1448     value = aValue;
1449
1450     if (id != Http::HdrType::BAD_HDR)
1451         ++ headerStatsTable[id].aliveCount;
1452
1453     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1454 }
1455
1456 HttpHeaderEntry::~HttpHeaderEntry()
1457 {
1458     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1459
1460     if (id != Http::HdrType::BAD_HDR) {
1461         assert(headerStatsTable[id].aliveCount);
1462         -- headerStatsTable[id].aliveCount;
1463         id = Http::HdrType::BAD_HDR; // it already is BAD_HDR, no sense in resetting it
1464     }
1465
1466 }
1467
1468 /* parses and inits header entry, returns true/false */
1469 HttpHeaderEntry *
1470 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1471 {
1472     /* note: name_start == field_start */
1473     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1474     int name_len = name_end ? name_end - field_start :0;
1475     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1476     /* note: value_end == field_end */
1477
1478     ++ HeaderEntryParsedCount;
1479
1480     /* do we have a valid field name within this field? */
1481
1482     if (!name_len || name_end > field_end)
1483         return NULL;
1484
1485     if (name_len > 65534) {
1486         /* String must be LESS THAN 64K and it adds a terminating NULL */
1487         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1488         return NULL;
1489     }
1490
1491     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1492         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1493                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1494
1495         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1496             --name_len;
1497
1498         if (!name_len)
1499             return NULL;
1500     }
1501
1502     /* now we know we can parse it */
1503
1504     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1505
1506     /* is it a "known" field? */
1507     Http::HdrType id = Http::HeaderLookupTable.lookup(SBuf(field_start,name_len));
1508     debugs(55, 9, "got hdr-id=" << id);
1509
1510     String name;
1511
1512     String value;
1513
1514     if (id == Http::HdrType::BAD_HDR)
1515         id = Http::HdrType::OTHER;
1516
1517     /* set field name */
1518     if (id == Http::HdrType::OTHER)
1519         name.limitInit(field_start, name_len);
1520     else
1521         name = Http::HeaderTable[id].name;
1522
1523     /* trim field value */
1524     while (value_start < field_end && xisspace(*value_start))
1525         ++value_start;
1526
1527     while (value_start < field_end && xisspace(field_end[-1]))
1528         --field_end;
1529
1530     if (field_end - value_start > 65534) {
1531         /* String must be LESS THAN 64K and it adds a terminating NULL */
1532         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1533
1534         if (id == Http::HdrType::OTHER)
1535             name.clean();
1536
1537         return NULL;
1538     }
1539
1540     /* set field value */
1541     value.limitInit(value_start, field_end - value_start);
1542
1543     if (id != Http::HdrType::BAD_HDR)
1544         ++ headerStatsTable[id].seenCount;
1545
1546     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1547
1548     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1549 }
1550
1551 HttpHeaderEntry *
1552 HttpHeaderEntry::clone() const
1553 {
1554     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1555 }
1556
1557 void
1558 HttpHeaderEntry::packInto(Packable * p) const
1559 {
1560     assert(p);
1561     p->append(name.rawBuf(), name.size());
1562     p->append(": ", 2);
1563     p->append(value.rawBuf(), value.size());
1564     p->append("\r\n", 2);
1565 }
1566
1567 int
1568 HttpHeaderEntry::getInt() const
1569 {
1570     int val = -1;
1571     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1572     httpHeaderNoteParsedEntry(id, value, !ok);
1573     /* XXX: Should we check ok - ie
1574      * return ok ? -1 : value;
1575      */
1576     return val;
1577 }
1578
1579 int64_t
1580 HttpHeaderEntry::getInt64() const
1581 {
1582     int64_t val = -1;
1583     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1584     httpHeaderNoteParsedEntry(id, value, !ok);
1585     /* XXX: Should we check ok - ie
1586      * return ok ? -1 : value;
1587      */
1588     return val;
1589 }
1590
1591 static void
1592 httpHeaderNoteParsedEntry(Http::HdrType id, String const &context, int error)
1593 {
1594     if (id != Http::HdrType::BAD_HDR)
1595         ++ headerStatsTable[id].parsCount;
1596
1597     if (error) {
1598         if (id != Http::HdrType::BAD_HDR)
1599             ++ headerStatsTable[id].errCount;
1600         debugs(55, 2, "cannot parse hdr field: '" << Http::HeaderTable[id].name << ": " << context << "'");
1601     }
1602 }
1603
1604 /*
1605  * Reports
1606  */
1607
1608 /* tmp variable used to pass stat info to dumpers */
1609 extern const HttpHeaderStat *dump_stat;     /* argh! */
1610 const HttpHeaderStat *dump_stat = NULL;
1611
1612 void
1613 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1614 {
1615     const int id = static_cast<int>(val);
1616     const bool valid_id = Http::any_valid_header(static_cast<Http::HdrType>(id));
1617     const char *name = valid_id ? Http::HeaderTable[id].name : "INVALID";
1618     int visible = count > 0;
1619     /* for entries with zero count, list only those that belong to current type of message */
1620
1621     if (!visible && valid_id && dump_stat->owner_mask)
1622         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1623
1624     if (visible)
1625         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1626                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1627 }
1628
1629 static void
1630 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1631 {
1632     if (count)
1633         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1634                           idx, (int) val, count,
1635                           xpercent(count, dump_stat->destroyedCount));
1636 }
1637
1638 static void
1639 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1640 {
1641     assert(hs);
1642     assert(e);
1643
1644     dump_stat = hs;
1645     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1646     storeAppendPrintf(e, "\nField type distribution\n");
1647     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1648                       "id", "name", "count", "#/header");
1649     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1650     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1651     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1652                       "id", "name", "count", "#/cc_field");
1653     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1654     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1655     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1656                       "id", "name", "count", "#/sc_field");
1657     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1658     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1659     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1660                       "id", "#flds", "count", "%total");
1661     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1662     storeAppendPrintf(e, "\n");
1663     dump_stat = NULL;
1664 }
1665
1666 void
1667 httpHeaderStoreReport(StoreEntry * e)
1668 {
1669     int i;
1670     assert(e);
1671
1672     HttpHeaderStats[0].parsedCount =
1673         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1674     HttpHeaderStats[0].ccParsedCount =
1675         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1676     HttpHeaderStats[0].destroyedCount =
1677         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1678     HttpHeaderStats[0].busyDestroyedCount =
1679         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1680
1681     for (i = 1; i < HttpHeaderStatCount; ++i) {
1682         httpHeaderStatDump(HttpHeaderStats + i, e);
1683     }
1684
1685     /* field stats for all messages */
1686     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1687
1688     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1689                       "id", "name", "#alive", "%err", "%repeat");
1690
1691     // scan heaaderTable and output
1692     for (int j = 0; Http::HeaderTable[j].name != nullptr; ++j) {
1693         auto stats = headerStatsTable[j];
1694         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1695                           Http::HeaderTable[j].id,
1696                           Http::HeaderTable[j].name,
1697                           stats.aliveCount,
1698                           xpercent(stats.errCount, stats.parsCount),
1699                           xpercent(stats.repCount, stats.seenCount));
1700     }
1701
1702     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1703                       HttpHeaderStats[hoRequest].parsedCount,
1704                       HttpHeaderStats[hoReply].parsedCount,
1705                       HttpHeaderStats[0].parsedCount);
1706     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1707 }
1708
1709 int
1710 HttpHeader::hasListMember(Http::HdrType id, const char *member, const char separator) const
1711 {
1712     int result = 0;
1713     const char *pos = NULL;
1714     const char *item;
1715     int ilen;
1716     int mlen = strlen(member);
1717
1718     assert(any_valid_header(id));
1719
1720     String header (getStrOrList(id));
1721
1722     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1723         if (strncasecmp(item, member, mlen) == 0
1724                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1725             result = 1;
1726             break;
1727         }
1728     }
1729
1730     return result;
1731 }
1732
1733 int
1734 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1735 {
1736     int result = 0;
1737     const char *pos = NULL;
1738     const char *item;
1739     int ilen;
1740     int mlen = strlen(member);
1741
1742     assert(name);
1743
1744     String header (getByName(name));
1745
1746     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1747         if (strncasecmp(item, member, mlen) == 0
1748                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1749             result = 1;
1750             break;
1751         }
1752     }
1753
1754     return result;
1755 }
1756
1757 void
1758 HttpHeader::removeHopByHopEntries()
1759 {
1760     removeConnectionHeaderEntries();
1761
1762     const HttpHeaderEntry *e;
1763     HttpHeaderPos pos = HttpHeaderInitPos;
1764     int headers_deleted = 0;
1765     while ((e = getEntry(&pos))) {
1766         Http::HdrType id = e->id;
1767         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1768             delAt(pos, headers_deleted);
1769             CBIT_CLR(mask, id);
1770         }
1771     }
1772 }
1773
1774 void
1775 HttpHeader::removeConnectionHeaderEntries()
1776 {
1777     if (has(Http::HdrType::CONNECTION)) {
1778         /* anything that matches Connection list member will be deleted */
1779         String strConnection;
1780
1781         (void) getList(Http::HdrType::CONNECTION, &strConnection);
1782         const HttpHeaderEntry *e;
1783         HttpHeaderPos pos = HttpHeaderInitPos;
1784         /*
1785          * think: on-average-best nesting of the two loops (hdrEntry
1786          * and strListItem) @?@
1787          */
1788         /*
1789          * maybe we should delete standard stuff ("keep-alive","close")
1790          * from strConnection first?
1791          */
1792
1793         int headers_deleted = 0;
1794         while ((e = getEntry(&pos))) {
1795             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1796                 delAt(pos, headers_deleted);
1797         }
1798         if (headers_deleted)
1799             refreshMask();
1800     }
1801 }
1802