src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/LookupTable.h"
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "HttpHdrCc.h"
  16 #include "HttpHdrContRange.h"
  17 #include "HttpHdrSc.h"
  18 #include "HttpHeader.h"
  19 #include "HttpHeaderFieldInfo.h"
  20 #include "HttpHeaderStat.h"
  21 #include "HttpHeaderTools.h"
  22 #include "MemBuf.h"
  23 #include "mgr/Registration.h"
  24 #include "profiler/Profiler.h"
  25 #include "rfc1123.h"
  26 #include "SquidConfig.h"
  27 #include "SquidString.h"
  28 #include "StatHist.h"
  29 #include "Store.h"
  30 #include "StrList.h"
  31 #include "TimeOrTag.h"
  32 #include "util.h"
  33
  34 #include <algorithm>
  35
  36 /* XXX: the whole set of API managing the entries vector should be rethought
  37  *      after the parse4r-ng effort is complete.
  38  */
  39
  40 /*
  41  * On naming conventions:
  42  *
  43  * HTTP/1.1 defines message-header as
  44  *
  45  * message-header = field-name ":" [ field-value ] CRLF
  46  * field-name     = token
  47  * field-value    = *( field-content | LWS )
  48  *
  49  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  50  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  51  *
  52  * HttpHeader is an object that represents all message-headers in a message.
  53  * HttpHeader does not manage start-line.
  54  *
  55  * HttpHeader is implemented as a collection of header "entries".
  56  * An entry is a (field_id, field_name, field_value) triplet.
  57  */
  58
  59 /*
  60  * local constants and vars
  61  */
  62
  63 /* TODO:
  64  * DONE 1. shift HDR_BAD_HDR to end of enum
  65  * DONE 2. shift headers data array to http/RegistredHeaders.cc
  66  * DONE 3. creatign LookupTable object from teh enum and array
  67  * (with HDR_BAD_HDR as invalid value)
  68  * DONE 4. replacing httpHeaderIdByName() uses with the lookup table
  69  * 5. merge HDR_BAD_HDR and HDR_ENUM_END into one thing
  70  * DONE 6. replacing httpHeaderNameById with direct array lookups
  71  * 7. being looking at the other arrays removal
  72  */
  73
  74
  75 LookupTable<http_hdr_type, HeaderTableRecord> headerLookupTable(HDR_BAD_HDR, headerTable);
  76 std::vector<HttpHeaderFieldStat> headerStatsTable(HDR_OTHER+1);
  77
  78 /*
  79  * headers with field values defined as #(values) in HTTP/1.1
  80  * Headers that are currently not recognized, are commented out.
  81  */
  82 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
  83 static http_hdr_type ListHeadersArr[] = {
  84     HDR_ACCEPT,
  85     HDR_ACCEPT_CHARSET,
  86     HDR_ACCEPT_ENCODING,
  87     HDR_ACCEPT_LANGUAGE,
  88     HDR_ACCEPT_RANGES,
  89     HDR_ALLOW,
  90     HDR_CACHE_CONTROL,
  91     HDR_CONTENT_ENCODING,
  92     HDR_CONTENT_LANGUAGE,
  93     HDR_CONNECTION,
  94     HDR_EXPECT,
  95     HDR_IF_MATCH,
  96     HDR_IF_NONE_MATCH,
  97     HDR_KEY,
  98     HDR_LINK,
  99     HDR_PRAGMA,
 100     HDR_PROXY_CONNECTION,
 101     HDR_PROXY_SUPPORT,
 102     HDR_TRANSFER_ENCODING,
 103     HDR_UPGRADE,
 104     HDR_VARY,
 105     HDR_VIA,
 106     HDR_WARNING,
 107     HDR_WWW_AUTHENTICATE,
 108     HDR_AUTHENTICATION_INFO,
 109     HDR_PROXY_AUTHENTICATION_INFO,
 110     /* HDR_TE, HDR_TRAILER */
 111 #if X_ACCELERATOR_VARY
 112     HDR_X_ACCELERATOR_VARY,
 113 #endif
 114 #if USE_ADAPTATION
 115     HDR_X_NEXT_SERVICES,
 116 #endif
 117     HDR_SURROGATE_CAPABILITY,
 118     HDR_SURROGATE_CONTROL,
 119     HDR_FORWARDED,
 120     HDR_X_FORWARDED_FOR
 121 };
 122
 123 /* general-headers */
 124 static http_hdr_type GeneralHeadersArr[] = {
 125     HDR_CACHE_CONTROL,
 126     HDR_CONNECTION,
 127     HDR_DATE,
 128     HDR_FORWARDED,
 129     HDR_X_FORWARDED_FOR,
 130     HDR_MIME_VERSION,
 131     HDR_PRAGMA,
 132     HDR_PROXY_CONNECTION,
 133     HDR_TRANSFER_ENCODING,
 134     HDR_UPGRADE,
 135     /* HDR_TRAILER, */
 136     HDR_VIA,
 137 };
 138
 139 /* entity-headers */
 140 static http_hdr_type EntityHeadersArr[] = {
 141     HDR_ALLOW,
 142     HDR_CONTENT_BASE,
 143     HDR_CONTENT_ENCODING,
 144     HDR_CONTENT_LANGUAGE,
 145     HDR_CONTENT_LENGTH,
 146     HDR_CONTENT_LOCATION,
 147     HDR_CONTENT_MD5,
 148     HDR_CONTENT_RANGE,
 149     HDR_CONTENT_TYPE,
 150     HDR_ETAG,
 151     HDR_EXPIRES,
 152     HDR_LAST_MODIFIED,
 153     HDR_LINK,
 154     HDR_OTHER
 155 };
 156
 157 /* request-only headers */
 158 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
 159 static http_hdr_type RequestHeadersArr[] = {
 160     HDR_ACCEPT,
 161     HDR_ACCEPT_CHARSET,
 162     HDR_ACCEPT_ENCODING,
 163     HDR_ACCEPT_LANGUAGE,
 164     HDR_AUTHORIZATION,
 165     HDR_EXPECT,
 166     HDR_FROM,
 167     HDR_HOST,
 168     HDR_HTTP2_SETTINGS,
 169     HDR_IF_MATCH,
 170     HDR_IF_MODIFIED_SINCE,
 171     HDR_IF_NONE_MATCH,
 172     HDR_IF_RANGE,
 173     HDR_IF_UNMODIFIED_SINCE,
 174     HDR_MAX_FORWARDS,
 175     HDR_ORIGIN,
 176     HDR_PROXY_AUTHORIZATION,
 177     HDR_RANGE,
 178     HDR_REFERER,
 179     HDR_REQUEST_RANGE,
 180     HDR_TE,
 181     HDR_USER_AGENT,
 182     HDR_SURROGATE_CAPABILITY
 183 };
 184
 185 /* reply-only headers */
 186 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
 187 static http_hdr_type ReplyHeadersArr[] = {
 188     HDR_ACCEPT_ENCODING,
 189     HDR_ACCEPT_RANGES,
 190     HDR_AGE,
 191     HDR_KEY,
 192     HDR_LOCATION,
 193     HDR_PROXY_AUTHENTICATE,
 194     HDR_PUBLIC,
 195     HDR_RETRY_AFTER,
 196     HDR_SERVER,
 197     HDR_SET_COOKIE,
 198     HDR_SET_COOKIE2,
 199     HDR_VARY,
 200     HDR_WARNING,
 201     HDR_WWW_AUTHENTICATE,
 202     HDR_X_CACHE,
 203     HDR_X_CACHE_LOOKUP,
 204     HDR_X_REQUEST_URI,
 205 #if X_ACCELERATOR_VARY
 206     HDR_X_ACCELERATOR_VARY,
 207 #endif
 208 #if USE_ADAPTATION
 209     HDR_X_NEXT_SERVICES,
 210 #endif
 211     HDR_X_SQUID_ERROR,
 212     HDR_SURROGATE_CONTROL
 213 };
 214
 215 /* hop-by-hop headers */
 216 static HttpHeaderMask HopByHopHeadersMask;
 217 static http_hdr_type HopByHopHeadersArr[] = {
 218     HDR_ALTERNATE_PROTOCOL,
 219     HDR_CONNECTION,
 220     HDR_HTTP2_SETTINGS,
 221     HDR_KEEP_ALIVE,
 222     /*HDR_PROXY_AUTHENTICATE, // removal handled specially for peer login */
 223     HDR_PROXY_AUTHORIZATION,
 224     HDR_TE,
 225     HDR_TRAILER,
 226     HDR_TRANSFER_ENCODING,
 227     HDR_UPGRADE,
 228     HDR_PROXY_CONNECTION
 229 };
 230
 231 /* header accounting */
 232 // NP: keep in sync with enum http_hdr_owner_type
 233 static HttpHeaderStat HttpHeaderStats[] = {
 234     HttpHeaderStat(/*hoNone*/ "all", NULL),
 235 #if USE_HTCP
 236     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
 237 #endif
 238     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
 239     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
 240 #if USE_OPENSSL
 241     /* hoErrorDetail */
 242 #endif
 243     /* hoEnd */
 244 };
 245 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 246
 247 static int HeaderEntryParsedCount = 0;
 248
 249 /*
 250  * forward declarations and local routines
 251  */
 252
 253 class StoreEntry;
 254 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 255
 256 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 257
 258 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 259
 260 /** store report about current header usage and other stats */
 261 static void httpHeaderStoreReport(StoreEntry * e);
 262
 263 /*
 264  * Module initialization routines
 265  */
 266
 267 static void
 268 httpHeaderRegisterWithCacheManager(void)
 269 {
 270     Mgr::RegisterAction("http_headers",
 271                         "HTTP Header Statistics",
 272                         httpHeaderStoreReport, 0, 1);
 273 }
 274
 275 void
 276 httpHeaderInitModule(void)
 277 {
 278     /* check that we have enough space for masks */
 279     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 280
 281     // check invariant: for each index in headerTable, (int)headerTable[index] = index
 282     for (int i = 0; headerTable[i].name; ++i)
 283         assert(headerTable[i].id == i);
 284
 285     // use headerLookupTable in place of Headers
 286
 287     /* create masks */
 288     httpHeaderMaskInit(&ListHeadersMask, 0);
 289     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 290
 291     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 292     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 293     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 294     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 295
 296     httpHeaderMaskInit(&RequestHeadersMask, 0);
 297     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 298     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 299     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 300
 301     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 302     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 303
 304     /* header stats initialized by class constructor */
 305     assert(HttpHeaderStatCount == hoReply + 1);
 306
 307     /* init dependent modules */
 308     httpHdrCcInitModule();
 309     httpHdrScInitModule();
 310
 311     httpHeaderRegisterWithCacheManager();
 312 }
 313
 314 void
 315 httpHeaderCleanModule(void)
 316 {
 317     httpHdrCcCleanModule();
 318     httpHdrScCleanModule();
 319 }
 320
 321 /*
 322  * HttpHeader Implementation
 323  */
 324
 325 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 326 {
 327     httpHeaderMaskInit(&mask, 0);
 328 }
 329
 330 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 331 {
 332     assert(anOwner > hoNone && anOwner < hoEnd);
 333     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 334     httpHeaderMaskInit(&mask, 0);
 335 }
 336
 337 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 338 {
 339     httpHeaderMaskInit(&mask, 0);
 340     update(&other, NULL); // will update the mask as well
 341 }
 342
 343 HttpHeader::~HttpHeader()
 344 {
 345     clean();
 346 }
 347
 348 HttpHeader &
 349 HttpHeader::operator =(const HttpHeader &other)
 350 {
 351     if (this != &other) {
 352         // we do not really care, but the caller probably does
 353         assert(owner == other.owner);
 354         clean();
 355         update(&other, NULL); // will update the mask as well
 356         len = other.len;
 357     }
 358     return *this;
 359 }
 360
 361 void
 362 HttpHeader::clean()
 363 {
 364
 365     assert(owner > hoNone && owner < hoEnd);
 366     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 367
 368     PROF_start(HttpHeaderClean);
 369
 370     if (owner <= hoReply) {
 371         /*
 372          * An unfortunate bug.  The entries array is initialized
 373          * such that count is set to zero.  httpHeaderClean() seems to
 374          * be called both when 'hdr' is created, and destroyed.  Thus,
 375          * we accumulate a large number of zero counts for 'hdr' before
 376          * it is ever used.  Can't think of a good way to fix it, except
 377          * adding a state variable that indicates whether or not 'hdr'
 378          * has been used.  As a hack, just never count zero-sized header
 379          * arrays.
 380          */
 381         if (!entries.empty())
 382             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 383
 384         ++ HttpHeaderStats[owner].destroyedCount;
 385
 386         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 387     } // if (owner <= hoReply)
 388
 389     for (std::vector<HttpHeaderEntry *>::iterator i = entries.begin(); i != entries.end(); ++i) {
 390         HttpHeaderEntry *e = *i;
 391         if (e == NULL)
 392             continue;
 393         if (e->id < 0 || e->id >= HDR_ENUM_END) {
 394             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 395         } else {
 396             if (owner <= hoReply)
 397                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 398             delete e;
 399         }
 400     }
 401
 402     entries.clear();
 403     httpHeaderMaskInit(&mask, 0);
 404     len = 0;
 405     PROF_stop(HttpHeaderClean);
 406 }
 407
 408 /* append entries (also see httpHeaderUpdate) */
 409 void
 410 HttpHeader::append(const HttpHeader * src)
 411 {
 412     const HttpHeaderEntry *e;
 413     HttpHeaderPos pos = HttpHeaderInitPos;
 414     assert(src);
 415     assert(src != this);
 416     debugs(55, 7, "appending hdr: " << this << " += " << src);
 417
 418     while ((e = src->getEntry(&pos))) {
 419         addEntry(e->clone());
 420     }
 421 }
 422
 423 /* use fresh entries to replace old ones */
 424 void
 425 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 426 {
 427     assert (old);
 428     old->update (fresh, denied_mask);
 429 }
 430
 431 void
 432 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 433 {
 434     const HttpHeaderEntry *e;
 435     HttpHeaderPos pos = HttpHeaderInitPos;
 436     assert(fresh);
 437     assert(this != fresh);
 438
 439     while ((e = fresh->getEntry(&pos))) {
 440         /* deny bad guys (ok to check for HDR_OTHER) here */
 441
 442         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 443             continue;
 444
 445         if (e->id != HDR_OTHER)
 446             delById(e->id);
 447         else
 448             delByName(e->name.termedBuf());
 449     }
 450
 451     pos = HttpHeaderInitPos;
 452     while ((e = fresh->getEntry(&pos))) {
 453         /* deny bad guys (ok to check for HDR_OTHER) here */
 454
 455         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 456             continue;
 457
 458         debugs(55, 7, "Updating header '" << headerTable[e->id].name << "' in cached entry");
 459
 460         addEntry(e->clone());
 461     }
 462 }
 463
 464 /* just handy in parsing: resets and returns false */
 465 int
 466 HttpHeader::reset()
 467 {
 468     clean();
 469     return 0;
 470 }
 471
 472 int
 473 HttpHeader::parse(const char *header_start, size_t hdrLen)
 474 {
 475     const char *field_ptr = header_start;
 476     const char *header_end = header_start + hdrLen; // XXX: remove
 477     HttpHeaderEntry *e, *e2;
 478     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 479
 480     PROF_start(HttpHeaderParse);
 481
 482     assert(header_start && header_end);
 483     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 484     ++ HttpHeaderStats[owner].parsedCount;
 485
 486     char *nulpos;
 487     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 488         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 489                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 490         PROF_stop(HttpHeaderParse);
 491         return reset();
 492     }
 493
 494     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 495      * continuation lines start with a (single) space or tab */
 496     while (field_ptr < header_end) {
 497         const char *field_start = field_ptr;
 498         const char *field_end;
 499
 500         do {
 501             const char *this_line = field_ptr;
 502             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 503
 504             if (!field_ptr) {
 505                 // missing <LF>
 506                 PROF_stop(HttpHeaderParse);
 507                 return reset();
 508             }
 509
 510             field_end = field_ptr;
 511
 512             ++field_ptr;    /* Move to next line */
 513
 514             if (field_end > this_line && field_end[-1] == '\r') {
 515                 --field_end;    /* Ignore CR LF */
 516
 517                 if (owner == hoRequest && field_end > this_line) {
 518                     bool cr_only = true;
 519                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 520                         if (*p != '\r')
 521                             cr_only = false;
 522                     }
 523                     if (cr_only) {
 524                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 525                                "header field to prevent request smuggling attacks: {" <<
 526                                getStringPrefix(header_start, hdrLen) << "}");
 527                         PROF_stop(HttpHeaderParse);
 528                         return reset();
 529                     }
 530                 }
 531             }
 532
 533             /* Barf on stray CR characters */
 534             if (memchr(this_line, '\r', field_end - this_line)) {
 535                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 536                        getStringPrefix(field_start, field_end-field_start) << "}");
 537
 538                 if (Config.onoff.relaxed_header_parser) {
 539                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 540
 541                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 542                         *p = ' ';
 543                         ++p;
 544                     }
 545                 } else {
 546                     PROF_stop(HttpHeaderParse);
 547                     return reset();
 548                 }
 549             }
 550
 551             if (this_line + 1 == field_end && this_line > field_start) {
 552                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 553                        getStringPrefix(header_start, hdrLen) << "}");
 554                 PROF_stop(HttpHeaderParse);
 555                 return reset();
 556             }
 557         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 558
 559         if (field_start == field_end) {
 560             if (field_ptr < header_end) {
 561                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 562                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 563                 PROF_stop(HttpHeaderParse);
 564                 return reset();
 565             }
 566
 567             break;      /* terminating blank line */
 568         }
 569
 570         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 571             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 572                    getStringPrefix(field_start, field_end-field_start) << "}");
 573             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 574
 575             if (Config.onoff.relaxed_header_parser)
 576                 continue;
 577
 578             PROF_stop(HttpHeaderParse);
 579             return reset();
 580         }
 581
 582         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 583             if (e->value != e2->value) {
 584                 int64_t l1, l2;
 585                 debugs(55, warnOnError, "WARNING: found two conflicting content-length headers in {" <<
 586                        getStringPrefix(header_start, hdrLen) << "}");
 587
 588                 if (!Config.onoff.relaxed_header_parser) {
 589                     delete e;
 590                     PROF_stop(HttpHeaderParse);
 591                     return reset();
 592                 }
 593
 594                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 595                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e->value << "'");
 596                     delete e;
 597                     continue;
 598                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 599                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e2->value << "'");
 600                     delById(e2->id);
 601                 } else if (l1 > l2) {
 602                     delById(e2->id);
 603                 } else {
 604                     delete e;
 605                     continue;
 606                 }
 607             } else {
 608                 debugs(55, warnOnError, "NOTICE: found double content-length header");
 609                 delete e;
 610
 611                 if (Config.onoff.relaxed_header_parser)
 612                     continue;
 613
 614                 PROF_stop(HttpHeaderParse);
 615                 return reset();
 616             }
 617         }
 618
 619         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 620             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 621                    getStringPrefix(field_start, field_end-field_start) << "}");
 622
 623             if (!Config.onoff.relaxed_header_parser) {
 624                 delete e;
 625                 PROF_stop(HttpHeaderParse);
 626                 return reset();
 627             }
 628         }
 629
 630         addEntry(e);
 631     }
 632
 633     if (chunked()) {
 634         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 635         delById(HDR_CONTENT_LENGTH);
 636     }
 637
 638     PROF_stop(HttpHeaderParse);
 639     return 1;           /* even if no fields where found, it is a valid header */
 640 }
 641
 642 /* packs all the entries using supplied packer */
 643 void
 644 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 645 {
 646     HttpHeaderPos pos = HttpHeaderInitPos;
 647     const HttpHeaderEntry *e;
 648     assert(p);
 649     debugs(55, 7, this << " into " << p <<
 650            (mask_sensitive_info ? " while masking" : ""));
 651     /* pack all entries one by one */
 652     while ((e = getEntry(&pos))) {
 653         if (!mask_sensitive_info) {
 654             e->packInto(p);
 655             continue;
 656         }
 657
 658         bool maskThisEntry = false;
 659         switch (e->id) {
 660         case HDR_AUTHORIZATION:
 661         case HDR_PROXY_AUTHORIZATION:
 662             maskThisEntry = true;
 663             break;
 664
 665         case HDR_FTP_ARGUMENTS:
 666             if (const HttpHeaderEntry *cmd = findEntry(HDR_FTP_COMMAND))
 667                 maskThisEntry = (cmd->value == "PASS");
 668             break;
 669
 670         default:
 671             break;
 672         }
 673         if (maskThisEntry) {
 674             p->append(e->name.rawBuf(), e->name.size());
 675             p->append(": ** NOT DISPLAYED **\r\n", 23);
 676         } else {
 677             e->packInto(p);
 678         }
 679
 680     }
 681     /* Pack in the "special" entries */
 682
 683     /* Cache-Control */
 684 }
 685
 686 /* returns next valid entry */
 687 HttpHeaderEntry *
 688 HttpHeader::getEntry(HttpHeaderPos * pos) const
 689 {
 690     assert(pos);
 691     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 692
 693     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 694         if (entries[*pos])
 695             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 696     }
 697
 698     return NULL;
 699 }
 700
 701 /*
 702  * returns a pointer to a specified entry if any
 703  * note that we return one entry so it does not make much sense to ask for
 704  * "list" headers
 705  */
 706 HttpHeaderEntry *
 707 HttpHeader::findEntry(http_hdr_type id) const
 708 {
 709     HttpHeaderPos pos = HttpHeaderInitPos;
 710     HttpHeaderEntry *e;
 711     assert_eid(id);
 712     assert(!CBIT_TEST(ListHeadersMask, id));
 713
 714     /* check mask first */
 715
 716     if (!CBIT_TEST(mask, id))
 717         return NULL;
 718
 719     /* looks like we must have it, do linear search */
 720     while ((e = getEntry(&pos))) {
 721         if (e->id == id)
 722             return e;
 723     }
 724
 725     /* hm.. we thought it was there, but it was not found */
 726     assert(0);
 727
 728     return NULL;        /* not reached */
 729 }
 730
 731 /*
 732  * same as httpHeaderFindEntry
 733  */
 734 HttpHeaderEntry *
 735 HttpHeader::findLastEntry(http_hdr_type id) const
 736 {
 737     HttpHeaderPos pos = HttpHeaderInitPos;
 738     HttpHeaderEntry *e;
 739     HttpHeaderEntry *result = NULL;
 740     assert_eid(id);
 741     assert(!CBIT_TEST(ListHeadersMask, id));
 742
 743     /* check mask first */
 744
 745     if (!CBIT_TEST(mask, id))
 746         return NULL;
 747
 748     /* looks like we must have it, do linear search */
 749     while ((e = getEntry(&pos))) {
 750         if (e->id == id)
 751             result = e;
 752     }
 753
 754     assert(result);     /* must be there! */
 755     return result;
 756 }
 757
 758 /*
 759  * deletes all fields with a given name if any, returns #fields deleted;
 760  */
 761 int
 762 HttpHeader::delByName(const char *name)
 763 {
 764     int count = 0;
 765     HttpHeaderPos pos = HttpHeaderInitPos;
 766     HttpHeaderEntry *e;
 767     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 768     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 769
 770     while ((e = getEntry(&pos))) {
 771         if (!e->name.caseCmp(name))
 772             delAt(pos, count);
 773         else
 774             CBIT_SET(mask, e->id);
 775     }
 776
 777     return count;
 778 }
 779
 780 /* deletes all entries with a given id, returns the #entries deleted */
 781 int
 782 HttpHeader::delById(http_hdr_type id)
 783 {
 784     int count = 0;
 785     HttpHeaderPos pos = HttpHeaderInitPos;
 786     HttpHeaderEntry *e;
 787     debugs(55, 8, this << " del-by-id " << id);
 788     assert_eid(id);
 789     assert(id != HDR_OTHER);        /* does not make sense */
 790
 791     if (!CBIT_TEST(mask, id))
 792         return 0;
 793
 794     while ((e = getEntry(&pos))) {
 795         if (e->id == id)
 796             delAt(pos, count);
 797     }
 798
 799     CBIT_CLR(mask, id);
 800     assert(count);
 801     return count;
 802 }
 803
 804 /*
 805  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 806  * possible to iterate(search) and delete fields at the same time
 807  * NOTE: Does not update the header mask. Caller must follow up with
 808  * a call to refreshMask() if headers_deleted was incremented.
 809  */
 810 void
 811 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 812 {
 813     HttpHeaderEntry *e;
 814     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
 815     e = static_cast<HttpHeaderEntry*>(entries[pos]);
 816     entries[pos] = NULL;
 817     /* decrement header length, allow for ": " and crlf */
 818     len -= e->name.size() + 2 + e->value.size() + 2;
 819     assert(len >= 0);
 820     delete e;
 821     ++headers_deleted;
 822 }
 823
 824 /*
 825  * Compacts the header storage
 826  */
 827 void
 828 HttpHeader::compact()
 829 {
 830     // TODO: optimize removal, or possibly make it so that's not needed.
 831     std::vector<HttpHeaderEntry *>::iterator newend;
 832     newend = std::remove(entries.begin(), entries.end(), static_cast<HttpHeaderEntry *>(NULL));
 833     entries.resize(newend-entries.begin());
 834 }
 835
 836 /*
 837  * Refreshes the header mask. Required after delAt() calls.
 838  */
 839 void
 840 HttpHeader::refreshMask()
 841 {
 842     httpHeaderMaskInit(&mask, 0);
 843     debugs(55, 7, "refreshing the mask in hdr " << this);
 844     HttpHeaderPos pos = HttpHeaderInitPos;
 845     while (HttpHeaderEntry *e = getEntry(&pos)) {
 846         CBIT_SET(mask, e->id);
 847     }
 848 }
 849
 850 /* appends an entry;
 851  * does not call e->clone() so one should not reuse "*e"
 852  */
 853 void
 854 HttpHeader::addEntry(HttpHeaderEntry * e)
 855 {
 856     assert(e);
 857     assert_eid(e->id);
 858     assert(e->name.size());
 859
 860     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 861
 862     if (CBIT_TEST(mask, e->id)) {
 863         ++ headerStatsTable[e->id].repCount;
 864     } else {
 865         CBIT_SET(mask, e->id);
 866     }
 867
 868     entries.push_back(e);
 869
 870     /* increment header length, allow for ": " and crlf */
 871     len += e->name.size() + 2 + e->value.size() + 2;
 872 }
 873
 874 /* inserts an entry;
 875  * does not call e->clone() so one should not reuse "*e"
 876  */
 877 void
 878 HttpHeader::insertEntry(HttpHeaderEntry * e)
 879 {
 880     assert(e);
 881     assert_eid(e->id);
 882
 883     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
 884
 885     if (CBIT_TEST(mask, e->id)) {
 886         ++ headerStatsTable[e->id].repCount; //TODO: use operator[] ?
 887     } else {
 888         CBIT_SET(mask, e->id);
 889     }
 890
 891     entries.insert(entries.begin(),e);
 892
 893     /* increment header length, allow for ": " and crlf */
 894     len += e->name.size() + 2 + e->value.size() + 2;
 895 }
 896
 897 bool
 898 HttpHeader::getList(http_hdr_type id, String *s) const
 899 {
 900     HttpHeaderEntry *e;
 901     HttpHeaderPos pos = HttpHeaderInitPos;
 902     debugs(55, 9, this << " joining for id " << id);
 903     /* only fields from ListHeaders array can be "listed" */
 904     assert(CBIT_TEST(ListHeadersMask, id));
 905
 906     if (!CBIT_TEST(mask, id))
 907         return false;
 908
 909     while ((e = getEntry(&pos))) {
 910         if (e->id == id)
 911             strListAdd(s, e->value.termedBuf(), ',');
 912     }
 913
 914     /*
 915      * note: we might get an empty (size==0) string if there was an "empty"
 916      * header. This results in an empty length String, which may have a NULL
 917      * buffer.
 918      */
 919     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 920     if (!s->size())
 921         debugs(55, 3, "empty list header: " << headerTable[id].name << "(" << id << ")");
 922     else
 923         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 924
 925     return true;
 926 }
 927
 928 /* return a list of entries with the same id separated by ',' and ws */
 929 String
 930 HttpHeader::getList(http_hdr_type id) const
 931 {
 932     HttpHeaderEntry *e;
 933     HttpHeaderPos pos = HttpHeaderInitPos;
 934     debugs(55, 9, this << "joining for id " << id);
 935     /* only fields from ListHeaders array can be "listed" */
 936     assert(CBIT_TEST(ListHeadersMask, id));
 937
 938     if (!CBIT_TEST(mask, id))
 939         return String();
 940
 941     String s;
 942
 943     while ((e = getEntry(&pos))) {
 944         if (e->id == id)
 945             strListAdd(&s, e->value.termedBuf(), ',');
 946     }
 947
 948     /*
 949      * note: we might get an empty (size==0) string if there was an "empty"
 950      * header. This results in an empty length String, which may have a NULL
 951      * buffer.
 952      */
 953     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 954     if (!s.size())
 955         debugs(55, 3, "empty list header: " << headerTable[id].name << "(" << id << ")");
 956     else
 957         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 958
 959     return s;
 960 }
 961
 962 /* return a string or list of entries with the same id separated by ',' and ws */
 963 String
 964 HttpHeader::getStrOrList(http_hdr_type id) const
 965 {
 966     HttpHeaderEntry *e;
 967
 968     if (CBIT_TEST(ListHeadersMask, id))
 969         return getList(id);
 970
 971     if ((e = findEntry(id)))
 972         return e->value;
 973
 974     return String();
 975 }
 976
 977 /*
 978  * Returns the value of the specified header and/or an undefined String.
 979  */
 980 String
 981 HttpHeader::getByName(const char *name) const
 982 {
 983     String result;
 984     // ignore presence: return undefined string if an empty header is present
 985     (void)getByNameIfPresent(name, result);
 986     return result;
 987 }
 988
 989 bool
 990 HttpHeader::getByNameIfPresent(const char *name, String &result) const
 991 {
 992     http_hdr_type id;
 993     HttpHeaderPos pos = HttpHeaderInitPos;
 994     HttpHeaderEntry *e;
 995
 996     assert(name);
 997
 998     /* First try the quick path */
 999     id = httpHeaderIdByNameDef(SBuf(name));
1000
1001     if (id != -1) {
1002         if (!has(id))
1003             return false;
1004         result = getStrOrList(id);
1005         return true;
1006     }
1007
1008     /* Sorry, an unknown header name. Do linear search */
1009     bool found = false;
1010     while ((e = getEntry(&pos))) {
1011         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1012             found = true;
1013             strListAdd(&result, e->value.termedBuf(), ',');
1014         }
1015     }
1016
1017     return found;
1018 }
1019
1020 /*
1021  * Returns a the value of the specified list member, if any.
1022  */
1023 String
1024 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1025 {
1026     String header;
1027     const char *pos = NULL;
1028     const char *item;
1029     int ilen;
1030     int mlen = strlen(member);
1031
1032     assert(name);
1033
1034     header = getByName(name);
1035
1036     String result;
1037
1038     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1039         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1040             result.append(item + mlen + 1, ilen - mlen - 1);
1041             break;
1042         }
1043     }
1044
1045     return result;
1046 }
1047
1048 /*
1049  * returns a the value of the specified list member, if any.
1050  */
1051 String
1052 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1053 {
1054     String header;
1055     const char *pos = NULL;
1056     const char *item;
1057     int ilen;
1058     int mlen = strlen(member);
1059
1060     assert(id >= 0);
1061
1062     header = getStrOrList(id);
1063     String result;
1064
1065     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1066         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1067             result.append(item + mlen + 1, ilen - mlen - 1);
1068             break;
1069         }
1070     }
1071
1072     header.clean();
1073     return result;
1074 }
1075
1076 /* test if a field is present */
1077 int
1078 HttpHeader::has(http_hdr_type id) const
1079 {
1080     assert_eid(id);
1081     assert(id != HDR_OTHER);
1082     debugs(55, 9, this << " lookup for " << id);
1083     return CBIT_TEST(mask, id);
1084 }
1085
1086 void
1087 HttpHeader::putInt(http_hdr_type id, int number)
1088 {
1089     assert_eid(id);
1090     assert(headerTable[id].type == ftInt);  /* must be of an appropriate type */
1091     assert(number >= 0);
1092     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1093 }
1094
1095 void
1096 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1097 {
1098     assert_eid(id);
1099     assert(headerTable[id].type == ftInt64);    /* must be of an appropriate type */
1100     assert(number >= 0);
1101     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1102 }
1103
1104 void
1105 HttpHeader::putTime(http_hdr_type id, time_t htime)
1106 {
1107     assert_eid(id);
1108     assert(headerTable[id].type == ftDate_1123);    /* must be of an appropriate type */
1109     assert(htime >= 0);
1110     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1111 }
1112
1113 void
1114 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1115 {
1116     assert_eid(id);
1117     assert(headerTable[id].type == ftDate_1123);    /* must be of an appropriate type */
1118     assert(htime >= 0);
1119     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1120 }
1121
1122 void
1123 HttpHeader::putStr(http_hdr_type id, const char *str)
1124 {
1125     assert_eid(id);
1126     assert(headerTable[id].type == ftStr);  /* must be of an appropriate type */
1127     assert(str);
1128     addEntry(new HttpHeaderEntry(id, NULL, str));
1129 }
1130
1131 void
1132 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1133 {
1134     assert(auth_scheme && realm);
1135     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1136 }
1137
1138 void
1139 HttpHeader::putCc(const HttpHdrCc * cc)
1140 {
1141     assert(cc);
1142     /* remove old directives if any */
1143     delById(HDR_CACHE_CONTROL);
1144     /* pack into mb */
1145     MemBuf mb;
1146     mb.init();
1147     cc->packInto(&mb);
1148     /* put */
1149     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1150     /* cleanup */
1151     mb.clean();
1152 }
1153
1154 void
1155 HttpHeader::putContRange(const HttpHdrContRange * cr)
1156 {
1157     assert(cr);
1158     /* remove old directives if any */
1159     delById(HDR_CONTENT_RANGE);
1160     /* pack into mb */
1161     MemBuf mb;
1162     mb.init();
1163     httpHdrContRangePackInto(cr, &mb);
1164     /* put */
1165     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1166     /* cleanup */
1167     mb.clean();
1168 }
1169
1170 void
1171 HttpHeader::putRange(const HttpHdrRange * range)
1172 {
1173     assert(range);
1174     /* remove old directives if any */
1175     delById(HDR_RANGE);
1176     /* pack into mb */
1177     MemBuf mb;
1178     mb.init();
1179     range->packInto(&mb);
1180     /* put */
1181     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1182     /* cleanup */
1183     mb.clean();
1184 }
1185
1186 void
1187 HttpHeader::putSc(HttpHdrSc *sc)
1188 {
1189     assert(sc);
1190     /* remove old directives if any */
1191     delById(HDR_SURROGATE_CONTROL);
1192     /* pack into mb */
1193     MemBuf mb;
1194     mb.init();
1195     sc->packInto(&mb);
1196     /* put */
1197     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1198     /* cleanup */
1199     mb.clean();
1200 }
1201
1202 void
1203 HttpHeader::putWarning(const int code, const char *const text)
1204 {
1205     char buf[512];
1206     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1207     putStr(HDR_WARNING, buf);
1208 }
1209
1210 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1211 void
1212 HttpHeader::putExt(const char *name, const char *value)
1213 {
1214     assert(name && value);
1215     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1216     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1217 }
1218
1219 int
1220 HttpHeader::getInt(http_hdr_type id) const
1221 {
1222     assert_eid(id);
1223     assert(headerTable[id].type == ftInt);  /* must be of an appropriate type */
1224     HttpHeaderEntry *e;
1225
1226     if ((e = findEntry(id)))
1227         return e->getInt();
1228
1229     return -1;
1230 }
1231
1232 int64_t
1233 HttpHeader::getInt64(http_hdr_type id) const
1234 {
1235     assert_eid(id);
1236     assert(headerTable[id].type == ftInt64);    /* must be of an appropriate type */
1237     HttpHeaderEntry *e;
1238
1239     if ((e = findEntry(id)))
1240         return e->getInt64();
1241
1242     return -1;
1243 }
1244
1245 time_t
1246 HttpHeader::getTime(http_hdr_type id) const
1247 {
1248     HttpHeaderEntry *e;
1249     time_t value = -1;
1250     assert_eid(id);
1251     assert(headerTable[id].type == ftDate_1123);    /* must be of an appropriate type */
1252
1253     if ((e = findEntry(id))) {
1254         value = parse_rfc1123(e->value.termedBuf());
1255         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1256     }
1257
1258     return value;
1259 }
1260
1261 /* sync with httpHeaderGetLastStr */
1262 const char *
1263 HttpHeader::getStr(http_hdr_type id) const
1264 {
1265     HttpHeaderEntry *e;
1266     assert_eid(id);
1267     assert(headerTable[id].type == ftStr);  /* must be of an appropriate type */
1268
1269     if ((e = findEntry(id))) {
1270         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1271         return e->value.termedBuf();
1272     }
1273
1274     return NULL;
1275 }
1276
1277 /* unusual */
1278 const char *
1279 HttpHeader::getLastStr(http_hdr_type id) const
1280 {
1281     HttpHeaderEntry *e;
1282     assert_eid(id);
1283     assert(headerTable[id].type == ftStr);  /* must be of an appropriate type */
1284
1285     if ((e = findLastEntry(id))) {
1286         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1287         return e->value.termedBuf();
1288     }
1289
1290     return NULL;
1291 }
1292
1293 HttpHdrCc *
1294 HttpHeader::getCc() const
1295 {
1296     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1297         return NULL;
1298     PROF_start(HttpHeader_getCc);
1299
1300     String s;
1301     getList(HDR_CACHE_CONTROL, &s);
1302
1303     HttpHdrCc *cc=new HttpHdrCc();
1304
1305     if (!cc->parse(s)) {
1306         delete cc;
1307         cc = NULL;
1308     }
1309
1310     ++ HttpHeaderStats[owner].ccParsedCount;
1311
1312     if (cc)
1313         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1314
1315     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1316
1317     PROF_stop(HttpHeader_getCc);
1318
1319     return cc;
1320 }
1321
1322 HttpHdrRange *
1323 HttpHeader::getRange() const
1324 {
1325     HttpHdrRange *r = NULL;
1326     HttpHeaderEntry *e;
1327     /* some clients will send "Request-Range" _and_ *matching* "Range"
1328      * who knows, some clients might send Request-Range only;
1329      * this "if" should work correctly in both cases;
1330      * hopefully no clients send mismatched headers! */
1331
1332     if ((e = findEntry(HDR_RANGE)) ||
1333             (e = findEntry(HDR_REQUEST_RANGE))) {
1334         r = HttpHdrRange::ParseCreate(&e->value);
1335         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1336     }
1337
1338     return r;
1339 }
1340
1341 HttpHdrSc *
1342 HttpHeader::getSc() const
1343 {
1344     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1345         return NULL;
1346
1347     String s;
1348
1349     (void) getList(HDR_SURROGATE_CONTROL, &s);
1350
1351     HttpHdrSc *sc = httpHdrScParseCreate(s);
1352
1353     ++ HttpHeaderStats[owner].ccParsedCount;
1354
1355     if (sc)
1356         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1357
1358     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1359
1360     return sc;
1361 }
1362
1363 HttpHdrContRange *
1364 HttpHeader::getContRange() const
1365 {
1366     HttpHdrContRange *cr = NULL;
1367     HttpHeaderEntry *e;
1368
1369     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1370         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1371         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1372     }
1373
1374     return cr;
1375 }
1376
1377 const char *
1378 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1379 {
1380     const char *field;
1381     int l;
1382     assert(auth_scheme);
1383     field = getStr(id);
1384
1385     if (!field)         /* no authorization field */
1386         return NULL;
1387
1388     l = strlen(auth_scheme);
1389
1390     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1391         return NULL;
1392
1393     field += l;
1394
1395     if (!xisspace(*field))  /* wrong scheme */
1396         return NULL;
1397
1398     /* skip white space */
1399     for (; field && xisspace(*field); ++field);
1400
1401     if (!*field)        /* no authorization cookie */
1402         return NULL;
1403
1404     static char decodedAuthToken[8192];
1405     struct base64_decode_ctx ctx;
1406     base64_decode_init(&ctx);
1407     size_t decodedLen = 0;
1408     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1409             !base64_decode_final(&ctx)) {
1410         return NULL;
1411     }
1412     decodedAuthToken[decodedLen] = '\0';
1413     return decodedAuthToken;
1414 }
1415
1416 ETag
1417 HttpHeader::getETag(http_hdr_type id) const
1418 {
1419     ETag etag = {NULL, -1};
1420     HttpHeaderEntry *e;
1421     assert(headerTable[id].type == ftETag);     /* must be of an appropriate type */
1422
1423     if ((e = findEntry(id)))
1424         etagParseInit(&etag, e->value.termedBuf());
1425
1426     return etag;
1427 }
1428
1429 TimeOrTag
1430 HttpHeader::getTimeOrTag(http_hdr_type id) const
1431 {
1432     TimeOrTag tot;
1433     HttpHeaderEntry *e;
1434     assert(headerTable[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1435     memset(&tot, 0, sizeof(tot));
1436
1437     if ((e = findEntry(id))) {
1438         const char *str = e->value.termedBuf();
1439         /* try as an ETag */
1440
1441         if (etagParseInit(&tot.tag, str)) {
1442             tot.valid = tot.tag.str != NULL;
1443             tot.time = -1;
1444         } else {
1445             /* or maybe it is time? */
1446             tot.time = parse_rfc1123(str);
1447             tot.valid = tot.time >= 0;
1448             tot.tag.str = NULL;
1449         }
1450     }
1451
1452     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1453     return tot;
1454 }
1455
1456 /*
1457  * HttpHeaderEntry
1458  */
1459
1460 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1461 {
1462     assert_eid(anId);
1463     id = anId;
1464
1465     if (id != HDR_OTHER)
1466         name = headerTable[id].name;
1467     else
1468         name = aName;
1469
1470     value = aValue;
1471
1472     ++ headerStatsTable[id].aliveCount;
1473
1474     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1475 }
1476
1477 HttpHeaderEntry::~HttpHeaderEntry()
1478 {
1479     assert_eid(id);
1480     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1481
1482     assert(headerStatsTable[id].aliveCount); // is this really needed?
1483
1484     -- headerStatsTable[id].aliveCount;
1485
1486     id = HDR_BAD_HDR;
1487 }
1488
1489 /* parses and inits header entry, returns true/false */
1490 HttpHeaderEntry *
1491 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1492 {
1493     /* note: name_start == field_start */
1494     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1495     int name_len = name_end ? name_end - field_start :0;
1496     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1497     /* note: value_end == field_end */
1498
1499     ++ HeaderEntryParsedCount;
1500
1501     /* do we have a valid field name within this field? */
1502
1503     if (!name_len || name_end > field_end)
1504         return NULL;
1505
1506     if (name_len > 65534) {
1507         /* String must be LESS THAN 64K and it adds a terminating NULL */
1508         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1509         return NULL;
1510     }
1511
1512     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1513         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1514                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1515
1516         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1517             --name_len;
1518
1519         if (!name_len)
1520             return NULL;
1521     }
1522
1523     /* now we know we can parse it */
1524
1525     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1526
1527     /* is it a "known" field? */
1528     http_hdr_type id = headerLookupTable.lookup(SBuf(field_start,name_len));
1529     debugs(55, 9, "got hdr id hdr: " << id);
1530
1531     String name;
1532
1533     String value;
1534
1535     if (id < 0)
1536         id = HDR_OTHER;
1537
1538     assert_eid(id);
1539
1540     /* set field name */
1541     if (id == HDR_OTHER)
1542         name.limitInit(field_start, name_len);
1543     else
1544         name = headerTable[id].name;
1545
1546     /* trim field value */
1547     while (value_start < field_end && xisspace(*value_start))
1548         ++value_start;
1549
1550     while (value_start < field_end && xisspace(field_end[-1]))
1551         --field_end;
1552
1553     if (field_end - value_start > 65534) {
1554         /* String must be LESS THAN 64K and it adds a terminating NULL */
1555         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1556
1557         if (id == HDR_OTHER)
1558             name.clean();
1559
1560         return NULL;
1561     }
1562
1563     /* set field value */
1564     value.limitInit(value_start, field_end - value_start);
1565
1566     ++ headerStatsTable[id].seenCount;
1567
1568     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1569
1570     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1571 }
1572
1573 HttpHeaderEntry *
1574 HttpHeaderEntry::clone() const
1575 {
1576     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1577 }
1578
1579 void
1580 HttpHeaderEntry::packInto(Packable * p) const
1581 {
1582     assert(p);
1583     p->append(name.rawBuf(), name.size());
1584     p->append(": ", 2);
1585     p->append(value.rawBuf(), value.size());
1586     p->append("\r\n", 2);
1587 }
1588
1589 int
1590 HttpHeaderEntry::getInt() const
1591 {
1592     assert_eid (id);
1593     int val = -1;
1594     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1595     httpHeaderNoteParsedEntry(id, value, !ok);
1596     /* XXX: Should we check ok - ie
1597      * return ok ? -1 : value;
1598      */
1599     return val;
1600 }
1601
1602 int64_t
1603 HttpHeaderEntry::getInt64() const
1604 {
1605     assert_eid (id);
1606     int64_t val = -1;
1607     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1608     httpHeaderNoteParsedEntry(id, value, !ok);
1609     /* XXX: Should we check ok - ie
1610      * return ok ? -1 : value;
1611      */
1612     return val;
1613 }
1614
1615 static void
1616 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1617 {
1618     ++ headerStatsTable[id].parsCount;
1619
1620     if (error) {
1621         ++ headerStatsTable[id].errCount;
1622         debugs(55, 2, "cannot parse hdr field: '" << headerTable[id].name << ": " << context << "'");
1623     }
1624 }
1625
1626 /*
1627  * Reports
1628  */
1629
1630 /* tmp variable used to pass stat info to dumpers */
1631 extern const HttpHeaderStat *dump_stat;     /* argh! */
1632 const HttpHeaderStat *dump_stat = NULL;
1633
1634 void
1635 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1636 {
1637     const int id = (int) val;
1638     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1639     const char *name = valid_id ? headerTable[id].name : "INVALID";
1640     int visible = count > 0;
1641     /* for entries with zero count, list only those that belong to current type of message */
1642
1643     if (!visible && valid_id && dump_stat->owner_mask)
1644         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1645
1646     if (visible)
1647         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1648                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1649 }
1650
1651 static void
1652 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1653 {
1654     if (count)
1655         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1656                           idx, (int) val, count,
1657                           xpercent(count, dump_stat->destroyedCount));
1658 }
1659
1660 static void
1661 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1662 {
1663     assert(hs && e);
1664
1665     dump_stat = hs;
1666     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1667     storeAppendPrintf(e, "\nField type distribution\n");
1668     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1669                       "id", "name", "count", "#/header");
1670     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1671     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1672     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1673                       "id", "name", "count", "#/cc_field");
1674     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1675     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1676     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1677                       "id", "name", "count", "#/sc_field");
1678     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1679     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1680     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1681                       "id", "#flds", "count", "%total");
1682     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1683     storeAppendPrintf(e, "\n");
1684     dump_stat = NULL;
1685 }
1686
1687 void
1688 httpHeaderStoreReport(StoreEntry * e)
1689 {
1690     int i;
1691 //    http_hdr_type ht;
1692     assert(e);
1693
1694     HttpHeaderStats[0].parsedCount =
1695         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1696     HttpHeaderStats[0].ccParsedCount =
1697         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1698     HttpHeaderStats[0].destroyedCount =
1699         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1700     HttpHeaderStats[0].busyDestroyedCount =
1701         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1702
1703     for (i = 1; i < HttpHeaderStatCount; ++i) {
1704         httpHeaderStatDump(HttpHeaderStats + i, e);
1705     }
1706
1707     /* field stats for all messages */
1708     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1709
1710     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1711                       "id", "name", "#alive", "%err", "%repeat");
1712
1713     // scan heaaderTable and output
1714     for (int j = 0; headerTable[j].name != nullptr; ++j) {
1715         auto stats = headerStatsTable[j];
1716         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1717             headerTable[j].id, headerTable[j].name, stats.aliveCount,
1718             xpercent(stats.errCount, stats.parsCount),
1719             xpercent(stats.repCount, stats.seenCount));
1720     }
1721
1722     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1723                       HttpHeaderStats[hoRequest].parsedCount,
1724                       HttpHeaderStats[hoReply].parsedCount,
1725                       HttpHeaderStats[0].parsedCount);
1726     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1727 }
1728
1729 // (ab)used by other modules.
1730 http_hdr_type
1731 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1732 {
1733     if (name_len > 0) {
1734         for (int i = 0; i < end; ++i) {
1735             if (name_len != info[i].name.size())
1736                 continue;
1737
1738             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1739                 return info[i].id;
1740         }
1741     }
1742
1743     return HDR_BAD_HDR;
1744 }
1745
1746 http_hdr_type
1747 httpHeaderIdByNameDef(const SBuf &name)
1748 {
1749     return headerLookupTable.lookup(name);
1750 }
1751
1752 http_hdr_type
1753 httpHeaderIdByNameDef(const char *name, int name_len)
1754 {
1755     return headerLookupTable.lookup(SBuf(name,name_len));
1756 }
1757
1758 const char *
1759 httpHeaderNameById(int id)
1760 {
1761     assert(id >= 0 && id < HDR_ENUM_END);
1762     return headerTable[id].name;
1763 }
1764
1765 int
1766 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1767 {
1768     int result = 0;
1769     const char *pos = NULL;
1770     const char *item;
1771     int ilen;
1772     int mlen = strlen(member);
1773
1774     assert(id >= 0);
1775
1776     String header (getStrOrList(id));
1777
1778     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1779         if (strncasecmp(item, member, mlen) == 0
1780                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1781             result = 1;
1782             break;
1783         }
1784     }
1785
1786     return result;
1787 }
1788
1789 int
1790 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1791 {
1792     int result = 0;
1793     const char *pos = NULL;
1794     const char *item;
1795     int ilen;
1796     int mlen = strlen(member);
1797
1798     assert(name);
1799
1800     String header (getByName(name));
1801
1802     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1803         if (strncasecmp(item, member, mlen) == 0
1804                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1805             result = 1;
1806             break;
1807         }
1808     }
1809
1810     return result;
1811 }
1812
1813 void
1814 HttpHeader::removeHopByHopEntries()
1815 {
1816     removeConnectionHeaderEntries();
1817
1818     const HttpHeaderEntry *e;
1819     HttpHeaderPos pos = HttpHeaderInitPos;
1820     int headers_deleted = 0;
1821     while ((e = getEntry(&pos))) {
1822         int id = e->id;
1823         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1824             delAt(pos, headers_deleted);
1825             CBIT_CLR(mask, id);
1826         }
1827     }
1828 }
1829
1830 void
1831 HttpHeader::removeConnectionHeaderEntries()
1832 {
1833     if (has(HDR_CONNECTION)) {
1834         /* anything that matches Connection list member will be deleted */
1835         String strConnection;
1836
1837         (void) getList(HDR_CONNECTION, &strConnection);
1838         const HttpHeaderEntry *e;
1839         HttpHeaderPos pos = HttpHeaderInitPos;
1840         /*
1841          * think: on-average-best nesting of the two loops (hdrEntry
1842          * and strListItem) @?@
1843          */
1844         /*
1845          * maybe we should delete standard stuff ("keep-alive","close")
1846          * from strConnection first?
1847          */
1848
1849         int headers_deleted = 0;
1850         while ((e = getEntry(&pos))) {
1851             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1852                 delAt(pos, headers_deleted);
1853         }
1854         if (headers_deleted)
1855             refreshMask();
1856     }
1857 }
1858