src/HttpHeader.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 55    HTTP Header
   6  * AUTHOR: Alex Rousskov
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "base64.h"
  38 #include "HttpHdrContRange.h"
  39 #include "HttpHdrSc.h"
  40 #include "HttpHeader.h"
  41 #include "MemBuf.h"
  42 #include "mgr/Registration.h"
  43 #include "rfc1123.h"
  44 #include "Store.h"
  45
  46 /*
  47  * On naming conventions:
  48  *
  49  * HTTP/1.1 defines message-header as
  50  *
  51  * message-header = field-name ":" [ field-value ] CRLF
  52  * field-name     = token
  53  * field-value    = *( field-content | LWS )
  54  *
  55  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  56  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  57  *
  58  * HttpHeader is an object that represents all message-headers in a message.
  59  * HttpHeader does not manage start-line.
  60  *
  61  * HttpHeader is implemented as a collection of header "entries".
  62  * An entry is a (field_id, field_name, field_value) triplet.
  63  */
  64
  65
  66 /*
  67  * local constants and vars
  68  */
  69
  70 /*
  71  * A table with major attributes for every known field.
  72  * We calculate name lengths and reorganize this array on start up.
  73  * After reorganization, field id can be used as an index to the table.
  74  */
  75 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  76     {"Accept", HDR_ACCEPT, ftStr},
  77
  78     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  79     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  80     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  81     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  82     {"Age", HDR_AGE, ftInt},
  83     {"Allow", HDR_ALLOW, ftStr},
  84     {"Authorization", HDR_AUTHORIZATION, ftStr},        /* for now */
  85     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  86     {"Connection", HDR_CONNECTION, ftStr},
  87     {"Content-Base", HDR_CONTENT_BASE, ftStr},
  88     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
  89     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  90     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  91     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  92     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  93     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
  94     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
  95     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
  96     {"Cookie", HDR_COOKIE, ftStr},
  97     {"Cookie2", HDR_COOKIE2, ftStr},
  98     {"Date", HDR_DATE, ftDate_1123},
  99     {"ETag", HDR_ETAG, ftETag},
 100     {"Expires", HDR_EXPIRES, ftDate_1123},
 101     {"Expect", HDR_EXPECT, ftStr},
 102     {"From", HDR_FROM, ftStr},
 103     {"Host", HDR_HOST, ftStr},
 104     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 105     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 106     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},        /* for now */
 107     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 108     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 109     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 110     {"Link", HDR_LINK, ftStr},
 111     {"Location", HDR_LOCATION, ftStr},
 112     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 113     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 114     {"Pragma", HDR_PRAGMA, ftStr},
 115     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 116     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 117     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 118     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 119     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 120     {"Public", HDR_PUBLIC, ftStr},
 121     {"Range", HDR_RANGE, ftPRange},
 122     {"Referer", HDR_REFERER, ftStr},
 123     {"Request-Range", HDR_REQUEST_RANGE, ftPRange},     /* usually matches HDR_RANGE */
 124     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!) */
 125     {"Server", HDR_SERVER, ftStr},
 126     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 127     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 128     {"TE", HDR_TE, ftStr},
 129     {"Title", HDR_TITLE, ftStr},
 130     {"Trailer", HDR_TRAILER, ftStr},
 131     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 132     {"Translate", HDR_TRANSLATE, ftStr},        /* for now. may need to crop */
 133     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 134     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 135     {"User-Agent", HDR_USER_AGENT, ftStr},
 136     {"Vary", HDR_VARY, ftStr},  /* for now */
 137     {"Via", HDR_VIA, ftStr},    /* for now */
 138     {"Warning", HDR_WARNING, ftStr},    /* for now */
 139     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 140     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 141     {"X-Cache", HDR_X_CACHE, ftStr},
 142     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 143     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 144     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 145     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 146     {"Negotiate", HDR_NEGOTIATE, ftStr},
 147 #if X_ACCELERATOR_VARY
 148     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 149 #endif
 150 #if USE_ADAPTATION
 151     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 152 #endif
 153     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 154     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 155     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 156     {"Other:", HDR_OTHER, ftStr}        /* ':' will not allow matches */
 157 };
 158
 159 static HttpHeaderFieldInfo *Headers = NULL;
 160
 161 http_hdr_type &operator++ (http_hdr_type &aHeader)
 162 {
 163     int tmp = (int)aHeader;
 164     aHeader = (http_hdr_type)(++tmp);
 165     return aHeader;
 166 }
 167
 168
 169 /*
 170  * headers with field values defined as #(values) in HTTP/1.1
 171  * Headers that are currently not recognized, are commented out.
 172  */
 173 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 174 static http_hdr_type ListHeadersArr[] = {
 175     HDR_ACCEPT,
 176     HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 177     HDR_ACCEPT_RANGES, HDR_ALLOW,
 178     HDR_CACHE_CONTROL,
 179     HDR_CONTENT_ENCODING,
 180     HDR_CONTENT_LANGUAGE,
 181     HDR_CONNECTION,
 182     HDR_EXPECT,
 183     HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 184     HDR_LINK, HDR_PRAGMA,
 185     HDR_PROXY_CONNECTION,
 186     HDR_PROXY_SUPPORT,
 187     HDR_TRANSFER_ENCODING,
 188     HDR_UPGRADE,
 189     HDR_VARY,
 190     HDR_VIA,
 191     HDR_WARNING,
 192     HDR_WWW_AUTHENTICATE,
 193     HDR_AUTHENTICATION_INFO,
 194     HDR_PROXY_AUTHENTICATION_INFO,
 195     /* HDR_TE, HDR_TRAILER */
 196 #if X_ACCELERATOR_VARY
 197     HDR_X_ACCELERATOR_VARY,
 198 #endif
 199 #if USE_ADAPTATION
 200     HDR_X_NEXT_SERVICES,
 201 #endif
 202     HDR_SURROGATE_CAPABILITY,
 203     HDR_SURROGATE_CONTROL,
 204     HDR_X_FORWARDED_FOR
 205 };
 206
 207 /* general-headers */
 208 static http_hdr_type GeneralHeadersArr[] = {
 209     HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 210     HDR_TRANSFER_ENCODING,
 211     HDR_UPGRADE,
 212     /* HDR_TRAILER, */
 213     HDR_VIA,
 214 };
 215
 216 /* entity-headers */
 217 static http_hdr_type EntityHeadersArr[] = {
 218     HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 219     HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 220     HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 221     HDR_OTHER
 222 };
 223
 224 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 225 static http_hdr_type ReplyHeadersArr[] = {
 226     HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 227     HDR_ACCEPT_RANGES, HDR_AGE,
 228     HDR_LOCATION, HDR_MAX_FORWARDS,
 229     HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_SET_COOKIE2,
 230     HDR_VARY,
 231     HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 232     HDR_X_CACHE_LOOKUP,
 233     HDR_X_REQUEST_URI,
 234 #if X_ACCELERATOR_VARY
 235     HDR_X_ACCELERATOR_VARY,
 236 #endif
 237 #if USE_ADAPTATION
 238     HDR_X_NEXT_SERVICES,
 239 #endif
 240     HDR_X_SQUID_ERROR,
 241     HDR_SURROGATE_CONTROL
 242 };
 243
 244 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 245 static http_hdr_type RequestHeadersArr[] = {
 246     HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 247     HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 248     HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION,
 249     HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 250     HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 251 };
 252
 253 static HttpHeaderMask HopByHopHeadersMask;
 254 static http_hdr_type HopByHopHeadersArr[] = {
 255     HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 256     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 257 };
 258
 259 /* header accounting */
 260 static HttpHeaderStat HttpHeaderStats[] = {
 261     {"all"},
 262 #if USE_HTCP
 263     {"HTCP reply"},
 264 #endif
 265     {"request"},
 266     {"reply"}
 267 };
 268 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 269
 270 static int HeaderEntryParsedCount = 0;
 271
 272 /*
 273  * local routines
 274  */
 275
 276 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 277
 278 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 279
 280 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 281 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 282
 283 /*
 284  * Module initialization routines
 285  */
 286
 287 static void
 288 httpHeaderRegisterWithCacheManager(void)
 289 {
 290     Mgr::RegisterAction("http_headers",
 291                         "HTTP Header Statistics",
 292                         httpHeaderStoreReport, 0, 1);
 293 }
 294
 295 void
 296 httpHeaderInitModule(void)
 297 {
 298     int i;
 299     /* check that we have enough space for masks */
 300     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 301     /* all headers must be described */
 302     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 303
 304     if (!Headers)
 305         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 306
 307     /* create masks */
 308     httpHeaderMaskInit(&ListHeadersMask, 0);
 309
 310     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 311
 312     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 313
 314     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 315
 316     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 317
 318     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 319
 320     httpHeaderMaskInit(&RequestHeadersMask, 0);
 321
 322     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 323
 324     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 325
 326     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 327
 328     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 329
 330     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 331
 332     /* init header stats */
 333     assert(HttpHeaderStatCount == hoReply + 1);
 334
 335     for (i = 0; i < HttpHeaderStatCount; i++)
 336         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 337
 338     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 339
 340     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 341
 342 #if USE_HTCP
 343
 344     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 345
 346 #endif
 347     /* init dependent modules */
 348     httpHdrCcInitModule();
 349
 350     httpHdrScInitModule();
 351
 352     httpHeaderRegisterWithCacheManager();
 353 }
 354
 355 void
 356 httpHeaderCleanModule(void)
 357 {
 358     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 359     Headers = NULL;
 360     httpHdrCcCleanModule();
 361     httpHdrScCleanModule();
 362 }
 363
 364 static void
 365 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 366 {
 367     assert(hs);
 368     assert(label);
 369     memset(hs, 0, sizeof(HttpHeaderStat));
 370     hs->label = label;
 371     statHistEnumInit(&hs->hdrUCountDistr, 32);  /* not a real enum */
 372     statHistEnumInit(&hs->fieldTypeDistr, HDR_ENUM_END);
 373     statHistEnumInit(&hs->ccTypeDistr, CC_ENUM_END);
 374     statHistEnumInit(&hs->scTypeDistr, SC_ENUM_END);
 375 }
 376
 377 /*
 378  * HttpHeader Implementation
 379  */
 380
 381 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 382 {
 383     httpHeaderMaskInit(&mask, 0);
 384 }
 385
 386 HttpHeader::HttpHeader(http_hdr_owner_type const &anOwner) : owner (anOwner), len (0)
 387 {
 388     assert(anOwner > hoNone && anOwner <= hoReply);
 389     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 390     httpHeaderMaskInit(&mask, 0);
 391 }
 392
 393 HttpHeader::~HttpHeader()
 394 {
 395     clean();
 396 }
 397
 398 void
 399 HttpHeader::clean()
 400 {
 401     HttpHeaderPos pos = HttpHeaderInitPos;
 402     HttpHeaderEntry *e;
 403
 404     assert(owner > hoNone && owner <= hoReply);
 405     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 406
 407     PROF_start(HttpHeaderClean);
 408
 409     /*
 410      * An unfortunate bug.  The entries array is initialized
 411      * such that count is set to zero.  httpHeaderClean() seems to
 412      * be called both when 'hdr' is created, and destroyed.  Thus,
 413      * we accumulate a large number of zero counts for 'hdr' before
 414      * it is ever used.  Can't think of a good way to fix it, except
 415      * adding a state variable that indicates whether or not 'hdr'
 416      * has been used.  As a hack, just never count zero-sized header
 417      * arrays.
 418      */
 419
 420     if (0 != entries.count)
 421         statHistCount(&HttpHeaderStats[owner].hdrUCountDistr, entries.count);
 422
 423     HttpHeaderStats[owner].destroyedCount++;
 424
 425     HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 426
 427     while ((e = getEntry(&pos))) {
 428         /* tmp hack to try to avoid coredumps */
 429
 430         if (e->id < 0 || e->id >= HDR_ENUM_END) {
 431             debugs(55, 0, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 432         } else {
 433             statHistCount(&HttpHeaderStats[owner].fieldTypeDistr, e->id);
 434             /* yes, this deletion leaves us in an inconsistent state */
 435             delete e;
 436         }
 437     }
 438     entries.clean();
 439     httpHeaderMaskInit(&mask, 0);
 440     PROF_stop(HttpHeaderClean);
 441 }
 442
 443 /* append entries (also see httpHeaderUpdate) */
 444 void
 445 HttpHeader::append(const HttpHeader * src)
 446 {
 447     const HttpHeaderEntry *e;
 448     HttpHeaderPos pos = HttpHeaderInitPos;
 449     assert(src);
 450     assert(src != this);
 451     debugs(55, 7, "appending hdr: " << this << " += " << src);
 452
 453     while ((e = src->getEntry(&pos))) {
 454         addEntry(e->clone());
 455     }
 456 }
 457
 458 /* use fresh entries to replace old ones */
 459 void
 460 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 461 {
 462     assert (old);
 463     old->update (fresh, denied_mask);
 464 }
 465
 466 void
 467 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 468 {
 469     const HttpHeaderEntry *e;
 470     HttpHeaderPos pos = HttpHeaderInitPos;
 471     assert(fresh);
 472     assert(this != fresh);
 473
 474     while ((e = fresh->getEntry(&pos))) {
 475         /* deny bad guys (ok to check for HDR_OTHER) here */
 476
 477         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 478             continue;
 479
 480         if (e->id != HDR_OTHER)
 481             delById(e->id);
 482         else
 483             delByName(e->name.termedBuf());
 484     }
 485
 486     pos = HttpHeaderInitPos;
 487     while ((e = fresh->getEntry(&pos))) {
 488         /* deny bad guys (ok to check for HDR_OTHER) here */
 489
 490         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 491             continue;
 492
 493         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 494
 495         addEntry(e->clone());
 496     }
 497 }
 498
 499 /* just handy in parsing: resets and returns false */
 500 int
 501 HttpHeader::reset()
 502 {
 503     http_hdr_owner_type ho;
 504     ho = owner;
 505     clean();
 506     *this = HttpHeader(ho);
 507     return 0;
 508 }
 509
 510 int
 511 HttpHeader::parse(const char *header_start, const char *header_end)
 512 {
 513     const char *field_ptr = header_start;
 514     HttpHeaderEntry *e, *e2;
 515
 516     PROF_start(HttpHeaderParse);
 517
 518     assert(header_start && header_end);
 519     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 520     HttpHeaderStats[owner].parsedCount++;
 521
 522     char *nulpos;
 523     if ((nulpos = (char*)memchr(header_start, '\0', header_end - header_start))) {
 524         debugs(55, 1, "WARNING: HTTP header contains NULL characters {" <<
 525                getStringPrefix(header_start, nulpos) << "}\nNULL\n{" << getStringPrefix(nulpos+1, header_end));
 526         goto reset;
 527     }
 528
 529     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 530      * continuation lines start with a (single) space or tab */
 531     while (field_ptr < header_end) {
 532         const char *field_start = field_ptr;
 533         const char *field_end;
 534
 535         do {
 536             const char *this_line = field_ptr;
 537             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 538
 539             if (!field_ptr)
 540                 goto reset;     /* missing <LF> */
 541
 542             field_end = field_ptr;
 543
 544             field_ptr++;        /* Move to next line */
 545
 546             if (field_end > this_line && field_end[-1] == '\r') {
 547                 field_end--;    /* Ignore CR LF */
 548
 549                 if (owner == hoRequest && field_end > this_line) {
 550                     bool cr_only = true;
 551                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 552                         if (*p != '\r')
 553                             cr_only = false;
 554                     }
 555                     if (cr_only) {
 556                         debugs(55, 1, "WARNING: Rejecting HTTP request with a CR+ "
 557                                "header field to prevent request smuggling attacks: {" <<
 558                                getStringPrefix(header_start, header_end) << "}");
 559                         goto reset;
 560                     }
 561                 }
 562             }
 563
 564             /* Barf on stray CR characters */
 565             if (memchr(this_line, '\r', field_end - this_line)) {
 566                 debugs(55, 1, "WARNING: suspicious CR characters in HTTP header {" <<
 567                        getStringPrefix(field_start, field_end) << "}");
 568
 569                 if (Config.onoff.relaxed_header_parser) {
 570                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 571
 572                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL)
 573                         *p++ = ' ';
 574                 } else
 575                     goto reset;
 576             }
 577
 578             if (this_line + 1 == field_end && this_line > field_start) {
 579                 debugs(55, 1, "WARNING: Blank continuation line in HTTP header {" <<
 580                        getStringPrefix(header_start, header_end) << "}");
 581                 goto reset;
 582             }
 583         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 584
 585         if (field_start == field_end) {
 586             if (field_ptr < header_end) {
 587                 debugs(55, 1, "WARNING: unparseable HTTP header field near {" <<
 588                        getStringPrefix(field_start, header_end) << "}");
 589                 goto reset;
 590             }
 591
 592             break;              /* terminating blank line */
 593         }
 594
 595         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 596             debugs(55, 1, "WARNING: unparseable HTTP header field {" <<
 597                    getStringPrefix(field_start, field_end) << "}");
 598             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 599                    " in {" << getStringPrefix(header_start, header_end) << "}");
 600
 601             if (Config.onoff.relaxed_header_parser)
 602                 continue;
 603             else
 604                 goto reset;
 605         }
 606
 607         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 608 //            if (e->value.cmp(e2->value.termedBuf()) != 0) {
 609             if (e->value != e2->value) {
 610                 int64_t l1, l2;
 611                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 612                        "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 613
 614                 if (!Config.onoff.relaxed_header_parser) {
 615                     delete e;
 616                     goto reset;
 617                 }
 618
 619                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 620                     debugs(55, 1, "WARNING: Unparseable content-length '" << e->value << "'");
 621                     delete e;
 622                     continue;
 623                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 624                     debugs(55, 1, "WARNING: Unparseable content-length '" << e2->value << "'");
 625                     delById(e2->id);
 626                 } else if (l1 > l2) {
 627                     delById(e2->id);
 628                 } else {
 629                     delete e;
 630                     continue;
 631                 }
 632             } else {
 633                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 634                        "NOTICE: found double content-length header");
 635
 636                 if (Config.onoff.relaxed_header_parser) {
 637                     delete e;
 638                     continue;
 639                 } else {
 640                     delete e;
 641                     goto reset;
 642                 }
 643             }
 644         }
 645
 646         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 647             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 648                    "WARNING: found whitespace in HTTP header name {" <<
 649                    getStringPrefix(field_start, field_end) << "}");
 650
 651             if (!Config.onoff.relaxed_header_parser) {
 652                 delete e;
 653                 goto reset;
 654             }
 655         }
 656
 657         addEntry(e);
 658     }
 659
 660     if (chunked()) {
 661         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 662         delById(HDR_CONTENT_LENGTH);
 663     }
 664
 665     PROF_stop(HttpHeaderParse);
 666     return 1;                   /* even if no fields where found, it is a valid header */
 667 reset:
 668     PROF_stop(HttpHeaderParse);
 669     return reset();
 670 }
 671
 672 /* packs all the entries using supplied packer */
 673 void
 674 HttpHeader::packInto(Packer * p) const
 675 {
 676     HttpHeaderPos pos = HttpHeaderInitPos;
 677     const HttpHeaderEntry *e;
 678     assert(p);
 679     debugs(55, 7, "packing hdr: (" << this << ")");
 680     /* pack all entries one by one */
 681     while ((e = getEntry(&pos)))
 682         e->packInto(p);
 683
 684     /* Pack in the "special" entries */
 685
 686     /* Cache-Control */
 687 }
 688
 689 /* returns next valid entry */
 690 HttpHeaderEntry *
 691 HttpHeader::getEntry(HttpHeaderPos * pos) const
 692 {
 693     assert(pos);
 694     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 695
 696     for ((*pos)++; *pos < (ssize_t)entries.count; (*pos)++) {
 697         if (entries.items[*pos])
 698             return (HttpHeaderEntry*)entries.items[*pos];
 699     }
 700
 701     return NULL;
 702 }
 703
 704 /*
 705  * returns a pointer to a specified entry if any
 706  * note that we return one entry so it does not make much sense to ask for
 707  * "list" headers
 708  */
 709 HttpHeaderEntry *
 710 HttpHeader::findEntry(http_hdr_type id) const
 711 {
 712     HttpHeaderPos pos = HttpHeaderInitPos;
 713     HttpHeaderEntry *e;
 714     assert_eid(id);
 715     assert(!CBIT_TEST(ListHeadersMask, id));
 716
 717     /* check mask first */
 718
 719     if (!CBIT_TEST(mask, id))
 720         return NULL;
 721
 722     /* looks like we must have it, do linear search */
 723     while ((e = getEntry(&pos))) {
 724         if (e->id == id)
 725             return e;
 726     }
 727
 728     /* hm.. we thought it was there, but it was not found */
 729     assert(0);
 730
 731     return NULL;                /* not reached */
 732 }
 733
 734 /*
 735  * same as httpHeaderFindEntry
 736  */
 737 HttpHeaderEntry *
 738 HttpHeader::findLastEntry(http_hdr_type id) const
 739 {
 740     HttpHeaderPos pos = HttpHeaderInitPos;
 741     HttpHeaderEntry *e;
 742     HttpHeaderEntry *result = NULL;
 743     assert_eid(id);
 744     assert(!CBIT_TEST(ListHeadersMask, id));
 745
 746     /* check mask first */
 747
 748     if (!CBIT_TEST(mask, id))
 749         return NULL;
 750
 751     /* looks like we must have it, do linear search */
 752     while ((e = getEntry(&pos))) {
 753         if (e->id == id)
 754             result = e;
 755     }
 756
 757     assert(result);             /* must be there! */
 758     return result;
 759 }
 760
 761 /*
 762  * deletes all fields with a given name if any, returns #fields deleted;
 763  */
 764 int
 765 HttpHeader::delByName(const char *name)
 766 {
 767     int count = 0;
 768     HttpHeaderPos pos = HttpHeaderInitPos;
 769     HttpHeaderEntry *e;
 770     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 771     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 772
 773     while ((e = getEntry(&pos))) {
 774         if (!e->name.caseCmp(name))
 775             delAt(pos, count);
 776         else
 777             CBIT_SET(mask, e->id);
 778     }
 779
 780     return count;
 781 }
 782
 783 /* deletes all entries with a given id, returns the #entries deleted */
 784 int
 785 HttpHeader::delById(http_hdr_type id)
 786 {
 787     int count = 0;
 788     HttpHeaderPos pos = HttpHeaderInitPos;
 789     HttpHeaderEntry *e;
 790     debugs(55, 8, this << " del-by-id " << id);
 791     assert_eid(id);
 792     assert(id != HDR_OTHER);            /* does not make sense */
 793
 794     if (!CBIT_TEST(mask, id))
 795         return 0;
 796
 797     while ((e = getEntry(&pos))) {
 798         if (e->id == id)
 799             delAt(pos, count);
 800     }
 801
 802     CBIT_CLR(mask, id);
 803     assert(count);
 804     return count;
 805 }
 806
 807 /*
 808  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 809  * possible to iterate(search) and delete fields at the same time
 810  * NOTE: Does not update the header mask. Caller must follow up with
 811  * a call to refreshMask() if headers_deleted was incremented.
 812  */
 813 void
 814 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 815 {
 816     HttpHeaderEntry *e;
 817     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 818     e = (HttpHeaderEntry*)entries.items[pos];
 819     entries.items[pos] = NULL;
 820     /* decrement header length, allow for ": " and crlf */
 821     len -= e->name.size() + 2 + e->value.size() + 2;
 822     assert(len >= 0);
 823     delete e;
 824     ++headers_deleted;
 825 }
 826
 827 /*
 828  * Compacts the header storage
 829  */
 830 void
 831 HttpHeader::compact()
 832 {
 833     entries.prune(NULL);
 834 }
 835
 836 /*
 837  * Refreshes the header mask. Required after delAt() calls.
 838  */
 839 void
 840 HttpHeader::refreshMask()
 841 {
 842     httpHeaderMaskInit(&mask, 0);
 843     debugs(55, 7, "refreshing the mask in hdr " << this);
 844     HttpHeaderPos pos = HttpHeaderInitPos;
 845     while (HttpHeaderEntry *e = getEntry(&pos)) {
 846         CBIT_SET(mask, e->id);
 847     }
 848 }
 849
 850 /* appends an entry;
 851  * does not call e->clone() so one should not reuse "*e"
 852  */
 853 void
 854 HttpHeader::addEntry(HttpHeaderEntry * e)
 855 {
 856     assert(e);
 857     assert_eid(e->id);
 858     assert(e->name.size());
 859
 860     debugs(55, 9, this << " adding entry: " << e->id << " at " <<
 861            entries.count);
 862
 863     if (CBIT_TEST(mask, e->id))
 864         Headers[e->id].stat.repCount++;
 865     else
 866         CBIT_SET(mask, e->id);
 867
 868     entries.push_back(e);
 869
 870     /* increment header length, allow for ": " and crlf */
 871     len += e->name.size() + 2 + e->value.size() + 2;
 872 }
 873
 874 /* inserts an entry;
 875  * does not call e->clone() so one should not reuse "*e"
 876  */
 877 void
 878 HttpHeader::insertEntry(HttpHeaderEntry * e)
 879 {
 880     assert(e);
 881     assert_eid(e->id);
 882
 883     debugs(55, 7, this << " adding entry: " << e->id << " at " <<
 884            entries.count);
 885
 886     if (CBIT_TEST(mask, e->id))
 887         Headers[e->id].stat.repCount++;
 888     else
 889         CBIT_SET(mask, e->id);
 890
 891     entries.insert(e);
 892
 893     /* increment header length, allow for ": " and crlf */
 894     len += e->name.size() + 2 + e->value.size() + 2;
 895 }
 896
 897 bool
 898 HttpHeader::getList(http_hdr_type id, String *s) const
 899 {
 900     HttpHeaderEntry *e;
 901     HttpHeaderPos pos = HttpHeaderInitPos;
 902     debugs(55, 9, this << " joining for id " << id);
 903     /* only fields from ListHeaders array can be "listed" */
 904     assert(CBIT_TEST(ListHeadersMask, id));
 905
 906     if (!CBIT_TEST(mask, id))
 907         return false;
 908
 909     while ((e = getEntry(&pos))) {
 910         if (e->id == id)
 911             strListAdd(s, e->value.termedBuf(), ',');
 912     }
 913
 914     /*
 915      * note: we might get an empty (size==0) string if there was an "empty"
 916      * header. This results in an empty length String, which may have a NULL
 917      * buffer.
 918      */
 919     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 920     if (!s->size())
 921         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 922     else
 923         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 924
 925     return true;
 926 }
 927
 928 /* return a list of entries with the same id separated by ',' and ws */
 929 String
 930 HttpHeader::getList(http_hdr_type id) const
 931 {
 932     HttpHeaderEntry *e;
 933     HttpHeaderPos pos = HttpHeaderInitPos;
 934     debugs(55, 9, this << "joining for id " << id);
 935     /* only fields from ListHeaders array can be "listed" */
 936     assert(CBIT_TEST(ListHeadersMask, id));
 937
 938     if (!CBIT_TEST(mask, id))
 939         return String();
 940
 941     String s;
 942
 943     while ((e = getEntry(&pos))) {
 944         if (e->id == id)
 945             strListAdd(&s, e->value.termedBuf(), ',');
 946     }
 947
 948     /*
 949      * note: we might get an empty (size==0) string if there was an "empty"
 950      * header. This results in an empty length String, which may have a NULL
 951      * buffer.
 952      */
 953     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 954     if (!s.size())
 955         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 956     else
 957         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 958
 959     return s;
 960 }
 961
 962 /* return a string or list of entries with the same id separated by ',' and ws */
 963 String
 964 HttpHeader::getStrOrList(http_hdr_type id) const
 965 {
 966     HttpHeaderEntry *e;
 967
 968     if (CBIT_TEST(ListHeadersMask, id))
 969         return getList(id);
 970
 971     if ((e = findEntry(id)))
 972         return e->value;
 973
 974     return String();
 975 }
 976
 977 /*
 978  * Returns the value of the specified header.
 979  */
 980 String
 981 HttpHeader::getByName(const char *name) const
 982 {
 983     http_hdr_type id;
 984     HttpHeaderPos pos = HttpHeaderInitPos;
 985     HttpHeaderEntry *e;
 986
 987     assert(name);
 988
 989     /* First try the quick path */
 990     id = httpHeaderIdByNameDef(name, strlen(name));
 991
 992     if (id != -1)
 993         return getStrOrList(id);
 994
 995     String result;
 996
 997     /* Sorry, an unknown header name. Do linear search */
 998     while ((e = getEntry(&pos))) {
 999         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1000             strListAdd(&result, e->value.termedBuf(), ',');
1001         }
1002     }
1003
1004     return result;
1005 }
1006
1007 /*
1008  * Returns a the value of the specified list member, if any.
1009  */
1010 String
1011 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1012 {
1013     String header;
1014     const char *pos = NULL;
1015     const char *item;
1016     int ilen;
1017     int mlen = strlen(member);
1018
1019     assert(name);
1020
1021     header = getByName(name);
1022
1023     String result;
1024
1025     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1026         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1027             result.append(item + mlen + 1, ilen - mlen - 1);
1028             break;
1029         }
1030     }
1031
1032     return result;
1033 }
1034
1035 /*
1036  * returns a the value of the specified list member, if any.
1037  */
1038 String
1039 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1040 {
1041     String header;
1042     const char *pos = NULL;
1043     const char *item;
1044     int ilen;
1045     int mlen = strlen(member);
1046
1047     assert(id >= 0);
1048
1049     header = getStrOrList(id);
1050     String result;
1051
1052     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1053         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1054             result.append(item + mlen + 1, ilen - mlen - 1);
1055             break;
1056         }
1057     }
1058
1059     header.clean();
1060     return result;
1061 }
1062
1063 /* test if a field is present */
1064 int
1065 HttpHeader::has(http_hdr_type id) const
1066 {
1067     assert_eid(id);
1068     assert(id != HDR_OTHER);
1069     debugs(55, 9, this << " lookup for " << id);
1070     return CBIT_TEST(mask, id);
1071 }
1072
1073 void
1074 HttpHeader::putInt(http_hdr_type id, int number)
1075 {
1076     assert_eid(id);
1077     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1078     assert(number >= 0);
1079     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1080 }
1081
1082 void
1083 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1084 {
1085     assert_eid(id);
1086     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1087     assert(number >= 0);
1088     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1089 }
1090
1091 void
1092 HttpHeader::putTime(http_hdr_type id, time_t htime)
1093 {
1094     assert_eid(id);
1095     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1096     assert(htime >= 0);
1097     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1098 }
1099
1100 void
1101 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1102 {
1103     assert_eid(id);
1104     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1105     assert(htime >= 0);
1106     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1107 }
1108
1109 void
1110 HttpHeader::putStr(http_hdr_type id, const char *str)
1111 {
1112     assert_eid(id);
1113     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1114     assert(str);
1115     addEntry(new HttpHeaderEntry(id, NULL, str));
1116 }
1117
1118 void
1119 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1120 {
1121     assert(auth_scheme && realm);
1122     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1123 }
1124
1125 void
1126 HttpHeader::putCc(const HttpHdrCc * cc)
1127 {
1128     MemBuf mb;
1129     Packer p;
1130     assert(cc);
1131     /* remove old directives if any */
1132     delById(HDR_CACHE_CONTROL);
1133     /* pack into mb */
1134     mb.init();
1135     packerToMemInit(&p, &mb);
1136     httpHdrCcPackInto(cc, &p);
1137     /* put */
1138     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1139     /* cleanup */
1140     packerClean(&p);
1141     mb.clean();
1142 }
1143
1144 void
1145 HttpHeader::putContRange(const HttpHdrContRange * cr)
1146 {
1147     MemBuf mb;
1148     Packer p;
1149     assert(cr);
1150     /* remove old directives if any */
1151     delById(HDR_CONTENT_RANGE);
1152     /* pack into mb */
1153     mb.init();
1154     packerToMemInit(&p, &mb);
1155     httpHdrContRangePackInto(cr, &p);
1156     /* put */
1157     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1158     /* cleanup */
1159     packerClean(&p);
1160     mb.clean();
1161 }
1162
1163 void
1164 HttpHeader::putRange(const HttpHdrRange * range)
1165 {
1166     MemBuf mb;
1167     Packer p;
1168     assert(range);
1169     /* remove old directives if any */
1170     delById(HDR_RANGE);
1171     /* pack into mb */
1172     mb.init();
1173     packerToMemInit(&p, &mb);
1174     range->packInto(&p);
1175     /* put */
1176     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1177     /* cleanup */
1178     packerClean(&p);
1179     mb.clean();
1180 }
1181
1182 void
1183 HttpHeader::putSc(HttpHdrSc *sc)
1184 {
1185     MemBuf mb;
1186     Packer p;
1187     assert(sc);
1188     /* remove old directives if any */
1189     delById(HDR_SURROGATE_CONTROL);
1190     /* pack into mb */
1191     mb.init();
1192     packerToMemInit(&p, &mb);
1193     httpHdrScPackInto(sc, &p);
1194     /* put */
1195     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1196     /* cleanup */
1197     packerClean(&p);
1198     mb.clean();
1199 }
1200
1201 void
1202 HttpHeader::putWarning(const int code, const char *const text)
1203 {
1204     char buf[512];
1205     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1206     putStr(HDR_WARNING, buf);
1207 }
1208
1209 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1210 void
1211 HttpHeader::putExt(const char *name, const char *value)
1212 {
1213     assert(name && value);
1214     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1215     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1216 }
1217
1218 int
1219 HttpHeader::getInt(http_hdr_type id) const
1220 {
1221     assert_eid(id);
1222     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1223     HttpHeaderEntry *e;
1224
1225     if ((e = findEntry(id)))
1226         return e->getInt();
1227
1228     return -1;
1229 }
1230
1231 int64_t
1232 HttpHeader::getInt64(http_hdr_type id) const
1233 {
1234     assert_eid(id);
1235     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1236     HttpHeaderEntry *e;
1237
1238     if ((e = findEntry(id)))
1239         return e->getInt64();
1240
1241     return -1;
1242 }
1243
1244 time_t
1245 HttpHeader::getTime(http_hdr_type id) const
1246 {
1247     HttpHeaderEntry *e;
1248     time_t value = -1;
1249     assert_eid(id);
1250     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1251
1252     if ((e = findEntry(id))) {
1253         value = parse_rfc1123(e->value.termedBuf());
1254         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1255     }
1256
1257     return value;
1258 }
1259
1260 /* sync with httpHeaderGetLastStr */
1261 const char *
1262 HttpHeader::getStr(http_hdr_type id) const
1263 {
1264     HttpHeaderEntry *e;
1265     assert_eid(id);
1266     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1267
1268     if ((e = findEntry(id))) {
1269         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1270         return e->value.termedBuf();
1271     }
1272
1273     return NULL;
1274 }
1275
1276 /* unusual */
1277 const char *
1278 HttpHeader::getLastStr(http_hdr_type id) const
1279 {
1280     HttpHeaderEntry *e;
1281     assert_eid(id);
1282     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1283
1284     if ((e = findLastEntry(id))) {
1285         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1286         return e->value.termedBuf();
1287     }
1288
1289     return NULL;
1290 }
1291
1292 HttpHdrCc *
1293 HttpHeader::getCc() const
1294 {
1295     HttpHdrCc *cc;
1296     String s;
1297
1298     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1299         return NULL;
1300     PROF_start(HttpHeader_getCc);
1301
1302     getList(HDR_CACHE_CONTROL, &s);
1303
1304     cc = httpHdrCcParseCreate(&s);
1305
1306     HttpHeaderStats[owner].ccParsedCount++;
1307
1308     if (cc)
1309         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1310
1311     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1312
1313     PROF_stop(HttpHeader_getCc);
1314
1315     return cc;
1316 }
1317
1318 HttpHdrRange *
1319 HttpHeader::getRange() const
1320 {
1321     HttpHdrRange *r = NULL;
1322     HttpHeaderEntry *e;
1323     /* some clients will send "Request-Range" _and_ *matching* "Range"
1324      * who knows, some clients might send Request-Range only;
1325      * this "if" should work correctly in both cases;
1326      * hopefully no clients send mismatched headers! */
1327
1328     if ((e = findEntry(HDR_RANGE)) ||
1329             (e = findEntry(HDR_REQUEST_RANGE))) {
1330         r = HttpHdrRange::ParseCreate(&e->value);
1331         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1332     }
1333
1334     return r;
1335 }
1336
1337 HttpHdrSc *
1338 HttpHeader::getSc() const
1339 {
1340     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1341         return NULL;
1342
1343     String s;
1344
1345     (void) getList(HDR_SURROGATE_CONTROL, &s);
1346
1347     HttpHdrSc *sc = httpHdrScParseCreate(&s);
1348
1349     HttpHeaderStats[owner].ccParsedCount++;
1350
1351     if (sc)
1352         httpHdrScUpdateStats(sc, &HttpHeaderStats[owner].scTypeDistr);
1353
1354     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1355
1356     return sc;
1357 }
1358
1359 HttpHdrContRange *
1360 HttpHeader::getContRange() const
1361 {
1362     HttpHdrContRange *cr = NULL;
1363     HttpHeaderEntry *e;
1364
1365     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1366         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1367         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1368     }
1369
1370     return cr;
1371 }
1372
1373 const char *
1374 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1375 {
1376     const char *field;
1377     int l;
1378     assert(auth_scheme);
1379     field = getStr(id);
1380
1381     if (!field)                 /* no authorization field */
1382         return NULL;
1383
1384     l = strlen(auth_scheme);
1385
1386     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1387         return NULL;
1388
1389     field += l;
1390
1391     if (!xisspace(*field))      /* wrong scheme */
1392         return NULL;
1393
1394     /* skip white space */
1395     field += xcountws(field);
1396
1397     if (!*field)                /* no authorization cookie */
1398         return NULL;
1399
1400     return base64_decode(field);
1401 }
1402
1403 ETag
1404 HttpHeader::getETag(http_hdr_type id) const
1405 {
1406     ETag etag = {NULL, -1};
1407     HttpHeaderEntry *e;
1408     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1409
1410     if ((e = findEntry(id)))
1411         etagParseInit(&etag, e->value.termedBuf());
1412
1413     return etag;
1414 }
1415
1416 TimeOrTag
1417 HttpHeader::getTimeOrTag(http_hdr_type id) const
1418 {
1419     TimeOrTag tot;
1420     HttpHeaderEntry *e;
1421     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1422     memset(&tot, 0, sizeof(tot));
1423
1424     if ((e = findEntry(id))) {
1425         const char *str = e->value.termedBuf();
1426         /* try as an ETag */
1427
1428         if (etagParseInit(&tot.tag, str)) {
1429             tot.valid = tot.tag.str != NULL;
1430             tot.time = -1;
1431         } else {
1432             /* or maybe it is time? */
1433             tot.time = parse_rfc1123(str);
1434             tot.valid = tot.time >= 0;
1435             tot.tag.str = NULL;
1436         }
1437     }
1438
1439     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1440     return tot;
1441 }
1442
1443 /*
1444  * HttpHeaderEntry
1445  */
1446
1447 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1448 {
1449     assert_eid(anId);
1450     id = anId;
1451
1452     if (id != HDR_OTHER)
1453         name = Headers[id].name;
1454     else
1455         name = aName;
1456
1457     value = aValue;
1458
1459     Headers[id].stat.aliveCount++;
1460
1461     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1462 }
1463
1464 HttpHeaderEntry::~HttpHeaderEntry()
1465 {
1466     assert_eid(id);
1467     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1468     /* clean name if needed */
1469
1470     if (id == HDR_OTHER)
1471         name.clean();
1472
1473     value.clean();
1474
1475     assert(Headers[id].stat.aliveCount);
1476
1477     Headers[id].stat.aliveCount--;
1478
1479     id = HDR_BAD_HDR;
1480 }
1481
1482 /* parses and inits header entry, returns true/false */
1483 HttpHeaderEntry *
1484 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1485 {
1486     /* note: name_start == field_start */
1487     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1488     int name_len = name_end ? name_end - field_start :0;
1489     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1490     /* note: value_end == field_end */
1491
1492     HeaderEntryParsedCount++;
1493
1494     /* do we have a valid field name within this field? */
1495
1496     if (!name_len || name_end > field_end)
1497         return NULL;
1498
1499     if (name_len > 65534) {
1500         /* String must be LESS THAN 64K and it adds a terminating NULL */
1501         debugs(55, 1, "WARNING: ignoring header name of " << name_len << " bytes");
1502         return NULL;
1503     }
1504
1505     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1506         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1507                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1508
1509         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1510             name_len--;
1511
1512         if (!name_len)
1513             return NULL;
1514     }
1515
1516     /* now we know we can parse it */
1517
1518     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1519
1520     /* is it a "known" field? */
1521     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1522
1523     String name;
1524
1525     String value;
1526
1527     if (id < 0)
1528         id = HDR_OTHER;
1529
1530     assert_eid(id);
1531
1532     /* set field name */
1533     if (id == HDR_OTHER)
1534         name.limitInit(field_start, name_len);
1535     else
1536         name = Headers[id].name;
1537
1538     /* trim field value */
1539     while (value_start < field_end && xisspace(*value_start))
1540         value_start++;
1541
1542     while (value_start < field_end && xisspace(field_end[-1]))
1543         field_end--;
1544
1545     if (field_end - value_start > 65534) {
1546         /* String must be LESS THAN 64K and it adds a terminating NULL */
1547         debugs(55, 1, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1548
1549         if (id == HDR_OTHER)
1550             name.clean();
1551
1552         return NULL;
1553     }
1554
1555     /* set field value */
1556     value.limitInit(value_start, field_end - value_start);
1557
1558     Headers[id].stat.seenCount++;
1559
1560     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1561
1562     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1563 }
1564
1565 HttpHeaderEntry *
1566 HttpHeaderEntry::clone() const
1567 {
1568     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1569 }
1570
1571 void
1572 HttpHeaderEntry::packInto(Packer * p) const
1573 {
1574     assert(p);
1575     packerAppend(p, name.rawBuf(), name.size());
1576     packerAppend(p, ": ", 2);
1577     packerAppend(p, value.rawBuf(), value.size());
1578     packerAppend(p, "\r\n", 2);
1579 }
1580
1581 int
1582 HttpHeaderEntry::getInt() const
1583 {
1584     assert_eid (id);
1585     assert (Headers[id].type == ftInt);
1586     int val = -1;
1587     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1588     httpHeaderNoteParsedEntry(id, value, !ok);
1589     /* XXX: Should we check ok - ie
1590      * return ok ? -1 : value;
1591      */
1592     return val;
1593 }
1594
1595 int64_t
1596 HttpHeaderEntry::getInt64() const
1597 {
1598     assert_eid (id);
1599     assert (Headers[id].type == ftInt64);
1600     int64_t val = -1;
1601     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1602     httpHeaderNoteParsedEntry(id, value, !ok);
1603     /* XXX: Should we check ok - ie
1604      * return ok ? -1 : value;
1605      */
1606     return val;
1607 }
1608
1609 static void
1610 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1611 {
1612     Headers[id].stat.parsCount++;
1613
1614     if (error) {
1615         Headers[id].stat.errCount++;
1616         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name << ": " << context << "'");
1617     }
1618 }
1619
1620 /*
1621  * Reports
1622  */
1623
1624 /* tmp variable used to pass stat info to dumpers */
1625 extern const HttpHeaderStat *dump_stat;         /* argh! */
1626 const HttpHeaderStat *dump_stat = NULL;
1627
1628 void
1629 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1630 {
1631     const int id = (int) val;
1632     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1633     const char *name = valid_id ? Headers[id].name.termedBuf() : "INVALID";
1634     int visible = count > 0;
1635     /* for entries with zero count, list only those that belong to current type of message */
1636
1637     if (!visible && valid_id && dump_stat->owner_mask)
1638         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1639
1640     if (visible)
1641         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1642                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1643 }
1644
1645 static void
1646 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1647 {
1648     if (count)
1649         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1650                           idx, (int) val, count,
1651                           xpercent(count, dump_stat->destroyedCount));
1652 }
1653
1654
1655 static void
1656 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1657 {
1658     assert(hs && e);
1659
1660     dump_stat = hs;
1661     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1662     storeAppendPrintf(e, "\nField type distribution\n");
1663     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1664                       "id", "name", "count", "#/header");
1665     statHistDump(&hs->fieldTypeDistr, e, httpHeaderFieldStatDumper);
1666     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1667     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1668                       "id", "name", "count", "#/cc_field");
1669     statHistDump(&hs->ccTypeDistr, e, httpHdrCcStatDumper);
1670     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1671     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1672                       "id", "name", "count", "#/sc_field");
1673     statHistDump(&hs->scTypeDistr, e, httpHdrScStatDumper);
1674     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1675     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1676                       "id", "#flds", "count", "%total");
1677     statHistDump(&hs->hdrUCountDistr, e, httpHeaderFldsPerHdrDumper);
1678     dump_stat = NULL;
1679 }
1680
1681 void
1682 httpHeaderStoreReport(StoreEntry * e)
1683 {
1684     int i;
1685     http_hdr_type ht;
1686     assert(e);
1687
1688     HttpHeaderStats[0].parsedCount =
1689         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1690     HttpHeaderStats[0].ccParsedCount =
1691         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1692     HttpHeaderStats[0].destroyedCount =
1693         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1694     HttpHeaderStats[0].busyDestroyedCount =
1695         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1696
1697     for (i = 1; i < HttpHeaderStatCount; i++) {
1698         httpHeaderStatDump(HttpHeaderStats + i, e);
1699         storeAppendPrintf(e, "%s\n", "<br>");
1700     }
1701
1702     /* field stats for all messages */
1703     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1704
1705     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1706                       "id", "name", "#alive", "%err", "%repeat");
1707
1708     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1709         HttpHeaderFieldInfo *f = Headers + ht;
1710         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1711                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1712                           xpercent(f->stat.errCount, f->stat.parsCount),
1713                           xpercent(f->stat.repCount, f->stat.seenCount));
1714     }
1715
1716     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1717                       HttpHeaderStats[hoRequest].parsedCount,
1718                       HttpHeaderStats[hoReply].parsedCount,
1719                       HttpHeaderStats[0].parsedCount);
1720     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1721 }
1722
1723 http_hdr_type
1724 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1725 {
1726     int i;
1727
1728     for (i = 0; i < end; ++i) {
1729         if (name_len >= 0 && name_len != info[i].name.size())
1730             continue;
1731
1732         if (!strncasecmp(name, info[i].name.termedBuf(),
1733                          name_len < 0 ? info[i].name.size() + 1 : name_len))
1734             return info[i].id;
1735     }
1736
1737     return HDR_BAD_HDR;
1738 }
1739
1740 http_hdr_type
1741 httpHeaderIdByNameDef(const char *name, int name_len)
1742 {
1743     if (!Headers)
1744         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1745
1746     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1747 }
1748
1749 const char *
1750 httpHeaderNameById(int id)
1751 {
1752     if (!Headers)
1753         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1754
1755     assert(id >= 0 && id < HDR_ENUM_END);
1756
1757     return Headers[id].name.termedBuf();
1758 }
1759
1760 int
1761 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1762 {
1763     int result = 0;
1764     const char *pos = NULL;
1765     const char *item;
1766     int ilen;
1767     int mlen = strlen(member);
1768
1769     assert(id >= 0);
1770
1771     String header (getStrOrList(id));
1772
1773     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1774         if (strncasecmp(item, member, mlen) == 0
1775                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1776             result = 1;
1777             break;
1778         }
1779     }
1780
1781     return result;
1782 }
1783
1784 int
1785 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1786 {
1787     int result = 0;
1788     const char *pos = NULL;
1789     const char *item;
1790     int ilen;
1791     int mlen = strlen(member);
1792
1793     assert(name);
1794
1795     String header (getByName(name));
1796
1797     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1798         if (strncasecmp(item, member, mlen) == 0
1799                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1800             result = 1;
1801             break;
1802         }
1803     }
1804
1805     return result;
1806 }
1807
1808 void
1809 HttpHeader::removeHopByHopEntries()
1810 {
1811     removeConnectionHeaderEntries();
1812
1813     const HttpHeaderEntry *e;
1814     HttpHeaderPos pos = HttpHeaderInitPos;
1815     int headers_deleted = 0;
1816     while ((e = getEntry(&pos))) {
1817         int id = e->id;
1818         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1819             delAt(pos, headers_deleted);
1820             CBIT_CLR(mask, id);
1821         }
1822     }
1823 }
1824
1825 void
1826 HttpHeader::removeConnectionHeaderEntries()
1827 {
1828     if (has(HDR_CONNECTION)) {
1829         /* anything that matches Connection list member will be deleted */
1830         String strConnection;
1831
1832         (void) getList(HDR_CONNECTION, &strConnection);
1833         const HttpHeaderEntry *e;
1834         HttpHeaderPos pos = HttpHeaderInitPos;
1835         /*
1836          * think: on-average-best nesting of the two loops (hdrEntry
1837          * and strListItem) @?@
1838          */
1839         /*
1840          * maybe we should delete standard stuff ("keep-alive","close")
1841          * from strConnection first?
1842          */
1843
1844         int headers_deleted = 0;
1845         while ((e = getEntry(&pos))) {
1846             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1847                 delAt(pos, headers_deleted);
1848         }
1849         if (headers_deleted)
1850             refreshMask();
1851     }
1852 }