src/HttpHeader.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 55    HTTP Header
   6  * AUTHOR: Alex Rousskov
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "base64.h"
  38 #include "HttpHdrContRange.h"
  39 #include "HttpHdrSc.h"
  40 #include "HttpHeader.h"
  41 #include "MemBuf.h"
  42 #include "mgr/Registration.h"
  43 #include "rfc1123.h"
  44 #include "Store.h"
  45
  46 /*
  47  * On naming conventions:
  48  *
  49  * HTTP/1.1 defines message-header as
  50  *
  51  * message-header = field-name ":" [ field-value ] CRLF
  52  * field-name     = token
  53  * field-value    = *( field-content | LWS )
  54  *
  55  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  56  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  57  *
  58  * HttpHeader is an object that represents all message-headers in a message.
  59  * HttpHeader does not manage start-line.
  60  *
  61  * HttpHeader is implemented as a collection of header "entries".
  62  * An entry is a (field_id, field_name, field_value) triplet.
  63  */
  64
  65
  66 /*
  67  * local constants and vars
  68  */
  69
  70 /*
  71  * A table with major attributes for every known field.
  72  * We calculate name lengths and reorganize this array on start up.
  73  * After reorganization, field id can be used as an index to the table.
  74  */
  75 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  76     {"Accept", HDR_ACCEPT, ftStr},
  77
  78     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  79     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  80     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  81     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  82     {"Age", HDR_AGE, ftInt},
  83     {"Allow", HDR_ALLOW, ftStr},
  84     {"Authorization", HDR_AUTHORIZATION, ftStr},        /* for now */
  85     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  86     {"Connection", HDR_CONNECTION, ftStr},
  87     {"Content-Base", HDR_CONTENT_BASE, ftStr},
  88     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
  89     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  90     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  91     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  92     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  93     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
  94     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
  95     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
  96     {"Cookie", HDR_COOKIE, ftStr},
  97     {"Cookie2", HDR_COOKIE2, ftStr},
  98     {"Date", HDR_DATE, ftDate_1123},
  99     {"ETag", HDR_ETAG, ftETag},
 100     {"Expires", HDR_EXPIRES, ftDate_1123},
 101     {"Expect", HDR_EXPECT, ftStr},
 102     {"From", HDR_FROM, ftStr},
 103     {"Host", HDR_HOST, ftStr},
 104     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 105     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 106     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},        /* for now */
 107     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 108     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 109     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 110     {"Link", HDR_LINK, ftStr},
 111     {"Location", HDR_LOCATION, ftStr},
 112     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 113     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 114     {"Pragma", HDR_PRAGMA, ftStr},
 115     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 116     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 117     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 118     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 119     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 120     {"Public", HDR_PUBLIC, ftStr},
 121     {"Range", HDR_RANGE, ftPRange},
 122     {"Referer", HDR_REFERER, ftStr},
 123     {"Request-Range", HDR_REQUEST_RANGE, ftPRange},     /* usually matches HDR_RANGE */
 124     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!) */
 125     {"Server", HDR_SERVER, ftStr},
 126     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 127     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 128     {"TE", HDR_TE, ftStr},
 129     {"Title", HDR_TITLE, ftStr},
 130     {"Trailer", HDR_TRAILER, ftStr},
 131     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 132     {"Translate", HDR_TRANSLATE, ftStr},        /* for now. may need to crop */
 133     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 134     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 135     {"User-Agent", HDR_USER_AGENT, ftStr},
 136     {"Vary", HDR_VARY, ftStr},  /* for now */
 137     {"Via", HDR_VIA, ftStr},    /* for now */
 138     {"Warning", HDR_WARNING, ftStr},    /* for now */
 139     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 140     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 141     {"X-Cache", HDR_X_CACHE, ftStr},
 142     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 143     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 144     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 145     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 146     {"Negotiate", HDR_NEGOTIATE, ftStr},
 147 #if X_ACCELERATOR_VARY
 148     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 149 #endif
 150 #if USE_ADAPTATION
 151     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 152 #endif
 153     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 154     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 155     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 156     {"Other:", HDR_OTHER, ftStr}        /* ':' will not allow matches */
 157 };
 158
 159 static HttpHeaderFieldInfo *Headers = NULL;
 160
 161 http_hdr_type &operator++ (http_hdr_type &aHeader)
 162 {
 163     int tmp = (int)aHeader;
 164     aHeader = (http_hdr_type)(++tmp);
 165     return aHeader;
 166 }
 167
 168
 169 /*
 170  * headers with field values defined as #(values) in HTTP/1.1
 171  * Headers that are currently not recognized, are commented out.
 172  */
 173 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 174 static http_hdr_type ListHeadersArr[] = {
 175     HDR_ACCEPT,
 176     HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 177     HDR_ACCEPT_RANGES, HDR_ALLOW,
 178     HDR_CACHE_CONTROL,
 179     HDR_CONTENT_ENCODING,
 180     HDR_CONTENT_LANGUAGE,
 181     HDR_CONNECTION,
 182     HDR_EXPECT,
 183     HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 184     HDR_LINK, HDR_PRAGMA,
 185     HDR_PROXY_CONNECTION,
 186     HDR_PROXY_SUPPORT,
 187     HDR_TRANSFER_ENCODING,
 188     HDR_UPGRADE,
 189     HDR_VARY,
 190     HDR_VIA,
 191     HDR_WARNING,
 192     HDR_WWW_AUTHENTICATE,
 193     HDR_AUTHENTICATION_INFO,
 194     HDR_PROXY_AUTHENTICATION_INFO,
 195     /* HDR_TE, HDR_TRAILER */
 196 #if X_ACCELERATOR_VARY
 197     HDR_X_ACCELERATOR_VARY,
 198 #endif
 199 #if USE_ADAPTATION
 200     HDR_X_NEXT_SERVICES,
 201 #endif
 202     HDR_SURROGATE_CAPABILITY,
 203     HDR_SURROGATE_CONTROL,
 204     HDR_X_FORWARDED_FOR
 205 };
 206
 207 /* general-headers */
 208 static http_hdr_type GeneralHeadersArr[] = {
 209     HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 210     HDR_TRANSFER_ENCODING,
 211     HDR_UPGRADE,
 212     /* HDR_TRAILER, */
 213     HDR_VIA,
 214 };
 215
 216 /* entity-headers */
 217 static http_hdr_type EntityHeadersArr[] = {
 218     HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 219     HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 220     HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 221     HDR_OTHER
 222 };
 223
 224 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 225 static http_hdr_type ReplyHeadersArr[] = {
 226     HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 227     HDR_ACCEPT_RANGES, HDR_AGE,
 228     HDR_LOCATION, HDR_MAX_FORWARDS,
 229     HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_SET_COOKIE2,
 230     HDR_VARY,
 231     HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 232     HDR_X_CACHE_LOOKUP,
 233     HDR_X_REQUEST_URI,
 234 #if X_ACCELERATOR_VARY
 235     HDR_X_ACCELERATOR_VARY,
 236 #endif
 237 #if USE_ADAPTATION
 238     HDR_X_NEXT_SERVICES,
 239 #endif
 240     HDR_X_SQUID_ERROR,
 241     HDR_SURROGATE_CONTROL
 242 };
 243
 244 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 245 static http_hdr_type RequestHeadersArr[] = {
 246     HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 247     HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 248     HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION,
 249     HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 250     HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 251 };
 252
 253 static HttpHeaderMask HopByHopHeadersMask;
 254 static http_hdr_type HopByHopHeadersArr[] = {
 255     HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 256     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 257 };
 258
 259 /* header accounting */
 260 static HttpHeaderStat HttpHeaderStats[] = {
 261     {"all"},
 262 #if USE_HTCP
 263     {"HTCP reply"},
 264 #endif
 265     {"request"},
 266     {"reply"}
 267 };
 268 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 269
 270 static int HeaderEntryParsedCount = 0;
 271
 272 /*
 273  * local routines
 274  */
 275
 276 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 277
 278 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 279
 280 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 281 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 282
 283 /*
 284  * Module initialization routines
 285  */
 286
 287 static void
 288 httpHeaderRegisterWithCacheManager(void)
 289 {
 290     Mgr::RegisterAction("http_headers",
 291                         "HTTP Header Statistics",
 292                         httpHeaderStoreReport, 0, 1);
 293 }
 294
 295 void
 296 httpHeaderInitModule(void)
 297 {
 298     int i;
 299     /* check that we have enough space for masks */
 300     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 301     /* all headers must be described */
 302     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 303
 304     if (!Headers)
 305         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 306
 307     /* create masks */
 308     httpHeaderMaskInit(&ListHeadersMask, 0);
 309
 310     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 311
 312     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 313
 314     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 315
 316     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 317
 318     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 319
 320     httpHeaderMaskInit(&RequestHeadersMask, 0);
 321
 322     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 323
 324     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 325
 326     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 327
 328     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 329
 330     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 331
 332     /* init header stats */
 333     assert(HttpHeaderStatCount == hoReply + 1);
 334
 335     for (i = 0; i < HttpHeaderStatCount; i++)
 336         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 337
 338     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 339
 340     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 341
 342 #if USE_HTCP
 343
 344     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 345
 346 #endif
 347     /* init dependent modules */
 348     httpHdrCcInitModule();
 349
 350     httpHdrScInitModule();
 351
 352     httpHeaderRegisterWithCacheManager();
 353 }
 354
 355 void
 356 httpHeaderCleanModule(void)
 357 {
 358     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 359     Headers = NULL;
 360     httpHdrCcCleanModule();
 361     httpHdrScCleanModule();
 362 }
 363
 364 static void
 365 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 366 {
 367     assert(hs);
 368     assert(label);
 369     memset(hs, 0, sizeof(HttpHeaderStat));
 370     hs->label = label;
 371     statHistEnumInit(&hs->hdrUCountDistr, 32);  /* not a real enum */
 372     statHistEnumInit(&hs->fieldTypeDistr, HDR_ENUM_END);
 373     statHistEnumInit(&hs->ccTypeDistr, CC_ENUM_END);
 374     statHistEnumInit(&hs->scTypeDistr, SC_ENUM_END);
 375 }
 376
 377 /*
 378  * HttpHeader Implementation
 379  */
 380
 381 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 382 {
 383     httpHeaderMaskInit(&mask, 0);
 384 }
 385
 386 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 387 {
 388     assert(anOwner > hoNone && anOwner <= hoReply);
 389     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 390     httpHeaderMaskInit(&mask, 0);
 391 }
 392
 393 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 394 {
 395     httpHeaderMaskInit(&mask, 0);
 396     update(&other, NULL); // will update the mask as well
 397 }
 398
 399 HttpHeader::~HttpHeader()
 400 {
 401     clean();
 402 }
 403
 404 HttpHeader &
 405 HttpHeader::operator =(const HttpHeader &other)
 406 {
 407     if (this != &other) {
 408         // we do not really care, but the caller probably does
 409         assert(owner == other.owner);
 410         clean();
 411         update(&other, NULL); // will update the mask as well
 412         len = other.len;
 413     }
 414     return *this;
 415 }
 416
 417 void
 418 HttpHeader::clean()
 419 {
 420     HttpHeaderPos pos = HttpHeaderInitPos;
 421     HttpHeaderEntry *e;
 422
 423     assert(owner > hoNone && owner <= hoReply);
 424     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 425
 426     PROF_start(HttpHeaderClean);
 427
 428     /*
 429      * An unfortunate bug.  The entries array is initialized
 430      * such that count is set to zero.  httpHeaderClean() seems to
 431      * be called both when 'hdr' is created, and destroyed.  Thus,
 432      * we accumulate a large number of zero counts for 'hdr' before
 433      * it is ever used.  Can't think of a good way to fix it, except
 434      * adding a state variable that indicates whether or not 'hdr'
 435      * has been used.  As a hack, just never count zero-sized header
 436      * arrays.
 437      */
 438
 439     if (0 != entries.count)
 440         statHistCount(&HttpHeaderStats[owner].hdrUCountDistr, entries.count);
 441
 442     HttpHeaderStats[owner].destroyedCount++;
 443
 444     HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 445
 446     while ((e = getEntry(&pos))) {
 447         /* tmp hack to try to avoid coredumps */
 448
 449         if (e->id < 0 || e->id >= HDR_ENUM_END) {
 450             debugs(55, 0, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 451         } else {
 452             statHistCount(&HttpHeaderStats[owner].fieldTypeDistr, e->id);
 453             /* yes, this deletion leaves us in an inconsistent state */
 454             delete e;
 455         }
 456     }
 457     entries.clean();
 458     httpHeaderMaskInit(&mask, 0);
 459     len = 0;
 460     PROF_stop(HttpHeaderClean);
 461 }
 462
 463 /* append entries (also see httpHeaderUpdate) */
 464 void
 465 HttpHeader::append(const HttpHeader * src)
 466 {
 467     const HttpHeaderEntry *e;
 468     HttpHeaderPos pos = HttpHeaderInitPos;
 469     assert(src);
 470     assert(src != this);
 471     debugs(55, 7, "appending hdr: " << this << " += " << src);
 472
 473     while ((e = src->getEntry(&pos))) {
 474         addEntry(e->clone());
 475     }
 476 }
 477
 478 /* use fresh entries to replace old ones */
 479 void
 480 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 481 {
 482     assert (old);
 483     old->update (fresh, denied_mask);
 484 }
 485
 486 void
 487 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 488 {
 489     const HttpHeaderEntry *e;
 490     HttpHeaderPos pos = HttpHeaderInitPos;
 491     assert(fresh);
 492     assert(this != fresh);
 493
 494     while ((e = fresh->getEntry(&pos))) {
 495         /* deny bad guys (ok to check for HDR_OTHER) here */
 496
 497         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 498             continue;
 499
 500         if (e->id != HDR_OTHER)
 501             delById(e->id);
 502         else
 503             delByName(e->name.termedBuf());
 504     }
 505
 506     pos = HttpHeaderInitPos;
 507     while ((e = fresh->getEntry(&pos))) {
 508         /* deny bad guys (ok to check for HDR_OTHER) here */
 509
 510         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 511             continue;
 512
 513         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 514
 515         addEntry(e->clone());
 516     }
 517 }
 518
 519 /* just handy in parsing: resets and returns false */
 520 int
 521 HttpHeader::reset()
 522 {
 523     clean();
 524     return 0;
 525 }
 526
 527 int
 528 HttpHeader::parse(const char *header_start, const char *header_end)
 529 {
 530     const char *field_ptr = header_start;
 531     HttpHeaderEntry *e, *e2;
 532
 533     PROF_start(HttpHeaderParse);
 534
 535     assert(header_start && header_end);
 536     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 537     HttpHeaderStats[owner].parsedCount++;
 538
 539     char *nulpos;
 540     if ((nulpos = (char*)memchr(header_start, '\0', header_end - header_start))) {
 541         debugs(55, 1, "WARNING: HTTP header contains NULL characters {" <<
 542                getStringPrefix(header_start, nulpos) << "}\nNULL\n{" << getStringPrefix(nulpos+1, header_end));
 543         goto reset;
 544     }
 545
 546     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 547      * continuation lines start with a (single) space or tab */
 548     while (field_ptr < header_end) {
 549         const char *field_start = field_ptr;
 550         const char *field_end;
 551
 552         do {
 553             const char *this_line = field_ptr;
 554             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 555
 556             if (!field_ptr)
 557                 goto reset;     /* missing <LF> */
 558
 559             field_end = field_ptr;
 560
 561             field_ptr++;        /* Move to next line */
 562
 563             if (field_end > this_line && field_end[-1] == '\r') {
 564                 field_end--;    /* Ignore CR LF */
 565
 566                 if (owner == hoRequest && field_end > this_line) {
 567                     bool cr_only = true;
 568                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 569                         if (*p != '\r')
 570                             cr_only = false;
 571                     }
 572                     if (cr_only) {
 573                         debugs(55, 1, "WARNING: Rejecting HTTP request with a CR+ "
 574                                "header field to prevent request smuggling attacks: {" <<
 575                                getStringPrefix(header_start, header_end) << "}");
 576                         goto reset;
 577                     }
 578                 }
 579             }
 580
 581             /* Barf on stray CR characters */
 582             if (memchr(this_line, '\r', field_end - this_line)) {
 583                 debugs(55, 1, "WARNING: suspicious CR characters in HTTP header {" <<
 584                        getStringPrefix(field_start, field_end) << "}");
 585
 586                 if (Config.onoff.relaxed_header_parser) {
 587                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 588
 589                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL)
 590                         *p++ = ' ';
 591                 } else
 592                     goto reset;
 593             }
 594
 595             if (this_line + 1 == field_end && this_line > field_start) {
 596                 debugs(55, 1, "WARNING: Blank continuation line in HTTP header {" <<
 597                        getStringPrefix(header_start, header_end) << "}");
 598                 goto reset;
 599             }
 600         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 601
 602         if (field_start == field_end) {
 603             if (field_ptr < header_end) {
 604                 debugs(55, 1, "WARNING: unparseable HTTP header field near {" <<
 605                        getStringPrefix(field_start, header_end) << "}");
 606                 goto reset;
 607             }
 608
 609             break;              /* terminating blank line */
 610         }
 611
 612         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 613             debugs(55, 1, "WARNING: unparseable HTTP header field {" <<
 614                    getStringPrefix(field_start, field_end) << "}");
 615             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 616                    " in {" << getStringPrefix(header_start, header_end) << "}");
 617
 618             if (Config.onoff.relaxed_header_parser)
 619                 continue;
 620             else
 621                 goto reset;
 622         }
 623
 624         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 625 //            if (e->value.cmp(e2->value.termedBuf()) != 0) {
 626             if (e->value != e2->value) {
 627                 int64_t l1, l2;
 628                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 629                        "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 630
 631                 if (!Config.onoff.relaxed_header_parser) {
 632                     delete e;
 633                     goto reset;
 634                 }
 635
 636                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 637                     debugs(55, 1, "WARNING: Unparseable content-length '" << e->value << "'");
 638                     delete e;
 639                     continue;
 640                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 641                     debugs(55, 1, "WARNING: Unparseable content-length '" << e2->value << "'");
 642                     delById(e2->id);
 643                 } else if (l1 > l2) {
 644                     delById(e2->id);
 645                 } else {
 646                     delete e;
 647                     continue;
 648                 }
 649             } else {
 650                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 651                        "NOTICE: found double content-length header");
 652
 653                 if (Config.onoff.relaxed_header_parser) {
 654                     delete e;
 655                     continue;
 656                 } else {
 657                     delete e;
 658                     goto reset;
 659                 }
 660             }
 661         }
 662
 663         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 664             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 665                    "WARNING: found whitespace in HTTP header name {" <<
 666                    getStringPrefix(field_start, field_end) << "}");
 667
 668             if (!Config.onoff.relaxed_header_parser) {
 669                 delete e;
 670                 goto reset;
 671             }
 672         }
 673
 674         addEntry(e);
 675     }
 676
 677     if (chunked()) {
 678         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 679         delById(HDR_CONTENT_LENGTH);
 680     }
 681
 682     PROF_stop(HttpHeaderParse);
 683     return 1;                   /* even if no fields where found, it is a valid header */
 684 reset:
 685     PROF_stop(HttpHeaderParse);
 686     return reset();
 687 }
 688
 689 /* packs all the entries using supplied packer */
 690 void
 691 HttpHeader::packInto(Packer * p) const
 692 {
 693     HttpHeaderPos pos = HttpHeaderInitPos;
 694     const HttpHeaderEntry *e;
 695     assert(p);
 696     debugs(55, 7, "packing hdr: (" << this << ")");
 697     /* pack all entries one by one */
 698     while ((e = getEntry(&pos)))
 699         e->packInto(p);
 700
 701     /* Pack in the "special" entries */
 702
 703     /* Cache-Control */
 704 }
 705
 706 /* returns next valid entry */
 707 HttpHeaderEntry *
 708 HttpHeader::getEntry(HttpHeaderPos * pos) const
 709 {
 710     assert(pos);
 711     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 712
 713     for ((*pos)++; *pos < (ssize_t)entries.count; (*pos)++) {
 714         if (entries.items[*pos])
 715             return (HttpHeaderEntry*)entries.items[*pos];
 716     }
 717
 718     return NULL;
 719 }
 720
 721 /*
 722  * returns a pointer to a specified entry if any
 723  * note that we return one entry so it does not make much sense to ask for
 724  * "list" headers
 725  */
 726 HttpHeaderEntry *
 727 HttpHeader::findEntry(http_hdr_type id) const
 728 {
 729     HttpHeaderPos pos = HttpHeaderInitPos;
 730     HttpHeaderEntry *e;
 731     assert_eid(id);
 732     assert(!CBIT_TEST(ListHeadersMask, id));
 733
 734     /* check mask first */
 735
 736     if (!CBIT_TEST(mask, id))
 737         return NULL;
 738
 739     /* looks like we must have it, do linear search */
 740     while ((e = getEntry(&pos))) {
 741         if (e->id == id)
 742             return e;
 743     }
 744
 745     /* hm.. we thought it was there, but it was not found */
 746     assert(0);
 747
 748     return NULL;                /* not reached */
 749 }
 750
 751 /*
 752  * same as httpHeaderFindEntry
 753  */
 754 HttpHeaderEntry *
 755 HttpHeader::findLastEntry(http_hdr_type id) const
 756 {
 757     HttpHeaderPos pos = HttpHeaderInitPos;
 758     HttpHeaderEntry *e;
 759     HttpHeaderEntry *result = NULL;
 760     assert_eid(id);
 761     assert(!CBIT_TEST(ListHeadersMask, id));
 762
 763     /* check mask first */
 764
 765     if (!CBIT_TEST(mask, id))
 766         return NULL;
 767
 768     /* looks like we must have it, do linear search */
 769     while ((e = getEntry(&pos))) {
 770         if (e->id == id)
 771             result = e;
 772     }
 773
 774     assert(result);             /* must be there! */
 775     return result;
 776 }
 777
 778 /*
 779  * deletes all fields with a given name if any, returns #fields deleted;
 780  */
 781 int
 782 HttpHeader::delByName(const char *name)
 783 {
 784     int count = 0;
 785     HttpHeaderPos pos = HttpHeaderInitPos;
 786     HttpHeaderEntry *e;
 787     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 788     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 789
 790     while ((e = getEntry(&pos))) {
 791         if (!e->name.caseCmp(name))
 792             delAt(pos, count);
 793         else
 794             CBIT_SET(mask, e->id);
 795     }
 796
 797     return count;
 798 }
 799
 800 /* deletes all entries with a given id, returns the #entries deleted */
 801 int
 802 HttpHeader::delById(http_hdr_type id)
 803 {
 804     int count = 0;
 805     HttpHeaderPos pos = HttpHeaderInitPos;
 806     HttpHeaderEntry *e;
 807     debugs(55, 8, this << " del-by-id " << id);
 808     assert_eid(id);
 809     assert(id != HDR_OTHER);            /* does not make sense */
 810
 811     if (!CBIT_TEST(mask, id))
 812         return 0;
 813
 814     while ((e = getEntry(&pos))) {
 815         if (e->id == id)
 816             delAt(pos, count);
 817     }
 818
 819     CBIT_CLR(mask, id);
 820     assert(count);
 821     return count;
 822 }
 823
 824 /*
 825  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 826  * possible to iterate(search) and delete fields at the same time
 827  * NOTE: Does not update the header mask. Caller must follow up with
 828  * a call to refreshMask() if headers_deleted was incremented.
 829  */
 830 void
 831 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 832 {
 833     HttpHeaderEntry *e;
 834     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 835     e = (HttpHeaderEntry*)entries.items[pos];
 836     entries.items[pos] = NULL;
 837     /* decrement header length, allow for ": " and crlf */
 838     len -= e->name.size() + 2 + e->value.size() + 2;
 839     assert(len >= 0);
 840     delete e;
 841     ++headers_deleted;
 842 }
 843
 844 /*
 845  * Compacts the header storage
 846  */
 847 void
 848 HttpHeader::compact()
 849 {
 850     entries.prune(NULL);
 851 }
 852
 853 /*
 854  * Refreshes the header mask. Required after delAt() calls.
 855  */
 856 void
 857 HttpHeader::refreshMask()
 858 {
 859     httpHeaderMaskInit(&mask, 0);
 860     debugs(55, 7, "refreshing the mask in hdr " << this);
 861     HttpHeaderPos pos = HttpHeaderInitPos;
 862     while (HttpHeaderEntry *e = getEntry(&pos)) {
 863         CBIT_SET(mask, e->id);
 864     }
 865 }
 866
 867 /* appends an entry;
 868  * does not call e->clone() so one should not reuse "*e"
 869  */
 870 void
 871 HttpHeader::addEntry(HttpHeaderEntry * e)
 872 {
 873     assert(e);
 874     assert_eid(e->id);
 875     assert(e->name.size());
 876
 877     debugs(55, 9, this << " adding entry: " << e->id << " at " <<
 878            entries.count);
 879
 880     if (CBIT_TEST(mask, e->id))
 881         Headers[e->id].stat.repCount++;
 882     else
 883         CBIT_SET(mask, e->id);
 884
 885     entries.push_back(e);
 886
 887     /* increment header length, allow for ": " and crlf */
 888     len += e->name.size() + 2 + e->value.size() + 2;
 889 }
 890
 891 /* inserts an entry;
 892  * does not call e->clone() so one should not reuse "*e"
 893  */
 894 void
 895 HttpHeader::insertEntry(HttpHeaderEntry * e)
 896 {
 897     assert(e);
 898     assert_eid(e->id);
 899
 900     debugs(55, 7, this << " adding entry: " << e->id << " at " <<
 901            entries.count);
 902
 903     if (CBIT_TEST(mask, e->id))
 904         Headers[e->id].stat.repCount++;
 905     else
 906         CBIT_SET(mask, e->id);
 907
 908     entries.insert(e);
 909
 910     /* increment header length, allow for ": " and crlf */
 911     len += e->name.size() + 2 + e->value.size() + 2;
 912 }
 913
 914 bool
 915 HttpHeader::getList(http_hdr_type id, String *s) const
 916 {
 917     HttpHeaderEntry *e;
 918     HttpHeaderPos pos = HttpHeaderInitPos;
 919     debugs(55, 9, this << " joining for id " << id);
 920     /* only fields from ListHeaders array can be "listed" */
 921     assert(CBIT_TEST(ListHeadersMask, id));
 922
 923     if (!CBIT_TEST(mask, id))
 924         return false;
 925
 926     while ((e = getEntry(&pos))) {
 927         if (e->id == id)
 928             strListAdd(s, e->value.termedBuf(), ',');
 929     }
 930
 931     /*
 932      * note: we might get an empty (size==0) string if there was an "empty"
 933      * header. This results in an empty length String, which may have a NULL
 934      * buffer.
 935      */
 936     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 937     if (!s->size())
 938         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 939     else
 940         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 941
 942     return true;
 943 }
 944
 945 /* return a list of entries with the same id separated by ',' and ws */
 946 String
 947 HttpHeader::getList(http_hdr_type id) const
 948 {
 949     HttpHeaderEntry *e;
 950     HttpHeaderPos pos = HttpHeaderInitPos;
 951     debugs(55, 9, this << "joining for id " << id);
 952     /* only fields from ListHeaders array can be "listed" */
 953     assert(CBIT_TEST(ListHeadersMask, id));
 954
 955     if (!CBIT_TEST(mask, id))
 956         return String();
 957
 958     String s;
 959
 960     while ((e = getEntry(&pos))) {
 961         if (e->id == id)
 962             strListAdd(&s, e->value.termedBuf(), ',');
 963     }
 964
 965     /*
 966      * note: we might get an empty (size==0) string if there was an "empty"
 967      * header. This results in an empty length String, which may have a NULL
 968      * buffer.
 969      */
 970     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 971     if (!s.size())
 972         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 973     else
 974         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 975
 976     return s;
 977 }
 978
 979 /* return a string or list of entries with the same id separated by ',' and ws */
 980 String
 981 HttpHeader::getStrOrList(http_hdr_type id) const
 982 {
 983     HttpHeaderEntry *e;
 984
 985     if (CBIT_TEST(ListHeadersMask, id))
 986         return getList(id);
 987
 988     if ((e = findEntry(id)))
 989         return e->value;
 990
 991     return String();
 992 }
 993
 994 /*
 995  * Returns the value of the specified header.
 996  */
 997 String
 998 HttpHeader::getByName(const char *name) const
 999 {
1000     http_hdr_type id;
1001     HttpHeaderPos pos = HttpHeaderInitPos;
1002     HttpHeaderEntry *e;
1003
1004     assert(name);
1005
1006     /* First try the quick path */
1007     id = httpHeaderIdByNameDef(name, strlen(name));
1008
1009     if (id != -1)
1010         return getStrOrList(id);
1011
1012     String result;
1013
1014     /* Sorry, an unknown header name. Do linear search */
1015     while ((e = getEntry(&pos))) {
1016         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1017             strListAdd(&result, e->value.termedBuf(), ',');
1018         }
1019     }
1020
1021     return result;
1022 }
1023
1024 /*
1025  * Returns a the value of the specified list member, if any.
1026  */
1027 String
1028 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1029 {
1030     String header;
1031     const char *pos = NULL;
1032     const char *item;
1033     int ilen;
1034     int mlen = strlen(member);
1035
1036     assert(name);
1037
1038     header = getByName(name);
1039
1040     String result;
1041
1042     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1043         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1044             result.append(item + mlen + 1, ilen - mlen - 1);
1045             break;
1046         }
1047     }
1048
1049     return result;
1050 }
1051
1052 /*
1053  * returns a the value of the specified list member, if any.
1054  */
1055 String
1056 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1057 {
1058     String header;
1059     const char *pos = NULL;
1060     const char *item;
1061     int ilen;
1062     int mlen = strlen(member);
1063
1064     assert(id >= 0);
1065
1066     header = getStrOrList(id);
1067     String result;
1068
1069     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1070         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1071             result.append(item + mlen + 1, ilen - mlen - 1);
1072             break;
1073         }
1074     }
1075
1076     header.clean();
1077     return result;
1078 }
1079
1080 /* test if a field is present */
1081 int
1082 HttpHeader::has(http_hdr_type id) const
1083 {
1084     assert_eid(id);
1085     assert(id != HDR_OTHER);
1086     debugs(55, 9, this << " lookup for " << id);
1087     return CBIT_TEST(mask, id);
1088 }
1089
1090 void
1091 HttpHeader::putInt(http_hdr_type id, int number)
1092 {
1093     assert_eid(id);
1094     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1095     assert(number >= 0);
1096     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1097 }
1098
1099 void
1100 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1101 {
1102     assert_eid(id);
1103     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1104     assert(number >= 0);
1105     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1106 }
1107
1108 void
1109 HttpHeader::putTime(http_hdr_type id, time_t htime)
1110 {
1111     assert_eid(id);
1112     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1113     assert(htime >= 0);
1114     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1115 }
1116
1117 void
1118 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1119 {
1120     assert_eid(id);
1121     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1122     assert(htime >= 0);
1123     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1124 }
1125
1126 void
1127 HttpHeader::putStr(http_hdr_type id, const char *str)
1128 {
1129     assert_eid(id);
1130     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1131     assert(str);
1132     addEntry(new HttpHeaderEntry(id, NULL, str));
1133 }
1134
1135 void
1136 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1137 {
1138     assert(auth_scheme && realm);
1139     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1140 }
1141
1142 void
1143 HttpHeader::putCc(const HttpHdrCc * cc)
1144 {
1145     MemBuf mb;
1146     Packer p;
1147     assert(cc);
1148     /* remove old directives if any */
1149     delById(HDR_CACHE_CONTROL);
1150     /* pack into mb */
1151     mb.init();
1152     packerToMemInit(&p, &mb);
1153     httpHdrCcPackInto(cc, &p);
1154     /* put */
1155     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1156     /* cleanup */
1157     packerClean(&p);
1158     mb.clean();
1159 }
1160
1161 void
1162 HttpHeader::putContRange(const HttpHdrContRange * cr)
1163 {
1164     MemBuf mb;
1165     Packer p;
1166     assert(cr);
1167     /* remove old directives if any */
1168     delById(HDR_CONTENT_RANGE);
1169     /* pack into mb */
1170     mb.init();
1171     packerToMemInit(&p, &mb);
1172     httpHdrContRangePackInto(cr, &p);
1173     /* put */
1174     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1175     /* cleanup */
1176     packerClean(&p);
1177     mb.clean();
1178 }
1179
1180 void
1181 HttpHeader::putRange(const HttpHdrRange * range)
1182 {
1183     MemBuf mb;
1184     Packer p;
1185     assert(range);
1186     /* remove old directives if any */
1187     delById(HDR_RANGE);
1188     /* pack into mb */
1189     mb.init();
1190     packerToMemInit(&p, &mb);
1191     range->packInto(&p);
1192     /* put */
1193     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1194     /* cleanup */
1195     packerClean(&p);
1196     mb.clean();
1197 }
1198
1199 void
1200 HttpHeader::putSc(HttpHdrSc *sc)
1201 {
1202     MemBuf mb;
1203     Packer p;
1204     assert(sc);
1205     /* remove old directives if any */
1206     delById(HDR_SURROGATE_CONTROL);
1207     /* pack into mb */
1208     mb.init();
1209     packerToMemInit(&p, &mb);
1210     httpHdrScPackInto(sc, &p);
1211     /* put */
1212     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1213     /* cleanup */
1214     packerClean(&p);
1215     mb.clean();
1216 }
1217
1218 void
1219 HttpHeader::putWarning(const int code, const char *const text)
1220 {
1221     char buf[512];
1222     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1223     putStr(HDR_WARNING, buf);
1224 }
1225
1226 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1227 void
1228 HttpHeader::putExt(const char *name, const char *value)
1229 {
1230     assert(name && value);
1231     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1232     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1233 }
1234
1235 int
1236 HttpHeader::getInt(http_hdr_type id) const
1237 {
1238     assert_eid(id);
1239     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1240     HttpHeaderEntry *e;
1241
1242     if ((e = findEntry(id)))
1243         return e->getInt();
1244
1245     return -1;
1246 }
1247
1248 int64_t
1249 HttpHeader::getInt64(http_hdr_type id) const
1250 {
1251     assert_eid(id);
1252     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1253     HttpHeaderEntry *e;
1254
1255     if ((e = findEntry(id)))
1256         return e->getInt64();
1257
1258     return -1;
1259 }
1260
1261 time_t
1262 HttpHeader::getTime(http_hdr_type id) const
1263 {
1264     HttpHeaderEntry *e;
1265     time_t value = -1;
1266     assert_eid(id);
1267     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1268
1269     if ((e = findEntry(id))) {
1270         value = parse_rfc1123(e->value.termedBuf());
1271         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1272     }
1273
1274     return value;
1275 }
1276
1277 /* sync with httpHeaderGetLastStr */
1278 const char *
1279 HttpHeader::getStr(http_hdr_type id) const
1280 {
1281     HttpHeaderEntry *e;
1282     assert_eid(id);
1283     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1284
1285     if ((e = findEntry(id))) {
1286         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1287         return e->value.termedBuf();
1288     }
1289
1290     return NULL;
1291 }
1292
1293 /* unusual */
1294 const char *
1295 HttpHeader::getLastStr(http_hdr_type id) const
1296 {
1297     HttpHeaderEntry *e;
1298     assert_eid(id);
1299     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1300
1301     if ((e = findLastEntry(id))) {
1302         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1303         return e->value.termedBuf();
1304     }
1305
1306     return NULL;
1307 }
1308
1309 HttpHdrCc *
1310 HttpHeader::getCc() const
1311 {
1312     HttpHdrCc *cc;
1313     String s;
1314
1315     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1316         return NULL;
1317     PROF_start(HttpHeader_getCc);
1318
1319     getList(HDR_CACHE_CONTROL, &s);
1320
1321     cc = httpHdrCcParseCreate(&s);
1322
1323     HttpHeaderStats[owner].ccParsedCount++;
1324
1325     if (cc)
1326         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1327
1328     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1329
1330     PROF_stop(HttpHeader_getCc);
1331
1332     return cc;
1333 }
1334
1335 HttpHdrRange *
1336 HttpHeader::getRange() const
1337 {
1338     HttpHdrRange *r = NULL;
1339     HttpHeaderEntry *e;
1340     /* some clients will send "Request-Range" _and_ *matching* "Range"
1341      * who knows, some clients might send Request-Range only;
1342      * this "if" should work correctly in both cases;
1343      * hopefully no clients send mismatched headers! */
1344
1345     if ((e = findEntry(HDR_RANGE)) ||
1346             (e = findEntry(HDR_REQUEST_RANGE))) {
1347         r = HttpHdrRange::ParseCreate(&e->value);
1348         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1349     }
1350
1351     return r;
1352 }
1353
1354 HttpHdrSc *
1355 HttpHeader::getSc() const
1356 {
1357     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1358         return NULL;
1359
1360     String s;
1361
1362     (void) getList(HDR_SURROGATE_CONTROL, &s);
1363
1364     HttpHdrSc *sc = httpHdrScParseCreate(&s);
1365
1366     HttpHeaderStats[owner].ccParsedCount++;
1367
1368     if (sc)
1369         httpHdrScUpdateStats(sc, &HttpHeaderStats[owner].scTypeDistr);
1370
1371     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1372
1373     return sc;
1374 }
1375
1376 HttpHdrContRange *
1377 HttpHeader::getContRange() const
1378 {
1379     HttpHdrContRange *cr = NULL;
1380     HttpHeaderEntry *e;
1381
1382     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1383         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1384         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1385     }
1386
1387     return cr;
1388 }
1389
1390 const char *
1391 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1392 {
1393     const char *field;
1394     int l;
1395     assert(auth_scheme);
1396     field = getStr(id);
1397
1398     if (!field)                 /* no authorization field */
1399         return NULL;
1400
1401     l = strlen(auth_scheme);
1402
1403     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1404         return NULL;
1405
1406     field += l;
1407
1408     if (!xisspace(*field))      /* wrong scheme */
1409         return NULL;
1410
1411     /* skip white space */
1412     field += xcountws(field);
1413
1414     if (!*field)                /* no authorization cookie */
1415         return NULL;
1416
1417     return base64_decode(field);
1418 }
1419
1420 ETag
1421 HttpHeader::getETag(http_hdr_type id) const
1422 {
1423     ETag etag = {NULL, -1};
1424     HttpHeaderEntry *e;
1425     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1426
1427     if ((e = findEntry(id)))
1428         etagParseInit(&etag, e->value.termedBuf());
1429
1430     return etag;
1431 }
1432
1433 TimeOrTag
1434 HttpHeader::getTimeOrTag(http_hdr_type id) const
1435 {
1436     TimeOrTag tot;
1437     HttpHeaderEntry *e;
1438     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1439     memset(&tot, 0, sizeof(tot));
1440
1441     if ((e = findEntry(id))) {
1442         const char *str = e->value.termedBuf();
1443         /* try as an ETag */
1444
1445         if (etagParseInit(&tot.tag, str)) {
1446             tot.valid = tot.tag.str != NULL;
1447             tot.time = -1;
1448         } else {
1449             /* or maybe it is time? */
1450             tot.time = parse_rfc1123(str);
1451             tot.valid = tot.time >= 0;
1452             tot.tag.str = NULL;
1453         }
1454     }
1455
1456     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1457     return tot;
1458 }
1459
1460 /*
1461  * HttpHeaderEntry
1462  */
1463
1464 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1465 {
1466     assert_eid(anId);
1467     id = anId;
1468
1469     if (id != HDR_OTHER)
1470         name = Headers[id].name;
1471     else
1472         name = aName;
1473
1474     value = aValue;
1475
1476     Headers[id].stat.aliveCount++;
1477
1478     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1479 }
1480
1481 HttpHeaderEntry::~HttpHeaderEntry()
1482 {
1483     assert_eid(id);
1484     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1485     /* clean name if needed */
1486
1487     if (id == HDR_OTHER)
1488         name.clean();
1489
1490     value.clean();
1491
1492     assert(Headers[id].stat.aliveCount);
1493
1494     Headers[id].stat.aliveCount--;
1495
1496     id = HDR_BAD_HDR;
1497 }
1498
1499 /* parses and inits header entry, returns true/false */
1500 HttpHeaderEntry *
1501 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1502 {
1503     /* note: name_start == field_start */
1504     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1505     int name_len = name_end ? name_end - field_start :0;
1506     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1507     /* note: value_end == field_end */
1508
1509     HeaderEntryParsedCount++;
1510
1511     /* do we have a valid field name within this field? */
1512
1513     if (!name_len || name_end > field_end)
1514         return NULL;
1515
1516     if (name_len > 65534) {
1517         /* String must be LESS THAN 64K and it adds a terminating NULL */
1518         debugs(55, 1, "WARNING: ignoring header name of " << name_len << " bytes");
1519         return NULL;
1520     }
1521
1522     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1523         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1524                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1525
1526         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1527             name_len--;
1528
1529         if (!name_len)
1530             return NULL;
1531     }
1532
1533     /* now we know we can parse it */
1534
1535     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1536
1537     /* is it a "known" field? */
1538     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1539
1540     String name;
1541
1542     String value;
1543
1544     if (id < 0)
1545         id = HDR_OTHER;
1546
1547     assert_eid(id);
1548
1549     /* set field name */
1550     if (id == HDR_OTHER)
1551         name.limitInit(field_start, name_len);
1552     else
1553         name = Headers[id].name;
1554
1555     /* trim field value */
1556     while (value_start < field_end && xisspace(*value_start))
1557         value_start++;
1558
1559     while (value_start < field_end && xisspace(field_end[-1]))
1560         field_end--;
1561
1562     if (field_end - value_start > 65534) {
1563         /* String must be LESS THAN 64K and it adds a terminating NULL */
1564         debugs(55, 1, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1565
1566         if (id == HDR_OTHER)
1567             name.clean();
1568
1569         return NULL;
1570     }
1571
1572     /* set field value */
1573     value.limitInit(value_start, field_end - value_start);
1574
1575     Headers[id].stat.seenCount++;
1576
1577     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1578
1579     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1580 }
1581
1582 HttpHeaderEntry *
1583 HttpHeaderEntry::clone() const
1584 {
1585     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1586 }
1587
1588 void
1589 HttpHeaderEntry::packInto(Packer * p) const
1590 {
1591     assert(p);
1592     packerAppend(p, name.rawBuf(), name.size());
1593     packerAppend(p, ": ", 2);
1594     packerAppend(p, value.rawBuf(), value.size());
1595     packerAppend(p, "\r\n", 2);
1596 }
1597
1598 int
1599 HttpHeaderEntry::getInt() const
1600 {
1601     assert_eid (id);
1602     assert (Headers[id].type == ftInt);
1603     int val = -1;
1604     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1605     httpHeaderNoteParsedEntry(id, value, !ok);
1606     /* XXX: Should we check ok - ie
1607      * return ok ? -1 : value;
1608      */
1609     return val;
1610 }
1611
1612 int64_t
1613 HttpHeaderEntry::getInt64() const
1614 {
1615     assert_eid (id);
1616     assert (Headers[id].type == ftInt64);
1617     int64_t val = -1;
1618     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1619     httpHeaderNoteParsedEntry(id, value, !ok);
1620     /* XXX: Should we check ok - ie
1621      * return ok ? -1 : value;
1622      */
1623     return val;
1624 }
1625
1626 static void
1627 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1628 {
1629     Headers[id].stat.parsCount++;
1630
1631     if (error) {
1632         Headers[id].stat.errCount++;
1633         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name << ": " << context << "'");
1634     }
1635 }
1636
1637 /*
1638  * Reports
1639  */
1640
1641 /* tmp variable used to pass stat info to dumpers */
1642 extern const HttpHeaderStat *dump_stat;         /* argh! */
1643 const HttpHeaderStat *dump_stat = NULL;
1644
1645 void
1646 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1647 {
1648     const int id = (int) val;
1649     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1650     const char *name = valid_id ? Headers[id].name.termedBuf() : "INVALID";
1651     int visible = count > 0;
1652     /* for entries with zero count, list only those that belong to current type of message */
1653
1654     if (!visible && valid_id && dump_stat->owner_mask)
1655         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1656
1657     if (visible)
1658         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1659                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1660 }
1661
1662 static void
1663 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1664 {
1665     if (count)
1666         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1667                           idx, (int) val, count,
1668                           xpercent(count, dump_stat->destroyedCount));
1669 }
1670
1671
1672 static void
1673 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1674 {
1675     assert(hs && e);
1676
1677     dump_stat = hs;
1678     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1679     storeAppendPrintf(e, "\nField type distribution\n");
1680     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1681                       "id", "name", "count", "#/header");
1682     statHistDump(&hs->fieldTypeDistr, e, httpHeaderFieldStatDumper);
1683     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1684     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1685                       "id", "name", "count", "#/cc_field");
1686     statHistDump(&hs->ccTypeDistr, e, httpHdrCcStatDumper);
1687     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1688     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1689                       "id", "name", "count", "#/sc_field");
1690     statHistDump(&hs->scTypeDistr, e, httpHdrScStatDumper);
1691     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1692     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1693                       "id", "#flds", "count", "%total");
1694     statHistDump(&hs->hdrUCountDistr, e, httpHeaderFldsPerHdrDumper);
1695     dump_stat = NULL;
1696 }
1697
1698 void
1699 httpHeaderStoreReport(StoreEntry * e)
1700 {
1701     int i;
1702     http_hdr_type ht;
1703     assert(e);
1704
1705     HttpHeaderStats[0].parsedCount =
1706         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1707     HttpHeaderStats[0].ccParsedCount =
1708         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1709     HttpHeaderStats[0].destroyedCount =
1710         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1711     HttpHeaderStats[0].busyDestroyedCount =
1712         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1713
1714     for (i = 1; i < HttpHeaderStatCount; i++) {
1715         httpHeaderStatDump(HttpHeaderStats + i, e);
1716         storeAppendPrintf(e, "%s\n", "<br>");
1717     }
1718
1719     /* field stats for all messages */
1720     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1721
1722     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1723                       "id", "name", "#alive", "%err", "%repeat");
1724
1725     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1726         HttpHeaderFieldInfo *f = Headers + ht;
1727         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1728                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1729                           xpercent(f->stat.errCount, f->stat.parsCount),
1730                           xpercent(f->stat.repCount, f->stat.seenCount));
1731     }
1732
1733     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1734                       HttpHeaderStats[hoRequest].parsedCount,
1735                       HttpHeaderStats[hoReply].parsedCount,
1736                       HttpHeaderStats[0].parsedCount);
1737     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1738 }
1739
1740 http_hdr_type
1741 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1742 {
1743     if (name_len > 0) {
1744         for (int i = 0; i < end; ++i) {
1745             if (name_len != info[i].name.size())
1746                 continue;
1747
1748             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1749                 return info[i].id;
1750         }
1751     }
1752
1753     return HDR_BAD_HDR;
1754 }
1755
1756 http_hdr_type
1757 httpHeaderIdByNameDef(const char *name, int name_len)
1758 {
1759     if (!Headers)
1760         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1761
1762     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1763 }
1764
1765 const char *
1766 httpHeaderNameById(int id)
1767 {
1768     if (!Headers)
1769         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1770
1771     assert(id >= 0 && id < HDR_ENUM_END);
1772
1773     return Headers[id].name.termedBuf();
1774 }
1775
1776 int
1777 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1778 {
1779     int result = 0;
1780     const char *pos = NULL;
1781     const char *item;
1782     int ilen;
1783     int mlen = strlen(member);
1784
1785     assert(id >= 0);
1786
1787     String header (getStrOrList(id));
1788
1789     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1790         if (strncasecmp(item, member, mlen) == 0
1791                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1792             result = 1;
1793             break;
1794         }
1795     }
1796
1797     return result;
1798 }
1799
1800 int
1801 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1802 {
1803     int result = 0;
1804     const char *pos = NULL;
1805     const char *item;
1806     int ilen;
1807     int mlen = strlen(member);
1808
1809     assert(name);
1810
1811     String header (getByName(name));
1812
1813     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1814         if (strncasecmp(item, member, mlen) == 0
1815                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1816             result = 1;
1817             break;
1818         }
1819     }
1820
1821     return result;
1822 }
1823
1824 void
1825 HttpHeader::removeHopByHopEntries()
1826 {
1827     removeConnectionHeaderEntries();
1828
1829     const HttpHeaderEntry *e;
1830     HttpHeaderPos pos = HttpHeaderInitPos;
1831     int headers_deleted = 0;
1832     while ((e = getEntry(&pos))) {
1833         int id = e->id;
1834         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1835             delAt(pos, headers_deleted);
1836             CBIT_CLR(mask, id);
1837         }
1838     }
1839 }
1840
1841 void
1842 HttpHeader::removeConnectionHeaderEntries()
1843 {
1844     if (has(HDR_CONNECTION)) {
1845         /* anything that matches Connection list member will be deleted */
1846         String strConnection;
1847
1848         (void) getList(HDR_CONNECTION, &strConnection);
1849         const HttpHeaderEntry *e;
1850         HttpHeaderPos pos = HttpHeaderInitPos;
1851         /*
1852          * think: on-average-best nesting of the two loops (hdrEntry
1853          * and strListItem) @?@
1854          */
1855         /*
1856          * maybe we should delete standard stuff ("keep-alive","close")
1857          * from strConnection first?
1858          */
1859
1860         int headers_deleted = 0;
1861         while ((e = getEntry(&pos))) {
1862             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1863                 delAt(pos, headers_deleted);
1864         }
1865         if (headers_deleted)
1866             refreshMask();
1867     }
1868 }