src/HttpHeader.cc

   1
   2 /*
   3  * $Id: HttpHeader.cc,v 1.139 2008/01/22 19:53:03 rousskov Exp $
   4  *
   5  * DEBUG: section 55    HTTP Header
   6  * AUTHOR: Alex Rousskov
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "CacheManager.h"
  38 #include "Store.h"
  39 #include "HttpHeader.h"
  40 #include "HttpHdrContRange.h"
  41 #include "HttpHdrSc.h"
  42 #include "MemBuf.h"
  43
  44 /*
  45  * On naming conventions:
  46  *
  47  * HTTP/1.1 defines message-header as
  48  *
  49  * message-header = field-name ":" [ field-value ] CRLF
  50  * field-name     = token
  51  * field-value    = *( field-content | LWS )
  52  *
  53  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  54  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  55  *
  56  * HttpHeader is an object that represents all message-headers in a message.
  57  * HttpHeader does not manage start-line.
  58  *
  59  * HttpHeader is implemented as a collection of header "entries".
  60  * An entry is a (field_id, field_name, field_value) triplet.
  61  */
  62
  63
  64 /*
  65  * local constants and vars
  66  */
  67
  68 /*
  69  * A table with major attributes for every known field.
  70  * We calculate name lengths and reorganize this array on start up.
  71  * After reorganization, field id can be used as an index to the table.
  72  */
  73 static const HttpHeaderFieldAttrs HeadersAttrs[] =
  74     {
  75         {"Accept", HDR_ACCEPT, ftStr},
  76
  77         {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  78         {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  79         {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  80         {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  81         {"Age", HDR_AGE, ftInt},
  82         {"Allow", HDR_ALLOW, ftStr},
  83         {"Authorization", HDR_AUTHORIZATION, ftStr},    /* for now */
  84         {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  85         {"Connection", HDR_CONNECTION, ftStr},
  86         {"Content-Base", HDR_CONTENT_BASE, ftStr},
  87         {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  88         {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  89         {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  90         {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  91         {"Content-MD5", HDR_CONTENT_MD5, ftStr},        /* for now */
  92         {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
  93         {"Content-Type", HDR_CONTENT_TYPE, ftStr},
  94         {"Cookie", HDR_COOKIE, ftStr},
  95         {"Date", HDR_DATE, ftDate_1123},
  96         {"ETag", HDR_ETAG, ftETag},
  97         {"Expires", HDR_EXPIRES, ftDate_1123},
  98         {"From", HDR_FROM, ftStr},
  99         {"Host", HDR_HOST, ftStr},
 100         {"If-Match", HDR_IF_MATCH, ftStr},      /* for now */
 101         {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 102         {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},    /* for now */
 103         {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 104         {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 105         {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 106         {"Link", HDR_LINK, ftStr},
 107         {"Location", HDR_LOCATION, ftStr},
 108         {"Max-Forwards", HDR_MAX_FORWARDS, ftInt},
 109         {"Mime-Version", HDR_MIME_VERSION, ftStr},      /* for now */
 110         {"Pragma", HDR_PRAGMA, ftStr},
 111         {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 112         {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 113         {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 114         {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 115         {"Public", HDR_PUBLIC, ftStr},
 116         {"Range", HDR_RANGE, ftPRange},
 117         {"Referer", HDR_REFERER, ftStr},
 118         {"Request-Range", HDR_REQUEST_RANGE, ftPRange}, /* usually matches HDR_RANGE */
 119         {"Retry-After", HDR_RETRY_AFTER, ftStr},        /* for now (ftDate_1123 or ftInt!) */
 120         {"Server", HDR_SERVER, ftStr},
 121         {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 122         {"TE", HDR_TE, ftStr},
 123         {"Title", HDR_TITLE, ftStr},
 124         {"Trailers", HDR_TRAILERS, ftStr},
 125         {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 126         {"Upgrade", HDR_UPGRADE, ftStr},        /* for now */
 127         {"User-Agent", HDR_USER_AGENT, ftStr},
 128         {"Vary", HDR_VARY, ftStr},      /* for now */
 129         {"Via", HDR_VIA, ftStr},        /* for now */
 130         {"Warning", HDR_WARNING, ftStr},        /* for now */
 131         {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 132         {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 133         {"X-Cache", HDR_X_CACHE, ftStr},
 134         {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 135         {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 136         {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 137         {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 138         {"Negotiate", HDR_NEGOTIATE, ftStr},
 139 #if X_ACCELERATOR_VARY
 140         {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 141 #endif
 142         {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 143         {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 144         {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 145         {"Other:", HDR_OTHER, ftStr}    /* ':' will not allow matches */
 146     };
 147
 148 static HttpHeaderFieldInfo *Headers = NULL;
 149
 150 http_hdr_type &operator++ (http_hdr_type &aHeader)
 151 {
 152     int tmp = (int)aHeader;
 153     aHeader = (http_hdr_type)(++tmp);
 154     return aHeader;
 155 }
 156
 157
 158 /*
 159  * headers with field values defined as #(values) in HTTP/1.1
 160  * Headers that are currently not recognized, are commented out.
 161  */
 162 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 163 static http_hdr_type ListHeadersArr[] =
 164     {
 165         HDR_ACCEPT,
 166         HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 167         HDR_ACCEPT_RANGES, HDR_ALLOW,
 168         HDR_CACHE_CONTROL,
 169         HDR_CONTENT_ENCODING,
 170         HDR_CONTENT_LANGUAGE,
 171         HDR_CONNECTION,
 172         HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 173         HDR_LINK, HDR_PRAGMA,
 174         HDR_PROXY_CONNECTION,
 175         HDR_TRANSFER_ENCODING,
 176         HDR_UPGRADE,
 177         HDR_VARY,
 178         HDR_VIA,
 179         /* HDR_WARNING, */
 180         HDR_WWW_AUTHENTICATE,
 181         HDR_AUTHENTICATION_INFO,
 182         HDR_PROXY_AUTHENTICATION_INFO,
 183         /* HDR_EXPECT, HDR_TE, HDR_TRAILER */
 184 #if X_ACCELERATOR_VARY
 185         HDR_X_ACCELERATOR_VARY,
 186 #endif
 187         HDR_SURROGATE_CAPABILITY,
 188         HDR_SURROGATE_CONTROL,
 189         HDR_X_FORWARDED_FOR
 190     };
 191
 192 /* general-headers */
 193 static http_hdr_type GeneralHeadersArr[] =
 194     {
 195         HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 196         HDR_TRANSFER_ENCODING,
 197         HDR_UPGRADE,
 198         /* HDR_TRAILER, */
 199         HDR_VIA,
 200     };
 201
 202 /* entity-headers */
 203 static http_hdr_type EntityHeadersArr[] =
 204     {
 205         HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 206         HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 207         HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 208         HDR_OTHER
 209     };
 210
 211 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 212 static http_hdr_type ReplyHeadersArr[] =
 213     {
 214         HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 215         HDR_ACCEPT_RANGES, HDR_AGE,
 216         HDR_LOCATION, HDR_MAX_FORWARDS,
 217         HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE,
 218         HDR_VARY,
 219         HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 220         HDR_X_CACHE_LOOKUP,
 221         HDR_X_REQUEST_URI,
 222 #if X_ACCELERATOR_VARY
 223         HDR_X_ACCELERATOR_VARY,
 224 #endif
 225         HDR_X_SQUID_ERROR,
 226         HDR_SURROGATE_CONTROL
 227     };
 228
 229 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 230 static http_hdr_type RequestHeadersArr[] =
 231     {
 232         HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 233         HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 234         HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION,
 235         HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 236         HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 237     };
 238
 239 static HttpHeaderMask HopByHopHeadersMask;
 240 static http_hdr_type HopByHopHeadersArr[] =
 241     {
 242         HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 243         HDR_TE, HDR_TRAILERS, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 244     };
 245
 246 /* header accounting */
 247 static HttpHeaderStat HttpHeaderStats[] =
 248     {
 249         {"all"},
 250 #if USE_HTCP
 251         {"HTCP reply"},
 252 #endif
 253         {"request"},
 254         {"reply"}
 255     };
 256 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 257
 258 static int HeaderEntryParsedCount = 0;
 259
 260 /*
 261  * local routines
 262  */
 263
 264 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 265
 266 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 267
 268 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 269 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 270
 271 /*
 272  * Module initialization routines
 273  */
 274
 275 static void
 276 httpHeaderRegisterWithCacheManager(void)
 277 {
 278     CacheManager::GetInstance()->
 279         registerAction("http_headers",
 280                        "HTTP Header Statistics",
 281                        httpHeaderStoreReport, 0, 1);
 282 }
 283
 284 void
 285 httpHeaderInitModule(void)
 286 {
 287     int i;
 288     /* check that we have enough space for masks */
 289     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 290     /* all headers must be described */
 291     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 292
 293     if (!Headers)
 294         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 295
 296     /* create masks */
 297     httpHeaderMaskInit(&ListHeadersMask, 0);
 298
 299     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 300
 301     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 302
 303     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 304
 305     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 306
 307     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 308
 309     httpHeaderMaskInit(&RequestHeadersMask, 0);
 310
 311     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 312
 313     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 314
 315     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 316
 317     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 318
 319     /* init header stats */
 320     assert(HttpHeaderStatCount == hoReply + 1);
 321
 322     for (i = 0; i < HttpHeaderStatCount; i++)
 323         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 324
 325     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 326
 327     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 328
 329 #if USE_HTCP
 330
 331     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 332
 333 #endif
 334     /* init dependent modules */
 335     httpHdrCcInitModule();
 336
 337     httpHdrScInitModule();
 338
 339     httpHeaderRegisterWithCacheManager();
 340 }
 341
 342 void
 343 httpHeaderCleanModule(void)
 344 {
 345     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 346     Headers = NULL;
 347     httpHdrCcCleanModule();
 348     httpHdrScCleanModule();
 349 }
 350
 351 static void
 352 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 353 {
 354     assert(hs);
 355     assert(label);
 356     memset(hs, 0, sizeof(HttpHeaderStat));
 357     hs->label = label;
 358     statHistEnumInit(&hs->hdrUCountDistr, 32);  /* not a real enum */
 359     statHistEnumInit(&hs->fieldTypeDistr, HDR_ENUM_END);
 360     statHistEnumInit(&hs->ccTypeDistr, CC_ENUM_END);
 361     statHistEnumInit(&hs->scTypeDistr, SC_ENUM_END);
 362 }
 363
 364 /*
 365  * HttpHeader Implementation
 366  */
 367
 368 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 369 {
 370     httpHeaderMaskInit(&mask, 0);
 371 }
 372
 373 HttpHeader::HttpHeader(http_hdr_owner_type const &anOwner) : owner (anOwner), len (0)
 374 {
 375     assert(anOwner > hoNone && anOwner <= hoReply);
 376     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 377     httpHeaderMaskInit(&mask, 0);
 378 }
 379
 380 HttpHeader::~HttpHeader()
 381 {
 382     clean();
 383 }
 384
 385 void
 386 HttpHeader::clean()
 387 {
 388     HttpHeaderPos pos = HttpHeaderInitPos;
 389     HttpHeaderEntry *e;
 390
 391     assert(owner > hoNone && owner <= hoReply);
 392     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 393
 394     PROF_start(HttpHeaderClean);
 395
 396     /*
 397      * An unfortunate bug.  The entries array is initialized
 398      * such that count is set to zero.  httpHeaderClean() seems to
 399      * be called both when 'hdr' is created, and destroyed.  Thus,
 400      * we accumulate a large number of zero counts for 'hdr' before
 401      * it is ever used.  Can't think of a good way to fix it, except
 402      * adding a state variable that indicates whether or not 'hdr'
 403      * has been used.  As a hack, just never count zero-sized header
 404      * arrays.
 405      */
 406
 407     if (0 != entries.count)
 408         statHistCount(&HttpHeaderStats[owner].hdrUCountDistr, entries.count);
 409
 410     HttpHeaderStats[owner].destroyedCount++;
 411
 412     HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 413
 414     while ((e = getEntry(&pos))) {
 415         /* tmp hack to try to avoid coredumps */
 416
 417         if (e->id < 0 || e->id >= HDR_ENUM_END) {
 418             debugs(55, 0, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 419         } else {
 420             statHistCount(&HttpHeaderStats[owner].fieldTypeDistr, e->id);
 421             /* yes, this deletion leaves us in an inconsistent state */
 422             delete e;
 423         }
 424     }
 425     entries.clean();
 426     httpHeaderMaskInit(&mask, 0);
 427     PROF_stop(HttpHeaderClean);
 428 }
 429
 430 /* append entries (also see httpHeaderUpdate) */
 431 void
 432 HttpHeader::append(const HttpHeader * src)
 433 {
 434     const HttpHeaderEntry *e;
 435     HttpHeaderPos pos = HttpHeaderInitPos;
 436     assert(src);
 437     assert(src != this);
 438     debugs(55, 7, "appending hdr: " << this << " += " << src);
 439
 440     while ((e = src->getEntry(&pos))) {
 441         addEntry(e->clone());
 442     }
 443 }
 444
 445 /* use fresh entries to replace old ones */
 446 void
 447 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 448 {
 449     assert (old);
 450     old->update (fresh, denied_mask);
 451 }
 452
 453 void
 454 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 455 {
 456     const HttpHeaderEntry *e;
 457     HttpHeaderPos pos = HttpHeaderInitPos;
 458     assert(fresh);
 459     assert(this != fresh);
 460
 461     while ((e = fresh->getEntry(&pos))) {
 462         /* deny bad guys (ok to check for HDR_OTHER) here */
 463
 464         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 465             continue;
 466
 467         if (e->id != HDR_OTHER)
 468             delById(e->id);
 469         else
 470             delByName(e->name.buf());
 471     }
 472
 473     pos = HttpHeaderInitPos;
 474     while ((e = fresh->getEntry(&pos))) {
 475         /* deny bad guys (ok to check for HDR_OTHER) here */
 476
 477         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 478             continue;
 479
 480         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 481
 482         addEntry(e->clone());
 483     }
 484 }
 485
 486 /* just handy in parsing: resets and returns false */
 487 int
 488 HttpHeader::reset()
 489 {
 490     http_hdr_owner_type ho;
 491     ho = owner;
 492     clean();
 493     *this = HttpHeader(ho);
 494     return 0;
 495 }
 496
 497 int
 498 HttpHeader::parse(const char *header_start, const char *header_end)
 499 {
 500     const char *field_ptr = header_start;
 501     HttpHeaderEntry *e, *e2;
 502
 503     PROF_start(HttpHeaderParse);
 504
 505     assert(header_start && header_end);
 506     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 507     HttpHeaderStats[owner].parsedCount++;
 508
 509     if (memchr(header_start, '\0', header_end - header_start)) {
 510         debugs(55, 1, "WARNING: HTTP header contains NULL characters {" <<
 511                       getStringPrefix(header_start, header_end) << "}");
 512         goto reset;
 513     }
 514
 515     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 516      * continuation lines start with a (single) space or tab */
 517     while (field_ptr < header_end) {
 518         const char *field_start = field_ptr;
 519         const char *field_end;
 520
 521         do {
 522             const char *this_line = field_ptr;
 523             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 524
 525             if (!field_ptr)
 526                 goto reset;     /* missing <LF> */
 527
 528             field_end = field_ptr;
 529
 530             field_ptr++;        /* Move to next line */
 531
 532             if (field_end > this_line && field_end[-1] == '\r') {
 533                 field_end--;    /* Ignore CR LF */
 534                 /* Ignore CR CR LF in relaxed mode */
 535
 536                 if (Config.onoff.relaxed_header_parser && field_end > this_line + 1 && field_end[-1] == '\r') {
 537                     debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 538                     "WARNING: Double CR characters in HTTP header {" << getStringPrefix(field_start, field_end) << "}");
 539                     field_end--;
 540                 }
 541             }
 542
 543             /* Barf on stray CR characters */
 544             if (memchr(this_line, '\r', field_end - this_line)) {
 545                 debugs(55, 1, "WARNING: suspicious CR characters in HTTP header {" <<
 546                               getStringPrefix(field_start, field_end) << "}");
 547
 548                 if (Config.onoff.relaxed_header_parser) {
 549                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 550
 551                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL)
 552                         *p++ = ' ';
 553                 } else
 554                     goto reset;
 555             }
 556
 557             if (this_line + 1 == field_end && this_line > field_start) {
 558                 debugs(55, 1, "WARNING: Blank continuation line in HTTP header {" <<
 559                               getStringPrefix(header_start, header_end) << "}");
 560                 goto reset;
 561             }
 562         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 563
 564         if (field_start == field_end) {
 565             if (field_ptr < header_end) {
 566                 debugs(55, 1, "WARNING: unparseable HTTP header field near {" <<
 567                               getStringPrefix(field_start, header_end) << "}");
 568                 goto reset;
 569             }
 570
 571             break;              /* terminating blank line */
 572         }
 573
 574         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 575             debugs(55, 1, "WARNING: unparseable HTTP header field {" <<
 576                           getStringPrefix(field_start, field_end) << "}");
 577             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 578             " in {" << getStringPrefix(header_start, header_end) << "}");
 579
 580             if (Config.onoff.relaxed_header_parser)
 581                 continue;
 582             else
 583                 goto reset;
 584         }
 585
 586         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 587             if (e->value.cmp(e2->value.buf()) != 0) {
 588                 int64_t l1, l2;
 589                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 590                   "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 591
 592                 if (!Config.onoff.relaxed_header_parser) {
 593                     delete e;
 594                     goto reset;
 595                 }
 596
 597                 if (!httpHeaderParseOffset(e->value.buf(), &l1)) {
 598                     debugs(55, 1, "WARNING: Unparseable content-length '" << e->value.buf() << "'");
 599                     delete e;
 600                     continue;
 601                 } else if (!httpHeaderParseOffset(e2->value.buf(), &l2)) {
 602                     debugs(55, 1, "WARNING: Unparseable content-length '" << e2->value.buf() << "'");
 603                     delById(e2->id);
 604                 } else if (l1 > l2) {
 605                     delById(e2->id);
 606                 } else {
 607                     delete e;
 608                     continue;
 609                 }
 610             } else {
 611                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 612                 "NOTICE: found double content-length header");
 613
 614                 if (Config.onoff.relaxed_header_parser) {
 615                     delete e;
 616                     continue;
 617                 } else {
 618                     delete e;
 619                     goto reset;
 620                 }
 621             }
 622         }
 623
 624         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.buf())) {
 625             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 626                    "WARNING: found whitespace in HTTP header name {" <<
 627                    getStringPrefix(field_start, field_end) << "}");
 628
 629             if (!Config.onoff.relaxed_header_parser) {
 630                 delete e;
 631                 goto reset;
 632             }
 633         }
 634
 635         addEntry(e);
 636     }
 637
 638     PROF_stop(HttpHeaderParse);
 639     return 1;                   /* even if no fields where found, it is a valid header */
 640 reset:
 641     PROF_stop(HttpHeaderParse);
 642     return reset();
 643 }
 644
 645 /* packs all the entries using supplied packer */
 646 void
 647 HttpHeader::packInto(Packer * p) const
 648 {
 649     HttpHeaderPos pos = HttpHeaderInitPos;
 650     const HttpHeaderEntry *e;
 651     assert(p);
 652     debugs(55, 7, "packing hdr: (" << this << ")");
 653     /* pack all entries one by one */
 654     while ((e = getEntry(&pos)))
 655         e->packInto(p);
 656
 657     /* Pack in the "special" entries */
 658
 659     /* Cache-Control */
 660 }
 661
 662 /* returns next valid entry */
 663 HttpHeaderEntry *
 664 HttpHeader::getEntry(HttpHeaderPos * pos) const
 665 {
 666     assert(pos);
 667     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 668
 669     for ((*pos)++; *pos < (ssize_t)entries.count; (*pos)++) {
 670         if (entries.items[*pos])
 671             return (HttpHeaderEntry*)entries.items[*pos];
 672     }
 673
 674     return NULL;
 675 }
 676
 677 /*
 678  * returns a pointer to a specified entry if any
 679  * note that we return one entry so it does not make much sense to ask for
 680  * "list" headers
 681  */
 682 HttpHeaderEntry *
 683 HttpHeader::findEntry(http_hdr_type id) const
 684 {
 685     HttpHeaderPos pos = HttpHeaderInitPos;
 686     HttpHeaderEntry *e;
 687     assert_eid(id);
 688     assert(!CBIT_TEST(ListHeadersMask, id));
 689
 690     /* check mask first */
 691
 692     if (!CBIT_TEST(mask, id))
 693         return NULL;
 694
 695     /* looks like we must have it, do linear search */
 696     while ((e = getEntry(&pos))) {
 697         if (e->id == id)
 698             return e;
 699     }
 700
 701     /* hm.. we thought it was there, but it was not found */
 702     assert(0);
 703
 704     return NULL;                /* not reached */
 705 }
 706
 707 /*
 708  * same as httpHeaderFindEntry
 709  */
 710 HttpHeaderEntry *
 711 HttpHeader::findLastEntry(http_hdr_type id) const
 712 {
 713     HttpHeaderPos pos = HttpHeaderInitPos;
 714     HttpHeaderEntry *e;
 715     HttpHeaderEntry *result = NULL;
 716     assert_eid(id);
 717     assert(!CBIT_TEST(ListHeadersMask, id));
 718
 719     /* check mask first */
 720
 721     if (!CBIT_TEST(mask, id))
 722         return NULL;
 723
 724     /* looks like we must have it, do linear search */
 725     while ((e = getEntry(&pos))) {
 726         if (e->id == id)
 727             result = e;
 728     }
 729
 730     assert(result);             /* must be there! */
 731     return result;
 732 }
 733
 734 /*
 735  * deletes all fields with a given name if any, returns #fields deleted;
 736  */
 737 int
 738 HttpHeader::delByName(const char *name)
 739 {
 740     int count = 0;
 741     HttpHeaderPos pos = HttpHeaderInitPos;
 742     HttpHeaderEntry *e;
 743     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 744     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 745
 746     while ((e = getEntry(&pos))) {
 747         if (!e->name.caseCmp(name))
 748             delAt(pos, count);
 749         else
 750             CBIT_SET(mask, e->id);
 751     }
 752
 753     return count;
 754 }
 755
 756 /* deletes all entries with a given id, returns the #entries deleted */
 757 int
 758 HttpHeader::delById(http_hdr_type id)
 759 {
 760     int count = 0;
 761     HttpHeaderPos pos = HttpHeaderInitPos;
 762     HttpHeaderEntry *e;
 763     debugs(55, 8, this << " del-by-id " << id);
 764     assert_eid(id);
 765     assert(id != HDR_OTHER);            /* does not make sense */
 766
 767     if (!CBIT_TEST(mask, id))
 768         return 0;
 769
 770     while ((e = getEntry(&pos))) {
 771         if (e->id == id)
 772             delAt(pos, count);
 773     }
 774
 775     CBIT_CLR(mask, id);
 776     assert(count);
 777     return count;
 778 }
 779
 780 /*
 781  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 782  * possible to iterate(search) and delete fields at the same time
 783  * NOTE: Does not update the header mask. Caller must follow up with
 784  * a call to refreshMask() if headers_deleted was incremented.
 785  */
 786 void
 787 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 788 {
 789     HttpHeaderEntry *e;
 790     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 791     e = (HttpHeaderEntry*)entries.items[pos];
 792     entries.items[pos] = NULL;
 793     /* decrement header length, allow for ": " and crlf */
 794     len -= e->name.size() + 2 + e->value.size() + 2;
 795     assert(len >= 0);
 796     delete e;
 797     ++headers_deleted;
 798 }
 799
 800 /*
 801  * Compacts the header storage
 802  */
 803 void
 804 HttpHeader::compact()
 805 {
 806     entries.prune(NULL);
 807 }
 808
 809 /*
 810  * Refreshes the header mask. Required after delAt() calls.
 811  */
 812 void
 813 HttpHeader::refreshMask()
 814 {
 815     httpHeaderMaskInit(&mask, 0);
 816     debugs(55, 7, "refreshing the mask in hdr " << this);
 817     HttpHeaderPos pos = HttpHeaderInitPos;
 818     while (HttpHeaderEntry *e = getEntry(&pos)) {
 819        CBIT_SET(mask, e->id);
 820     }
 821 }
 822
 823 /* appends an entry;
 824  * does not call e->clone() so one should not reuse "*e"
 825  */
 826 void
 827 HttpHeader::addEntry(HttpHeaderEntry * e)
 828 {
 829     assert(e);
 830     assert_eid(e->id);
 831     assert(e->name.size());
 832
 833     debugs(55, 9, this << " adding entry: " << e->id << " at " <<
 834            entries.count);
 835
 836     if (CBIT_TEST(mask, e->id))
 837         Headers[e->id].stat.repCount++;
 838     else
 839         CBIT_SET(mask, e->id);
 840
 841     entries.push_back(e);
 842
 843     /* increment header length, allow for ": " and crlf */
 844     len += e->name.size() + 2 + e->value.size() + 2;
 845 }
 846
 847 /* inserts an entry;
 848  * does not call e->clone() so one should not reuse "*e"
 849  */
 850 void
 851 HttpHeader::insertEntry(HttpHeaderEntry * e)
 852 {
 853     assert(e);
 854     assert_eid(e->id);
 855
 856     debugs(55, 7, this << " adding entry: " << e->id << " at " <<
 857            entries.count);
 858
 859     if (CBIT_TEST(mask, e->id))
 860         Headers[e->id].stat.repCount++;
 861     else
 862         CBIT_SET(mask, e->id);
 863
 864     entries.insert(e);
 865
 866     /* increment header length, allow for ": " and crlf */
 867     len += e->name.size() + 2 + e->value.size() + 2;
 868 }
 869
 870 bool
 871 HttpHeader::getList(http_hdr_type id, String *s) const
 872 {
 873     HttpHeaderEntry *e;
 874     HttpHeaderPos pos = HttpHeaderInitPos;
 875     debugs(55, 9, this << " joining for id " << id);
 876     /* only fields from ListHeaders array can be "listed" */
 877     assert(CBIT_TEST(ListHeadersMask, id));
 878
 879     if (!CBIT_TEST(mask, id))
 880         return false;
 881
 882     while ((e = getEntry(&pos))) {
 883         if (e->id == id)
 884             strListAdd(s, e->value.buf(), ',');
 885     }
 886
 887     /*
 888      * note: we might get an empty (size==0) string if there was an "empty"
 889      * header. This results in an empty length String, which may have a NULL
 890      * buffer.
 891      */
 892     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 893     if (!s->size())
 894         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 895     else
 896         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 897
 898     return true;
 899 }
 900
 901 /* return a list of entries with the same id separated by ',' and ws */
 902 String
 903 HttpHeader::getList(http_hdr_type id) const
 904 {
 905     HttpHeaderEntry *e;
 906     HttpHeaderPos pos = HttpHeaderInitPos;
 907     debugs(55, 9, this << "joining for id " << id);
 908     /* only fields from ListHeaders array can be "listed" */
 909     assert(CBIT_TEST(ListHeadersMask, id));
 910
 911     if (!CBIT_TEST(mask, id))
 912         return String();
 913
 914     String s;
 915
 916     while ((e = getEntry(&pos))) {
 917         if (e->id == id)
 918             strListAdd(&s, e->value.buf(), ',');
 919     }
 920
 921     /*
 922      * note: we might get an empty (size==0) string if there was an "empty"
 923      * header. This results in an empty length String, which may have a NULL
 924      * buffer.
 925      */
 926     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 927     if (!s.size())
 928         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 929     else
 930         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 931
 932     return s;
 933 }
 934
 935 /* return a string or list of entries with the same id separated by ',' and ws */
 936 String
 937 HttpHeader::getStrOrList(http_hdr_type id) const
 938 {
 939     HttpHeaderEntry *e;
 940
 941     if (CBIT_TEST(ListHeadersMask, id))
 942         return getList(id);
 943
 944     if ((e = findEntry(id)))
 945         return e->value;
 946
 947     return String();
 948 }
 949
 950 /*
 951  * Returns the value of the specified header.
 952  */
 953 String
 954 HttpHeader::getByName(const char *name) const
 955 {
 956     http_hdr_type id;
 957     HttpHeaderPos pos = HttpHeaderInitPos;
 958     HttpHeaderEntry *e;
 959
 960     assert(name);
 961
 962     /* First try the quick path */
 963     id = httpHeaderIdByNameDef(name, strlen(name));
 964
 965     if (id != -1)
 966         return getStrOrList(id);
 967
 968     String result;
 969
 970     /* Sorry, an unknown header name. Do linear search */
 971     while ((e = getEntry(&pos))) {
 972         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
 973             strListAdd(&result, e->value.buf(), ',');
 974         }
 975     }
 976
 977     return result;
 978 }
 979
 980 /*
 981  * Returns a the value of the specified list member, if any.
 982  */
 983 String
 984 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
 985 {
 986     String header;
 987     const char *pos = NULL;
 988     const char *item;
 989     int ilen;
 990     int mlen = strlen(member);
 991
 992     assert(name);
 993
 994     header = getByName(name);
 995
 996     String result;
 997
 998     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
 999         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1000             result.append(item + mlen + 1, ilen - mlen - 1);
1001             break;
1002         }
1003     }
1004
1005     return result;
1006 }
1007
1008 /*
1009  * returns a the value of the specified list member, if any.
1010  */
1011 String
1012 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1013 {
1014     String header;
1015     const char *pos = NULL;
1016     const char *item;
1017     int ilen;
1018     int mlen = strlen(member);
1019
1020     assert(id >= 0);
1021
1022     header = getStrOrList(id);
1023     String result;
1024
1025     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1026         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1027             result.append(item + mlen + 1, ilen - mlen - 1);
1028             break;
1029         }
1030     }
1031
1032     header.clean();
1033     return result;
1034 }
1035
1036 /* test if a field is present */
1037 int
1038 HttpHeader::has(http_hdr_type id) const
1039 {
1040     assert_eid(id);
1041     assert(id != HDR_OTHER);
1042     debugs(55, 9, this << " lookup for " << id);
1043     return CBIT_TEST(mask, id);
1044 }
1045
1046 void
1047 HttpHeader::putInt(http_hdr_type id, int number)
1048 {
1049     assert_eid(id);
1050     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1051     assert(number >= 0);
1052     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1053 }
1054
1055 void
1056 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1057 {
1058     assert_eid(id);
1059     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1060     assert(number >= 0);
1061     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1062 }
1063
1064 void
1065 HttpHeader::putTime(http_hdr_type id, time_t htime)
1066 {
1067     assert_eid(id);
1068     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1069     assert(htime >= 0);
1070     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1071 }
1072
1073 void
1074 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1075 {
1076     assert_eid(id);
1077     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1078     assert(htime >= 0);
1079     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1080 }
1081
1082 void
1083 HttpHeader::putStr(http_hdr_type id, const char *str)
1084 {
1085     assert_eid(id);
1086     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1087     assert(str);
1088     addEntry(new HttpHeaderEntry(id, NULL, str));
1089 }
1090
1091 void
1092 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1093 {
1094     assert(auth_scheme && realm);
1095     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1096 }
1097
1098 void
1099 HttpHeader::putCc(const HttpHdrCc * cc)
1100 {
1101     MemBuf mb;
1102     Packer p;
1103     assert(cc);
1104     /* remove old directives if any */
1105     delById(HDR_CACHE_CONTROL);
1106     /* pack into mb */
1107     mb.init();
1108     packerToMemInit(&p, &mb);
1109     httpHdrCcPackInto(cc, &p);
1110     /* put */
1111     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1112     /* cleanup */
1113     packerClean(&p);
1114     mb.clean();
1115 }
1116
1117 void
1118 HttpHeader::putContRange(const HttpHdrContRange * cr)
1119 {
1120     MemBuf mb;
1121     Packer p;
1122     assert(cr);
1123     /* remove old directives if any */
1124     delById(HDR_CONTENT_RANGE);
1125     /* pack into mb */
1126     mb.init();
1127     packerToMemInit(&p, &mb);
1128     httpHdrContRangePackInto(cr, &p);
1129     /* put */
1130     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1131     /* cleanup */
1132     packerClean(&p);
1133     mb.clean();
1134 }
1135
1136 void
1137 HttpHeader::putRange(const HttpHdrRange * range)
1138 {
1139     MemBuf mb;
1140     Packer p;
1141     assert(range);
1142     /* remove old directives if any */
1143     delById(HDR_RANGE);
1144     /* pack into mb */
1145     mb.init();
1146     packerToMemInit(&p, &mb);
1147     range->packInto(&p);
1148     /* put */
1149     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1150     /* cleanup */
1151     packerClean(&p);
1152     mb.clean();
1153 }
1154
1155 void
1156 HttpHeader::putSc(HttpHdrSc *sc)
1157 {
1158     MemBuf mb;
1159     Packer p;
1160     assert(sc);
1161     /* remove old directives if any */
1162     delById(HDR_RANGE);
1163     /* pack into mb */
1164     mb.init();
1165     packerToMemInit(&p, &mb);
1166     httpHdrScPackInto(sc, &p);
1167     /* put */
1168     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1169     /* cleanup */
1170     packerClean(&p);
1171     mb.clean();
1172 }
1173
1174 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1175 void
1176 HttpHeader::putExt(const char *name, const char *value)
1177 {
1178     assert(name && value);
1179     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1180     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1181 }
1182
1183 int
1184 HttpHeader::getInt(http_hdr_type id) const
1185 {
1186     assert_eid(id);
1187     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1188     HttpHeaderEntry *e;
1189
1190     if ((e = findEntry(id)))
1191         return e->getInt();
1192
1193     return -1;
1194 }
1195
1196 int64_t
1197 HttpHeader::getInt64(http_hdr_type id) const
1198 {
1199     assert_eid(id);
1200     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1201     HttpHeaderEntry *e;
1202
1203     if ((e = findEntry(id)))
1204         return e->getInt64();
1205
1206     return -1;
1207 }
1208
1209 time_t
1210 HttpHeader::getTime(http_hdr_type id) const
1211 {
1212     HttpHeaderEntry *e;
1213     time_t value = -1;
1214     assert_eid(id);
1215     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1216
1217     if ((e = findEntry(id))) {
1218         value = parse_rfc1123(e->value.buf());
1219         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1220     }
1221
1222     return value;
1223 }
1224
1225 /* sync with httpHeaderGetLastStr */
1226 const char *
1227 HttpHeader::getStr(http_hdr_type id) const
1228 {
1229     HttpHeaderEntry *e;
1230     assert_eid(id);
1231     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1232
1233     if ((e = findEntry(id))) {
1234         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1235         return e->value.buf();
1236     }
1237
1238     return NULL;
1239 }
1240
1241 /* unusual */
1242 const char *
1243 HttpHeader::getLastStr(http_hdr_type id) const
1244 {
1245     HttpHeaderEntry *e;
1246     assert_eid(id);
1247     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1248
1249     if ((e = findLastEntry(id))) {
1250         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1251         return e->value.buf();
1252     }
1253
1254     return NULL;
1255 }
1256
1257 HttpHdrCc *
1258 HttpHeader::getCc() const
1259 {
1260     HttpHdrCc *cc;
1261     String s;
1262
1263     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1264         return NULL;
1265     PROF_start(HttpHeader_getCc);
1266
1267     getList(HDR_CACHE_CONTROL, &s);
1268
1269     cc = httpHdrCcParseCreate(&s);
1270
1271     HttpHeaderStats[owner].ccParsedCount++;
1272
1273     if (cc)
1274         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1275
1276     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1277
1278     PROF_stop(HttpHeader_getCc);
1279
1280     return cc;
1281 }
1282
1283 HttpHdrRange *
1284 HttpHeader::getRange() const
1285 {
1286     HttpHdrRange *r = NULL;
1287     HttpHeaderEntry *e;
1288     /* some clients will send "Request-Range" _and_ *matching* "Range"
1289      * who knows, some clients might send Request-Range only;
1290      * this "if" should work correctly in both cases;
1291      * hopefully no clients send mismatched headers! */
1292
1293     if ((e = findEntry(HDR_RANGE)) ||
1294             (e = findEntry(HDR_REQUEST_RANGE))) {
1295         r = HttpHdrRange::ParseCreate(&e->value);
1296         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1297     }
1298
1299     return r;
1300 }
1301
1302 HttpHdrSc *
1303 HttpHeader::getSc() const
1304 {
1305     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1306         return NULL;
1307
1308     String s;
1309
1310     (void) getList(HDR_SURROGATE_CONTROL, &s);
1311
1312     HttpHdrSc *sc = httpHdrScParseCreate(&s);
1313
1314     HttpHeaderStats[owner].ccParsedCount++;
1315
1316     if (sc)
1317         httpHdrScUpdateStats(sc, &HttpHeaderStats[owner].scTypeDistr);
1318
1319     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1320
1321     return sc;
1322 }
1323
1324 HttpHdrContRange *
1325 HttpHeader::getContRange() const
1326 {
1327     HttpHdrContRange *cr = NULL;
1328     HttpHeaderEntry *e;
1329
1330     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1331         cr = httpHdrContRangeParseCreate(e->value.buf());
1332         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1333     }
1334
1335     return cr;
1336 }
1337
1338 const char *
1339 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1340 {
1341     const char *field;
1342     int l;
1343     assert(auth_scheme);
1344     field = getStr(id);
1345
1346     if (!field)                 /* no authorization field */
1347         return NULL;
1348
1349     l = strlen(auth_scheme);
1350
1351     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1352         return NULL;
1353
1354     field += l;
1355
1356     if (!xisspace(*field))      /* wrong scheme */
1357         return NULL;
1358
1359     /* skip white space */
1360     field += xcountws(field);
1361
1362     if (!*field)                /* no authorization cookie */
1363         return NULL;
1364
1365     return base64_decode(field);
1366 }
1367
1368 ETag
1369 HttpHeader::getETag(http_hdr_type id) const
1370 {
1371     ETag etag =
1372         {NULL, -1};
1373     HttpHeaderEntry *e;
1374     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1375
1376     if ((e = findEntry(id)))
1377         etagParseInit(&etag, e->value.buf());
1378
1379     return etag;
1380 }
1381
1382 TimeOrTag
1383 HttpHeader::getTimeOrTag(http_hdr_type id) const
1384 {
1385     TimeOrTag tot;
1386     HttpHeaderEntry *e;
1387     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1388     memset(&tot, 0, sizeof(tot));
1389
1390     if ((e = findEntry(id))) {
1391         const char *str = e->value.buf();
1392         /* try as an ETag */
1393
1394         if (etagParseInit(&tot.tag, str)) {
1395             tot.valid = tot.tag.str != NULL;
1396             tot.time = -1;
1397         } else {
1398             /* or maybe it is time? */
1399             tot.time = parse_rfc1123(str);
1400             tot.valid = tot.time >= 0;
1401             tot.tag.str = NULL;
1402         }
1403     }
1404
1405     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1406     return tot;
1407 }
1408
1409 /*
1410  * HttpHeaderEntry
1411  */
1412
1413 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1414 {
1415     assert_eid(anId);
1416     id = anId;
1417
1418     if (id != HDR_OTHER)
1419         name = Headers[id].name;
1420     else
1421         name = aName;
1422
1423     value = aValue;
1424
1425     Headers[id].stat.aliveCount++;
1426
1427     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name.buf() << " : " << value.buf());
1428 }
1429
1430 HttpHeaderEntry::~HttpHeaderEntry()
1431 {
1432     assert_eid(id);
1433     debugs(55, 9, "destroying entry " << this << ": '" << name.buf() << ": " << value.buf() << "'");
1434     /* clean name if needed */
1435
1436     if (id == HDR_OTHER)
1437         name.clean();
1438
1439     value.clean();
1440
1441     assert(Headers[id].stat.aliveCount);
1442
1443     Headers[id].stat.aliveCount--;
1444
1445     id = HDR_BAD_HDR;
1446 }
1447
1448 /* parses and inits header entry, returns true/false */
1449 HttpHeaderEntry *
1450 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1451 {
1452     /* note: name_start == field_start */
1453     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1454     int name_len = name_end ? name_end - field_start : 0;
1455     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1456     /* note: value_end == field_end */
1457
1458     HeaderEntryParsedCount++;
1459
1460     /* do we have a valid field name within this field? */
1461
1462     if (!name_len || name_end > field_end)
1463         return NULL;
1464
1465     if (name_len > 65534) {
1466         /* String must be LESS THAN 64K and it adds a terminating NULL */
1467         debugs(55, 1, "WARNING: ignoring header name of " << name_len << " bytes");
1468         return NULL;
1469     }
1470
1471     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1472         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1473                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1474
1475         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1476             name_len--;
1477
1478         if (!name_len)
1479             return NULL;
1480     }
1481
1482     /* now we know we can parse it */
1483
1484     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1485
1486     /* is it a "known" field? */
1487     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1488
1489     String name;
1490
1491     String value;
1492
1493     if (id < 0)
1494         id = HDR_OTHER;
1495
1496     assert_eid(id);
1497
1498     /* set field name */
1499     if (id == HDR_OTHER)
1500         name.limitInit(field_start, name_len);
1501     else
1502         name = Headers[id].name;
1503
1504     /* trim field value */
1505     while (value_start < field_end && xisspace(*value_start))
1506         value_start++;
1507
1508     while (value_start < field_end && xisspace(field_end[-1]))
1509         field_end--;
1510
1511     if (field_end - value_start > 65534) {
1512         /* String must be LESS THAN 64K and it adds a terminating NULL */
1513         debugs(55, 1, "WARNING: ignoring '" << name.buf() << "' header of " << (field_end - value_start) << " bytes");
1514
1515         if (id == HDR_OTHER)
1516             name.clean();
1517
1518         return NULL;
1519     }
1520
1521     /* set field value */
1522     value.limitInit(value_start, field_end - value_start);
1523
1524     Headers[id].stat.seenCount++;
1525
1526     Headers[id].stat.aliveCount++;
1527
1528     debugs(55, 9, "parsed HttpHeaderEntry: '" << name.buf() << ": " << value.buf() << "'");
1529
1530     return new HttpHeaderEntry(id, name.buf(), value.buf());
1531 }
1532
1533 HttpHeaderEntry *
1534 HttpHeaderEntry::clone() const
1535 {
1536     return new HttpHeaderEntry(id, name.buf(), value.buf());
1537 }
1538
1539 void
1540 HttpHeaderEntry::packInto(Packer * p) const
1541 {
1542     assert(p);
1543     packerAppend(p, name.buf(), name.size());
1544     packerAppend(p, ": ", 2);
1545     packerAppend(p, value.buf(), value.size());
1546     packerAppend(p, "\r\n", 2);
1547 }
1548
1549 int
1550 HttpHeaderEntry::getInt() const
1551 {
1552     assert_eid (id);
1553     assert (Headers[id].type == ftInt);
1554     int val = -1;
1555     int ok = httpHeaderParseInt(value.buf(), &val);
1556     httpHeaderNoteParsedEntry(id, value, !ok);
1557     /* XXX: Should we check ok - ie
1558      * return ok ? -1 : value;
1559      */
1560     return val;
1561 }
1562
1563 int64_t
1564 HttpHeaderEntry::getInt64() const
1565 {
1566     assert_eid (id);
1567     assert (Headers[id].type == ftInt64);
1568     int64_t val = -1;
1569     int ok = httpHeaderParseOffset(value.buf(), &val);
1570     httpHeaderNoteParsedEntry(id, value, !ok);
1571     /* XXX: Should we check ok - ie
1572      * return ok ? -1 : value;
1573      */
1574     return val;
1575 }
1576
1577 static void
1578 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1579 {
1580     Headers[id].stat.parsCount++;
1581
1582     if (error) {
1583         Headers[id].stat.errCount++;
1584         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name.buf() << ": " << context.buf() << "'");
1585     }
1586 }
1587
1588 /*
1589  * Reports
1590  */
1591
1592 /* tmp variable used to pass stat info to dumpers */
1593 extern const HttpHeaderStat *dump_stat;         /* argh! */
1594 const HttpHeaderStat *dump_stat = NULL;
1595
1596 static void
1597 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1598 {
1599     const int id = (int) val;
1600     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1601     const char *name = valid_id ? Headers[id].name.buf() : "INVALID";
1602     int visible = count > 0;
1603     /* for entries with zero count, list only those that belong to current type of message */
1604
1605     if (!visible && valid_id && dump_stat->owner_mask)
1606         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1607
1608     if (visible)
1609         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1610                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1611 }
1612
1613 static void
1614 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1615 {
1616     if (count)
1617         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1618                           idx, (int) val, count,
1619                           xpercent(count, dump_stat->destroyedCount));
1620 }
1621
1622
1623 static void
1624 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1625 {
1626     assert(hs && e);
1627
1628     dump_stat = hs;
1629     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1630     storeAppendPrintf(e, "\nField type distribution\n");
1631     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1632                       "id", "name", "count", "#/header");
1633     statHistDump(&hs->fieldTypeDistr, e, httpHeaderFieldStatDumper);
1634     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1635     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1636                       "id", "name", "count", "#/cc_field");
1637     statHistDump(&hs->ccTypeDistr, e, httpHdrCcStatDumper);
1638     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1639     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1640                       "id", "name", "count", "#/sc_field");
1641     statHistDump(&hs->scTypeDistr, e, httpHdrScStatDumper);
1642     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1643     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1644                       "id", "#flds", "count", "%total");
1645     statHistDump(&hs->hdrUCountDistr, e, httpHeaderFldsPerHdrDumper);
1646     dump_stat = NULL;
1647 }
1648
1649 void
1650 httpHeaderStoreReport(StoreEntry * e)
1651 {
1652     int i;
1653     http_hdr_type ht;
1654     assert(e);
1655
1656     HttpHeaderStats[0].parsedCount =
1657         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1658     HttpHeaderStats[0].ccParsedCount =
1659         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1660     HttpHeaderStats[0].destroyedCount =
1661         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1662     HttpHeaderStats[0].busyDestroyedCount =
1663         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1664
1665     for (i = 1; i < HttpHeaderStatCount; i++) {
1666         httpHeaderStatDump(HttpHeaderStats + i, e);
1667         storeAppendPrintf(e, "%s\n", "<br>");
1668     }
1669
1670     /* field stats for all messages */
1671     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1672
1673     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1674                       "id", "name", "#alive", "%err", "%repeat");
1675
1676     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1677         HttpHeaderFieldInfo *f = Headers + ht;
1678         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1679                           f->id, f->name.buf(), f->stat.aliveCount,
1680                           xpercent(f->stat.errCount, f->stat.parsCount),
1681                           xpercent(f->stat.repCount, f->stat.seenCount));
1682     }
1683
1684     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1685                       HttpHeaderStats[hoRequest].parsedCount,
1686                       HttpHeaderStats[hoReply].parsedCount,
1687                       HttpHeaderStats[0].parsedCount);
1688     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1689 }
1690
1691 http_hdr_type
1692 httpHeaderIdByName(const char *name, int name_len, const HttpHeaderFieldInfo * info, int end)
1693 {
1694     int i;
1695
1696     for (i = 0; i < end; ++i) {
1697         if (name_len >= 0 && name_len != info[i].name.size())
1698             continue;
1699
1700         if (!strncasecmp(name, info[i].name.buf(),
1701                          name_len < 0 ? info[i].name.size() + 1 : name_len))
1702             return info[i].id;
1703     }
1704
1705     return HDR_BAD_HDR;
1706 }
1707
1708 http_hdr_type
1709 httpHeaderIdByNameDef(const char *name, int name_len)
1710 {
1711     if (!Headers)
1712         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1713
1714     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1715 }
1716
1717 const char *
1718 httpHeaderNameById(int id)
1719 {
1720     if (!Headers)
1721         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1722
1723     assert(id >= 0 && id < HDR_ENUM_END);
1724
1725     return Headers[id].name.buf();
1726 }
1727
1728 int
1729 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1730 {
1731     int result = 0;
1732     const char *pos = NULL;
1733     const char *item;
1734     int ilen;
1735     int mlen = strlen(member);
1736
1737     assert(id >= 0);
1738
1739     String header (getStrOrList(id));
1740
1741     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1742         if (strncmp(item, member, mlen) == 0
1743                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1744             result = 1;
1745             break;
1746         }
1747     }
1748
1749     return result;
1750 }
1751
1752 int
1753 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1754 {
1755     int result = 0;
1756     const char *pos = NULL;
1757     const char *item;
1758     int ilen;
1759     int mlen = strlen(member);
1760
1761     assert(name);
1762
1763     String header (getByName(name));
1764
1765     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1766         if (strncmp(item, member, mlen) == 0
1767                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1768             result = 1;
1769             break;
1770         }
1771     }
1772
1773     return result;
1774 }
1775
1776 void
1777 HttpHeader::removeHopByHopEntries()
1778 {
1779     removeConnectionHeaderEntries();
1780
1781     const HttpHeaderEntry *e;
1782     HttpHeaderPos pos = HttpHeaderInitPos;
1783     int headers_deleted = 0;
1784     while ((e = getEntry(&pos))) {
1785         int id = e->id;
1786         if(CBIT_TEST(HopByHopHeadersMask, id)){
1787             delAt(pos, headers_deleted);
1788             CBIT_CLR(mask, id);
1789         }
1790     }
1791 }
1792
1793 void
1794 HttpHeader::removeConnectionHeaderEntries()
1795 {
1796     if (has(HDR_CONNECTION)) {
1797         /* anything that matches Connection list member will be deleted */
1798         String strConnection;
1799
1800         (void) getList(HDR_CONNECTION, &strConnection);
1801         const HttpHeaderEntry *e;
1802         HttpHeaderPos pos = HttpHeaderInitPos;
1803         /*
1804          * think: on-average-best nesting of the two loops (hdrEntry
1805          * and strListItem) @?@
1806          */
1807         /*
1808          * maybe we should delete standard stuff ("keep-alive","close")
1809          * from strConnection first?
1810          */
1811
1812         int headers_deleted = 0;
1813         while ((e = getEntry(&pos))) {
1814             if (strListIsMember(&strConnection, e->name.buf(), ','))
1815                 delAt(pos, headers_deleted);
1816         }
1817         if (headers_deleted)
1818             refreshMask();
1819     }
1820 }