src/HttpHeader.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 55    HTTP Header
   6  * AUTHOR: Alex Rousskov
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "base64.h"
  38 #include "HttpHdrCc.h"
  39 #include "HttpHdrContRange.h"
  40 #include "HttpHdrSc.h"
  41 #include "HttpHeader.h"
  42 #include "HttpHeaderStat.h"
  43 #include "MemBuf.h"
  44 #include "mgr/Registration.h"
  45 #include "profiler/Profiler.h"
  46 #include "protos.h"
  47 #include "rfc1123.h"
  48 #include "StatHist.h"
  49 #include "Store.h"
  50 #include "TimeOrTag.h"
  51
  52 /*
  53  * On naming conventions:
  54  *
  55  * HTTP/1.1 defines message-header as
  56  *
  57  * message-header = field-name ":" [ field-value ] CRLF
  58  * field-name     = token
  59  * field-value    = *( field-content | LWS )
  60  *
  61  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  62  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  63  *
  64  * HttpHeader is an object that represents all message-headers in a message.
  65  * HttpHeader does not manage start-line.
  66  *
  67  * HttpHeader is implemented as a collection of header "entries".
  68  * An entry is a (field_id, field_name, field_value) triplet.
  69  */
  70
  71 /*
  72  * local constants and vars
  73  */
  74
  75 /*
  76  * A table with major attributes for every known field.
  77  * We calculate name lengths and reorganize this array on start up.
  78  * After reorganization, field id can be used as an index to the table.
  79  */
  80 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  81     {"Accept", HDR_ACCEPT, ftStr},
  82
  83     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  84     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  85     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  86     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  87     {"Age", HDR_AGE, ftInt},
  88     {"Allow", HDR_ALLOW, ftStr},
  89     {"Authorization", HDR_AUTHORIZATION, ftStr},        /* for now */
  90     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  91     {"Connection", HDR_CONNECTION, ftStr},
  92     {"Content-Base", HDR_CONTENT_BASE, ftStr},
  93     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
  94     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  95     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  96     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  97     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  98     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
  99     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
 100     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
 101     {"Cookie", HDR_COOKIE, ftStr},
 102     {"Cookie2", HDR_COOKIE2, ftStr},
 103     {"Date", HDR_DATE, ftDate_1123},
 104     {"ETag", HDR_ETAG, ftETag},
 105     {"Expect", HDR_EXPECT, ftStr},
 106     {"Expires", HDR_EXPIRES, ftDate_1123},
 107     {"From", HDR_FROM, ftStr},
 108     {"Host", HDR_HOST, ftStr},
 109     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 110     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 111     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},        /* for now */
 112     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 113     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 114     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 115     {"Link", HDR_LINK, ftStr},
 116     {"Location", HDR_LOCATION, ftStr},
 117     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 118     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 119     {"Negotiate", HDR_NEGOTIATE, ftStr},
 120     {"Origin", HDR_ORIGIN, ftStr},
 121     {"Pragma", HDR_PRAGMA, ftStr},
 122     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 123     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 124     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 125     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 126     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 127     {"Public", HDR_PUBLIC, ftStr},
 128     {"Range", HDR_RANGE, ftPRange},
 129     {"Referer", HDR_REFERER, ftStr},
 130     {"Request-Range", HDR_REQUEST_RANGE, ftPRange},     /* usually matches HDR_RANGE */
 131     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!) */
 132     {"Server", HDR_SERVER, ftStr},
 133     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 134     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 135     {"TE", HDR_TE, ftStr},
 136     {"Title", HDR_TITLE, ftStr},
 137     {"Trailer", HDR_TRAILER, ftStr},
 138     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 139     {"Translate", HDR_TRANSLATE, ftStr},        /* for now. may need to crop */
 140     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 141     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 142     {"User-Agent", HDR_USER_AGENT, ftStr},
 143     {"Vary", HDR_VARY, ftStr},  /* for now */
 144     {"Via", HDR_VIA, ftStr},    /* for now */
 145     {"Warning", HDR_WARNING, ftStr},    /* for now */
 146     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 147     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 148     {"X-Cache", HDR_X_CACHE, ftStr},
 149     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 150     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 151     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 152     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 153 #if X_ACCELERATOR_VARY
 154     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 155 #endif
 156 #if USE_ADAPTATION
 157     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 158 #endif
 159     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 160     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 161     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 162     {"Other:", HDR_OTHER, ftStr}        /* ':' will not allow matches */
 163 };
 164
 165 static HttpHeaderFieldInfo *Headers = NULL;
 166
 167 http_hdr_type &operator++ (http_hdr_type &aHeader)
 168 {
 169     int tmp = (int)aHeader;
 170     aHeader = (http_hdr_type)(++tmp);
 171     return aHeader;
 172 }
 173
 174 /*
 175  * headers with field values defined as #(values) in HTTP/1.1
 176  * Headers that are currently not recognized, are commented out.
 177  */
 178 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 179 static http_hdr_type ListHeadersArr[] = {
 180     HDR_ACCEPT,
 181     HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 182     HDR_ACCEPT_RANGES, HDR_ALLOW,
 183     HDR_CACHE_CONTROL,
 184     HDR_CONTENT_ENCODING,
 185     HDR_CONTENT_LANGUAGE,
 186     HDR_CONNECTION,
 187     HDR_EXPECT,
 188     HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 189     HDR_LINK, HDR_PRAGMA,
 190     HDR_PROXY_CONNECTION,
 191     HDR_PROXY_SUPPORT,
 192     HDR_TRANSFER_ENCODING,
 193     HDR_UPGRADE,
 194     HDR_VARY,
 195     HDR_VIA,
 196     HDR_WARNING,
 197     HDR_WWW_AUTHENTICATE,
 198     HDR_AUTHENTICATION_INFO,
 199     HDR_PROXY_AUTHENTICATION_INFO,
 200     /* HDR_TE, HDR_TRAILER */
 201 #if X_ACCELERATOR_VARY
 202     HDR_X_ACCELERATOR_VARY,
 203 #endif
 204 #if USE_ADAPTATION
 205     HDR_X_NEXT_SERVICES,
 206 #endif
 207     HDR_SURROGATE_CAPABILITY,
 208     HDR_SURROGATE_CONTROL,
 209     HDR_X_FORWARDED_FOR
 210 };
 211
 212 /* general-headers */
 213 static http_hdr_type GeneralHeadersArr[] = {
 214     HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 215     HDR_TRANSFER_ENCODING,
 216     HDR_UPGRADE,
 217     /* HDR_TRAILER, */
 218     HDR_VIA,
 219 };
 220
 221 /* entity-headers */
 222 static http_hdr_type EntityHeadersArr[] = {
 223     HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 224     HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 225     HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 226     HDR_OTHER
 227 };
 228
 229 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 230 static http_hdr_type ReplyHeadersArr[] = {
 231     HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 232     HDR_ACCEPT_RANGES, HDR_AGE,
 233     HDR_LOCATION, HDR_MAX_FORWARDS,
 234     HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_SET_COOKIE2,
 235     HDR_ORIGIN,
 236     HDR_VARY,
 237     HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 238     HDR_X_CACHE_LOOKUP,
 239     HDR_X_REQUEST_URI,
 240 #if X_ACCELERATOR_VARY
 241     HDR_X_ACCELERATOR_VARY,
 242 #endif
 243 #if USE_ADAPTATION
 244     HDR_X_NEXT_SERVICES,
 245 #endif
 246     HDR_X_SQUID_ERROR,
 247     HDR_SURROGATE_CONTROL
 248 };
 249
 250 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 251 static http_hdr_type RequestHeadersArr[] = {
 252     HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 253     HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 254     HDR_IF_RANGE, HDR_MAX_FORWARDS,
 255     HDR_ORIGIN,
 256     HDR_PROXY_CONNECTION,
 257     HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 258     HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 259 };
 260
 261 static HttpHeaderMask HopByHopHeadersMask;
 262 static http_hdr_type HopByHopHeadersArr[] = {
 263     HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 264     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 265 };
 266
 267 /* header accounting */
 268 static HttpHeaderStat HttpHeaderStats[] = {
 269     {"all"},
 270 #if USE_HTCP
 271     {"HTCP reply"},
 272 #endif
 273     {"request"},
 274     {"reply"}
 275 };
 276 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 277
 278 static int HeaderEntryParsedCount = 0;
 279
 280 /*
 281  * local routines
 282  */
 283
 284 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 285
 286 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 287
 288 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 289 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 290
 291 /*
 292  * Module initialization routines
 293  */
 294
 295 static void
 296 httpHeaderRegisterWithCacheManager(void)
 297 {
 298     Mgr::RegisterAction("http_headers",
 299                         "HTTP Header Statistics",
 300                         httpHeaderStoreReport, 0, 1);
 301 }
 302
 303 void
 304 httpHeaderInitModule(void)
 305 {
 306     int i;
 307     /* check that we have enough space for masks */
 308     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 309     /* all headers must be described */
 310     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 311
 312     if (!Headers)
 313         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 314
 315     /* create masks */
 316     httpHeaderMaskInit(&ListHeadersMask, 0);
 317
 318     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 319
 320     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 321
 322     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 323
 324     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 325
 326     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 327
 328     httpHeaderMaskInit(&RequestHeadersMask, 0);
 329
 330     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 331
 332     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 333
 334     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 335
 336     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 337
 338     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 339
 340     /* init header stats */
 341     assert(HttpHeaderStatCount == hoReply + 1);
 342
 343     for (i = 0; i < HttpHeaderStatCount; ++i)
 344         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 345
 346     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 347
 348     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 349
 350 #if USE_HTCP
 351
 352     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 353
 354 #endif
 355     /* init dependent modules */
 356     httpHdrCcInitModule();
 357
 358     httpHdrScInitModule();
 359
 360     httpHeaderRegisterWithCacheManager();
 361 }
 362
 363 void
 364 httpHeaderCleanModule(void)
 365 {
 366     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 367     Headers = NULL;
 368     httpHdrCcCleanModule();
 369     httpHdrScCleanModule();
 370 }
 371
 372 static void
 373 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 374 {
 375     assert(hs);
 376     assert(label);
 377     memset(hs, 0, sizeof(HttpHeaderStat));
 378     hs->label = label;
 379     hs->hdrUCountDistr.enumInit(32);    /* not a real enum */
 380     hs->fieldTypeDistr.enumInit(HDR_ENUM_END);
 381     hs->ccTypeDistr.enumInit(CC_ENUM_END);
 382     hs->scTypeDistr.enumInit(SC_ENUM_END);
 383 }
 384
 385 /*
 386  * HttpHeader Implementation
 387  */
 388
 389 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 390 {
 391     httpHeaderMaskInit(&mask, 0);
 392 }
 393
 394 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 395 {
 396     assert(anOwner > hoNone && anOwner < hoEnd);
 397     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 398     httpHeaderMaskInit(&mask, 0);
 399 }
 400
 401 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 402 {
 403     httpHeaderMaskInit(&mask, 0);
 404     update(&other, NULL); // will update the mask as well
 405 }
 406
 407 HttpHeader::~HttpHeader()
 408 {
 409     clean();
 410 }
 411
 412 HttpHeader &
 413 HttpHeader::operator =(const HttpHeader &other)
 414 {
 415     if (this != &other) {
 416         // we do not really care, but the caller probably does
 417         assert(owner == other.owner);
 418         clean();
 419         update(&other, NULL); // will update the mask as well
 420         len = other.len;
 421     }
 422     return *this;
 423 }
 424
 425 void
 426 HttpHeader::clean()
 427 {
 428     HttpHeaderPos pos = HttpHeaderInitPos;
 429     HttpHeaderEntry *e;
 430
 431     assert(owner > hoNone && owner < hoEnd);
 432     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 433
 434     PROF_start(HttpHeaderClean);
 435
 436     /*
 437      * An unfortunate bug.  The entries array is initialized
 438      * such that count is set to zero.  httpHeaderClean() seems to
 439      * be called both when 'hdr' is created, and destroyed.  Thus,
 440      * we accumulate a large number of zero counts for 'hdr' before
 441      * it is ever used.  Can't think of a good way to fix it, except
 442      * adding a state variable that indicates whether or not 'hdr'
 443      * has been used.  As a hack, just never count zero-sized header
 444      * arrays.
 445      */
 446
 447     if (owner <= hoReply) {
 448         if (0 != entries.count)
 449             HttpHeaderStats[owner].hdrUCountDistr.count(entries.count);
 450
 451         ++ HttpHeaderStats[owner].destroyedCount;
 452
 453         HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 454
 455         while ((e = getEntry(&pos))) {
 456             /* tmp hack to try to avoid coredumps */
 457
 458             if (e->id < 0 || e->id >= HDR_ENUM_END) {
 459                 debugs(55, DBG_CRITICAL, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 460             } else {
 461                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 462                 /* yes, this deletion leaves us in an inconsistent state */
 463                 delete e;
 464             }
 465         }
 466     } // if (owner <= hoReply)
 467     entries.clean();
 468     httpHeaderMaskInit(&mask, 0);
 469     len = 0;
 470     PROF_stop(HttpHeaderClean);
 471 }
 472
 473 /* append entries (also see httpHeaderUpdate) */
 474 void
 475 HttpHeader::append(const HttpHeader * src)
 476 {
 477     const HttpHeaderEntry *e;
 478     HttpHeaderPos pos = HttpHeaderInitPos;
 479     assert(src);
 480     assert(src != this);
 481     debugs(55, 7, "appending hdr: " << this << " += " << src);
 482
 483     while ((e = src->getEntry(&pos))) {
 484         addEntry(e->clone());
 485     }
 486 }
 487
 488 /* use fresh entries to replace old ones */
 489 void
 490 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 491 {
 492     assert (old);
 493     old->update (fresh, denied_mask);
 494 }
 495
 496 void
 497 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 498 {
 499     const HttpHeaderEntry *e;
 500     HttpHeaderPos pos = HttpHeaderInitPos;
 501     assert(fresh);
 502     assert(this != fresh);
 503
 504     while ((e = fresh->getEntry(&pos))) {
 505         /* deny bad guys (ok to check for HDR_OTHER) here */
 506
 507         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 508             continue;
 509
 510         if (e->id != HDR_OTHER)
 511             delById(e->id);
 512         else
 513             delByName(e->name.termedBuf());
 514     }
 515
 516     pos = HttpHeaderInitPos;
 517     while ((e = fresh->getEntry(&pos))) {
 518         /* deny bad guys (ok to check for HDR_OTHER) here */
 519
 520         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 521             continue;
 522
 523         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 524
 525         addEntry(e->clone());
 526     }
 527 }
 528
 529 /* just handy in parsing: resets and returns false */
 530 int
 531 HttpHeader::reset()
 532 {
 533     clean();
 534     return 0;
 535 }
 536
 537 int
 538 HttpHeader::parse(const char *header_start, const char *header_end)
 539 {
 540     const char *field_ptr = header_start;
 541     HttpHeaderEntry *e, *e2;
 542
 543     PROF_start(HttpHeaderParse);
 544
 545     assert(header_start && header_end);
 546     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 547     ++ HttpHeaderStats[owner].parsedCount;
 548
 549     char *nulpos;
 550     if ((nulpos = (char*)memchr(header_start, '\0', header_end - header_start))) {
 551         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 552                getStringPrefix(header_start, nulpos) << "}\nNULL\n{" << getStringPrefix(nulpos+1, header_end));
 553         goto reset;
 554     }
 555
 556     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 557      * continuation lines start with a (single) space or tab */
 558     while (field_ptr < header_end) {
 559         const char *field_start = field_ptr;
 560         const char *field_end;
 561
 562         do {
 563             const char *this_line = field_ptr;
 564             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 565
 566             if (!field_ptr)
 567                 goto reset;     /* missing <LF> */
 568
 569             field_end = field_ptr;
 570
 571             ++field_ptr;        /* Move to next line */
 572
 573             if (field_end > this_line && field_end[-1] == '\r') {
 574                 --field_end;    /* Ignore CR LF */
 575
 576                 if (owner == hoRequest && field_end > this_line) {
 577                     bool cr_only = true;
 578                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 579                         if (*p != '\r')
 580                             cr_only = false;
 581                     }
 582                     if (cr_only) {
 583                         debugs(55, DBG_IMPORTANT, "WARNING: Rejecting HTTP request with a CR+ "
 584                                "header field to prevent request smuggling attacks: {" <<
 585                                getStringPrefix(header_start, header_end) << "}");
 586                         goto reset;
 587                     }
 588                 }
 589             }
 590
 591             /* Barf on stray CR characters */
 592             if (memchr(this_line, '\r', field_end - this_line)) {
 593                 debugs(55, DBG_IMPORTANT, "WARNING: suspicious CR characters in HTTP header {" <<
 594                        getStringPrefix(field_start, field_end) << "}");
 595
 596                 if (Config.onoff.relaxed_header_parser) {
 597                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 598
 599                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 600                         *p = ' ';
 601                         ++p;
 602                     }
 603                 } else
 604                     goto reset;
 605             }
 606
 607             if (this_line + 1 == field_end && this_line > field_start) {
 608                 debugs(55, DBG_IMPORTANT, "WARNING: Blank continuation line in HTTP header {" <<
 609                        getStringPrefix(header_start, header_end) << "}");
 610                 goto reset;
 611             }
 612         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 613
 614         if (field_start == field_end) {
 615             if (field_ptr < header_end) {
 616                 debugs(55, DBG_IMPORTANT, "WARNING: unparseable HTTP header field near {" <<
 617                        getStringPrefix(field_start, header_end) << "}");
 618                 goto reset;
 619             }
 620
 621             break;              /* terminating blank line */
 622         }
 623
 624         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 625             debugs(55, DBG_IMPORTANT, "WARNING: unparseable HTTP header field {" <<
 626                    getStringPrefix(field_start, field_end) << "}");
 627             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 628                    " in {" << getStringPrefix(header_start, header_end) << "}");
 629
 630             if (Config.onoff.relaxed_header_parser)
 631                 continue;
 632             else
 633                 goto reset;
 634         }
 635
 636         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 637 //            if (e->value.cmp(e2->value.termedBuf()) != 0) {
 638             if (e->value != e2->value) {
 639                 int64_t l1, l2;
 640                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 641                        "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 642
 643                 if (!Config.onoff.relaxed_header_parser) {
 644                     delete e;
 645                     goto reset;
 646                 }
 647
 648                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 649                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e->value << "'");
 650                     delete e;
 651                     continue;
 652                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 653                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e2->value << "'");
 654                     delById(e2->id);
 655                 } else if (l1 > l2) {
 656                     delById(e2->id);
 657                 } else {
 658                     delete e;
 659                     continue;
 660                 }
 661             } else {
 662                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 663                        "NOTICE: found double content-length header");
 664
 665                 if (Config.onoff.relaxed_header_parser) {
 666                     delete e;
 667                     continue;
 668                 } else {
 669                     delete e;
 670                     goto reset;
 671                 }
 672             }
 673         }
 674
 675         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 676             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 677                    "WARNING: found whitespace in HTTP header name {" <<
 678                    getStringPrefix(field_start, field_end) << "}");
 679
 680             if (!Config.onoff.relaxed_header_parser) {
 681                 delete e;
 682                 goto reset;
 683             }
 684         }
 685
 686         addEntry(e);
 687     }
 688
 689     if (chunked()) {
 690         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 691         delById(HDR_CONTENT_LENGTH);
 692     }
 693
 694     PROF_stop(HttpHeaderParse);
 695     return 1;                   /* even if no fields where found, it is a valid header */
 696 reset:
 697     PROF_stop(HttpHeaderParse);
 698     return reset();
 699 }
 700
 701 /* packs all the entries using supplied packer */
 702 void
 703 HttpHeader::packInto(Packer * p, bool mask_sensitive_info) const
 704 {
 705     HttpHeaderPos pos = HttpHeaderInitPos;
 706     const HttpHeaderEntry *e;
 707     assert(p);
 708     debugs(55, 7, "packing hdr: (" << this << ")");
 709     /* pack all entries one by one */
 710     while ((e = getEntry(&pos))) {
 711         if (!mask_sensitive_info) {
 712             e->packInto(p);
 713             continue;
 714         }
 715         switch (e->id) {
 716         case HDR_AUTHORIZATION:
 717         case HDR_PROXY_AUTHORIZATION:
 718             packerAppend(p, e->name.rawBuf(), e->name.size());
 719             packerAppend(p, ": ** NOT DISPLAYED **\r\n", 23);
 720             break;
 721         default:
 722             e->packInto(p);
 723             break;
 724         }
 725     }
 726     /* Pack in the "special" entries */
 727
 728     /* Cache-Control */
 729 }
 730
 731 /* returns next valid entry */
 732 HttpHeaderEntry *
 733 HttpHeader::getEntry(HttpHeaderPos * pos) const
 734 {
 735     assert(pos);
 736     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 737
 738     for (++(*pos); *pos < (ssize_t)entries.count; ++(*pos)) {
 739         if (entries.items[*pos])
 740             return (HttpHeaderEntry*)entries.items[*pos];
 741     }
 742
 743     return NULL;
 744 }
 745
 746 /*
 747  * returns a pointer to a specified entry if any
 748  * note that we return one entry so it does not make much sense to ask for
 749  * "list" headers
 750  */
 751 HttpHeaderEntry *
 752 HttpHeader::findEntry(http_hdr_type id) const
 753 {
 754     HttpHeaderPos pos = HttpHeaderInitPos;
 755     HttpHeaderEntry *e;
 756     assert_eid(id);
 757     assert(!CBIT_TEST(ListHeadersMask, id));
 758
 759     /* check mask first */
 760
 761     if (!CBIT_TEST(mask, id))
 762         return NULL;
 763
 764     /* looks like we must have it, do linear search */
 765     while ((e = getEntry(&pos))) {
 766         if (e->id == id)
 767             return e;
 768     }
 769
 770     /* hm.. we thought it was there, but it was not found */
 771     assert(0);
 772
 773     return NULL;                /* not reached */
 774 }
 775
 776 /*
 777  * same as httpHeaderFindEntry
 778  */
 779 HttpHeaderEntry *
 780 HttpHeader::findLastEntry(http_hdr_type id) const
 781 {
 782     HttpHeaderPos pos = HttpHeaderInitPos;
 783     HttpHeaderEntry *e;
 784     HttpHeaderEntry *result = NULL;
 785     assert_eid(id);
 786     assert(!CBIT_TEST(ListHeadersMask, id));
 787
 788     /* check mask first */
 789
 790     if (!CBIT_TEST(mask, id))
 791         return NULL;
 792
 793     /* looks like we must have it, do linear search */
 794     while ((e = getEntry(&pos))) {
 795         if (e->id == id)
 796             result = e;
 797     }
 798
 799     assert(result);             /* must be there! */
 800     return result;
 801 }
 802
 803 /*
 804  * deletes all fields with a given name if any, returns #fields deleted;
 805  */
 806 int
 807 HttpHeader::delByName(const char *name)
 808 {
 809     int count = 0;
 810     HttpHeaderPos pos = HttpHeaderInitPos;
 811     HttpHeaderEntry *e;
 812     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 813     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 814
 815     while ((e = getEntry(&pos))) {
 816         if (!e->name.caseCmp(name))
 817             delAt(pos, count);
 818         else
 819             CBIT_SET(mask, e->id);
 820     }
 821
 822     return count;
 823 }
 824
 825 /* deletes all entries with a given id, returns the #entries deleted */
 826 int
 827 HttpHeader::delById(http_hdr_type id)
 828 {
 829     int count = 0;
 830     HttpHeaderPos pos = HttpHeaderInitPos;
 831     HttpHeaderEntry *e;
 832     debugs(55, 8, this << " del-by-id " << id);
 833     assert_eid(id);
 834     assert(id != HDR_OTHER);            /* does not make sense */
 835
 836     if (!CBIT_TEST(mask, id))
 837         return 0;
 838
 839     while ((e = getEntry(&pos))) {
 840         if (e->id == id)
 841             delAt(pos, count);
 842     }
 843
 844     CBIT_CLR(mask, id);
 845     assert(count);
 846     return count;
 847 }
 848
 849 /*
 850  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 851  * possible to iterate(search) and delete fields at the same time
 852  * NOTE: Does not update the header mask. Caller must follow up with
 853  * a call to refreshMask() if headers_deleted was incremented.
 854  */
 855 void
 856 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 857 {
 858     HttpHeaderEntry *e;
 859     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 860     e = (HttpHeaderEntry*)entries.items[pos];
 861     entries.items[pos] = NULL;
 862     /* decrement header length, allow for ": " and crlf */
 863     len -= e->name.size() + 2 + e->value.size() + 2;
 864     assert(len >= 0);
 865     delete e;
 866     ++headers_deleted;
 867 }
 868
 869 /*
 870  * Compacts the header storage
 871  */
 872 void
 873 HttpHeader::compact()
 874 {
 875     entries.prune(NULL);
 876 }
 877
 878 /*
 879  * Refreshes the header mask. Required after delAt() calls.
 880  */
 881 void
 882 HttpHeader::refreshMask()
 883 {
 884     httpHeaderMaskInit(&mask, 0);
 885     debugs(55, 7, "refreshing the mask in hdr " << this);
 886     HttpHeaderPos pos = HttpHeaderInitPos;
 887     while (HttpHeaderEntry *e = getEntry(&pos)) {
 888         CBIT_SET(mask, e->id);
 889     }
 890 }
 891
 892 /* appends an entry;
 893  * does not call e->clone() so one should not reuse "*e"
 894  */
 895 void
 896 HttpHeader::addEntry(HttpHeaderEntry * e)
 897 {
 898     assert(e);
 899     assert_eid(e->id);
 900     assert(e->name.size());
 901
 902     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 903
 904     if (CBIT_TEST(mask, e->id))
 905         ++ Headers[e->id].stat.repCount;
 906     else
 907         CBIT_SET(mask, e->id);
 908
 909     entries.push_back(e);
 910
 911     /* increment header length, allow for ": " and crlf */
 912     len += e->name.size() + 2 + e->value.size() + 2;
 913 }
 914
 915 /* inserts an entry;
 916  * does not call e->clone() so one should not reuse "*e"
 917  */
 918 void
 919 HttpHeader::insertEntry(HttpHeaderEntry * e)
 920 {
 921     assert(e);
 922     assert_eid(e->id);
 923
 924     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 925
 926     if (CBIT_TEST(mask, e->id))
 927         ++ Headers[e->id].stat.repCount;
 928     else
 929         CBIT_SET(mask, e->id);
 930
 931     entries.insert(e);
 932
 933     /* increment header length, allow for ": " and crlf */
 934     len += e->name.size() + 2 + e->value.size() + 2;
 935 }
 936
 937 bool
 938 HttpHeader::getList(http_hdr_type id, String *s) const
 939 {
 940     HttpHeaderEntry *e;
 941     HttpHeaderPos pos = HttpHeaderInitPos;
 942     debugs(55, 9, this << " joining for id " << id);
 943     /* only fields from ListHeaders array can be "listed" */
 944     assert(CBIT_TEST(ListHeadersMask, id));
 945
 946     if (!CBIT_TEST(mask, id))
 947         return false;
 948
 949     while ((e = getEntry(&pos))) {
 950         if (e->id == id)
 951             strListAdd(s, e->value.termedBuf(), ',');
 952     }
 953
 954     /*
 955      * note: we might get an empty (size==0) string if there was an "empty"
 956      * header. This results in an empty length String, which may have a NULL
 957      * buffer.
 958      */
 959     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 960     if (!s->size())
 961         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 962     else
 963         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 964
 965     return true;
 966 }
 967
 968 /* return a list of entries with the same id separated by ',' and ws */
 969 String
 970 HttpHeader::getList(http_hdr_type id) const
 971 {
 972     HttpHeaderEntry *e;
 973     HttpHeaderPos pos = HttpHeaderInitPos;
 974     debugs(55, 9, this << "joining for id " << id);
 975     /* only fields from ListHeaders array can be "listed" */
 976     assert(CBIT_TEST(ListHeadersMask, id));
 977
 978     if (!CBIT_TEST(mask, id))
 979         return String();
 980
 981     String s;
 982
 983     while ((e = getEntry(&pos))) {
 984         if (e->id == id)
 985             strListAdd(&s, e->value.termedBuf(), ',');
 986     }
 987
 988     /*
 989      * note: we might get an empty (size==0) string if there was an "empty"
 990      * header. This results in an empty length String, which may have a NULL
 991      * buffer.
 992      */
 993     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 994     if (!s.size())
 995         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 996     else
 997         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 998
 999     return s;
1000 }
1001
1002 /* return a string or list of entries with the same id separated by ',' and ws */
1003 String
1004 HttpHeader::getStrOrList(http_hdr_type id) const
1005 {
1006     HttpHeaderEntry *e;
1007
1008     if (CBIT_TEST(ListHeadersMask, id))
1009         return getList(id);
1010
1011     if ((e = findEntry(id)))
1012         return e->value;
1013
1014     return String();
1015 }
1016
1017 /*
1018  * Returns the value of the specified header.
1019  */
1020 String
1021 HttpHeader::getByName(const char *name) const
1022 {
1023     http_hdr_type id;
1024     HttpHeaderPos pos = HttpHeaderInitPos;
1025     HttpHeaderEntry *e;
1026
1027     assert(name);
1028
1029     /* First try the quick path */
1030     id = httpHeaderIdByNameDef(name, strlen(name));
1031
1032     if (id != -1)
1033         return getStrOrList(id);
1034
1035     String result;
1036
1037     /* Sorry, an unknown header name. Do linear search */
1038     while ((e = getEntry(&pos))) {
1039         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1040             strListAdd(&result, e->value.termedBuf(), ',');
1041         }
1042     }
1043
1044     return result;
1045 }
1046
1047 /*
1048  * Returns a the value of the specified list member, if any.
1049  */
1050 String
1051 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1052 {
1053     String header;
1054     const char *pos = NULL;
1055     const char *item;
1056     int ilen;
1057     int mlen = strlen(member);
1058
1059     assert(name);
1060
1061     header = getByName(name);
1062
1063     String result;
1064
1065     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1066         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1067             result.append(item + mlen + 1, ilen - mlen - 1);
1068             break;
1069         }
1070     }
1071
1072     return result;
1073 }
1074
1075 /*
1076  * returns a the value of the specified list member, if any.
1077  */
1078 String
1079 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1080 {
1081     String header;
1082     const char *pos = NULL;
1083     const char *item;
1084     int ilen;
1085     int mlen = strlen(member);
1086
1087     assert(id >= 0);
1088
1089     header = getStrOrList(id);
1090     String result;
1091
1092     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1093         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1094             result.append(item + mlen + 1, ilen - mlen - 1);
1095             break;
1096         }
1097     }
1098
1099     header.clean();
1100     return result;
1101 }
1102
1103 /* test if a field is present */
1104 int
1105 HttpHeader::has(http_hdr_type id) const
1106 {
1107     assert_eid(id);
1108     assert(id != HDR_OTHER);
1109     debugs(55, 9, this << " lookup for " << id);
1110     return CBIT_TEST(mask, id);
1111 }
1112
1113 void
1114 HttpHeader::putInt(http_hdr_type id, int number)
1115 {
1116     assert_eid(id);
1117     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1118     assert(number >= 0);
1119     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1120 }
1121
1122 void
1123 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1124 {
1125     assert_eid(id);
1126     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1127     assert(number >= 0);
1128     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1129 }
1130
1131 void
1132 HttpHeader::putTime(http_hdr_type id, time_t htime)
1133 {
1134     assert_eid(id);
1135     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1136     assert(htime >= 0);
1137     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1138 }
1139
1140 void
1141 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1142 {
1143     assert_eid(id);
1144     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1145     assert(htime >= 0);
1146     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1147 }
1148
1149 void
1150 HttpHeader::putStr(http_hdr_type id, const char *str)
1151 {
1152     assert_eid(id);
1153     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1154     assert(str);
1155     addEntry(new HttpHeaderEntry(id, NULL, str));
1156 }
1157
1158 void
1159 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1160 {
1161     assert(auth_scheme && realm);
1162     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1163 }
1164
1165 void
1166 HttpHeader::putCc(const HttpHdrCc * cc)
1167 {
1168     MemBuf mb;
1169     Packer p;
1170     assert(cc);
1171     /* remove old directives if any */
1172     delById(HDR_CACHE_CONTROL);
1173     /* pack into mb */
1174     mb.init();
1175     packerToMemInit(&p, &mb);
1176     cc->packInto(&p);
1177     /* put */
1178     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1179     /* cleanup */
1180     packerClean(&p);
1181     mb.clean();
1182 }
1183
1184 void
1185 HttpHeader::putContRange(const HttpHdrContRange * cr)
1186 {
1187     MemBuf mb;
1188     Packer p;
1189     assert(cr);
1190     /* remove old directives if any */
1191     delById(HDR_CONTENT_RANGE);
1192     /* pack into mb */
1193     mb.init();
1194     packerToMemInit(&p, &mb);
1195     httpHdrContRangePackInto(cr, &p);
1196     /* put */
1197     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1198     /* cleanup */
1199     packerClean(&p);
1200     mb.clean();
1201 }
1202
1203 void
1204 HttpHeader::putRange(const HttpHdrRange * range)
1205 {
1206     MemBuf mb;
1207     Packer p;
1208     assert(range);
1209     /* remove old directives if any */
1210     delById(HDR_RANGE);
1211     /* pack into mb */
1212     mb.init();
1213     packerToMemInit(&p, &mb);
1214     range->packInto(&p);
1215     /* put */
1216     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1217     /* cleanup */
1218     packerClean(&p);
1219     mb.clean();
1220 }
1221
1222 void
1223 HttpHeader::putSc(HttpHdrSc *sc)
1224 {
1225     MemBuf mb;
1226     Packer p;
1227     assert(sc);
1228     /* remove old directives if any */
1229     delById(HDR_SURROGATE_CONTROL);
1230     /* pack into mb */
1231     mb.init();
1232     packerToMemInit(&p, &mb);
1233     sc->packInto(&p);
1234     /* put */
1235     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1236     /* cleanup */
1237     packerClean(&p);
1238     mb.clean();
1239 }
1240
1241 void
1242 HttpHeader::putWarning(const int code, const char *const text)
1243 {
1244     char buf[512];
1245     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1246     putStr(HDR_WARNING, buf);
1247 }
1248
1249 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1250 void
1251 HttpHeader::putExt(const char *name, const char *value)
1252 {
1253     assert(name && value);
1254     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1255     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1256 }
1257
1258 int
1259 HttpHeader::getInt(http_hdr_type id) const
1260 {
1261     assert_eid(id);
1262     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1263     HttpHeaderEntry *e;
1264
1265     if ((e = findEntry(id)))
1266         return e->getInt();
1267
1268     return -1;
1269 }
1270
1271 int64_t
1272 HttpHeader::getInt64(http_hdr_type id) const
1273 {
1274     assert_eid(id);
1275     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1276     HttpHeaderEntry *e;
1277
1278     if ((e = findEntry(id)))
1279         return e->getInt64();
1280
1281     return -1;
1282 }
1283
1284 time_t
1285 HttpHeader::getTime(http_hdr_type id) const
1286 {
1287     HttpHeaderEntry *e;
1288     time_t value = -1;
1289     assert_eid(id);
1290     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1291
1292     if ((e = findEntry(id))) {
1293         value = parse_rfc1123(e->value.termedBuf());
1294         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1295     }
1296
1297     return value;
1298 }
1299
1300 /* sync with httpHeaderGetLastStr */
1301 const char *
1302 HttpHeader::getStr(http_hdr_type id) const
1303 {
1304     HttpHeaderEntry *e;
1305     assert_eid(id);
1306     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1307
1308     if ((e = findEntry(id))) {
1309         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1310         return e->value.termedBuf();
1311     }
1312
1313     return NULL;
1314 }
1315
1316 /* unusual */
1317 const char *
1318 HttpHeader::getLastStr(http_hdr_type id) const
1319 {
1320     HttpHeaderEntry *e;
1321     assert_eid(id);
1322     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1323
1324     if ((e = findLastEntry(id))) {
1325         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1326         return e->value.termedBuf();
1327     }
1328
1329     return NULL;
1330 }
1331
1332 HttpHdrCc *
1333 HttpHeader::getCc() const
1334 {
1335     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1336         return NULL;
1337     PROF_start(HttpHeader_getCc);
1338
1339     String s;
1340     getList(HDR_CACHE_CONTROL, &s);
1341
1342     HttpHdrCc *cc=new HttpHdrCc();
1343
1344     if (!cc->parse(s)) {
1345         delete cc;
1346         cc = NULL;
1347     }
1348
1349     ++ HttpHeaderStats[owner].ccParsedCount;
1350
1351     if (cc)
1352         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1353
1354     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1355
1356     PROF_stop(HttpHeader_getCc);
1357
1358     return cc;
1359 }
1360
1361 HttpHdrRange *
1362 HttpHeader::getRange() const
1363 {
1364     HttpHdrRange *r = NULL;
1365     HttpHeaderEntry *e;
1366     /* some clients will send "Request-Range" _and_ *matching* "Range"
1367      * who knows, some clients might send Request-Range only;
1368      * this "if" should work correctly in both cases;
1369      * hopefully no clients send mismatched headers! */
1370
1371     if ((e = findEntry(HDR_RANGE)) ||
1372             (e = findEntry(HDR_REQUEST_RANGE))) {
1373         r = HttpHdrRange::ParseCreate(&e->value);
1374         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1375     }
1376
1377     return r;
1378 }
1379
1380 HttpHdrSc *
1381 HttpHeader::getSc() const
1382 {
1383     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1384         return NULL;
1385
1386     String s;
1387
1388     (void) getList(HDR_SURROGATE_CONTROL, &s);
1389
1390     HttpHdrSc *sc = httpHdrScParseCreate(s);
1391
1392     ++ HttpHeaderStats[owner].ccParsedCount;
1393
1394     if (sc)
1395         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1396
1397     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1398
1399     return sc;
1400 }
1401
1402 HttpHdrContRange *
1403 HttpHeader::getContRange() const
1404 {
1405     HttpHdrContRange *cr = NULL;
1406     HttpHeaderEntry *e;
1407
1408     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1409         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1410         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1411     }
1412
1413     return cr;
1414 }
1415
1416 const char *
1417 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1418 {
1419     const char *field;
1420     int l;
1421     assert(auth_scheme);
1422     field = getStr(id);
1423
1424     if (!field)                 /* no authorization field */
1425         return NULL;
1426
1427     l = strlen(auth_scheme);
1428
1429     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1430         return NULL;
1431
1432     field += l;
1433
1434     if (!xisspace(*field))      /* wrong scheme */
1435         return NULL;
1436
1437     /* skip white space */
1438     for (; field && xisspace(*field); ++field);
1439
1440     if (!*field)                /* no authorization cookie */
1441         return NULL;
1442
1443     static char decodedAuthToken[8192];
1444     const int decodedLen = base64_decode(decodedAuthToken, sizeof(decodedAuthToken)-1, field);
1445     decodedAuthToken[decodedLen] = '\0';
1446     return decodedAuthToken;
1447 }
1448
1449 ETag
1450 HttpHeader::getETag(http_hdr_type id) const
1451 {
1452     ETag etag = {NULL, -1};
1453     HttpHeaderEntry *e;
1454     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1455
1456     if ((e = findEntry(id)))
1457         etagParseInit(&etag, e->value.termedBuf());
1458
1459     return etag;
1460 }
1461
1462 TimeOrTag
1463 HttpHeader::getTimeOrTag(http_hdr_type id) const
1464 {
1465     TimeOrTag tot;
1466     HttpHeaderEntry *e;
1467     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1468     memset(&tot, 0, sizeof(tot));
1469
1470     if ((e = findEntry(id))) {
1471         const char *str = e->value.termedBuf();
1472         /* try as an ETag */
1473
1474         if (etagParseInit(&tot.tag, str)) {
1475             tot.valid = tot.tag.str != NULL;
1476             tot.time = -1;
1477         } else {
1478             /* or maybe it is time? */
1479             tot.time = parse_rfc1123(str);
1480             tot.valid = tot.time >= 0;
1481             tot.tag.str = NULL;
1482         }
1483     }
1484
1485     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1486     return tot;
1487 }
1488
1489 /*
1490  * HttpHeaderEntry
1491  */
1492
1493 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1494 {
1495     assert_eid(anId);
1496     id = anId;
1497
1498     if (id != HDR_OTHER)
1499         name = Headers[id].name;
1500     else
1501         name = aName;
1502
1503     value = aValue;
1504
1505     ++ Headers[id].stat.aliveCount;
1506
1507     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1508 }
1509
1510 HttpHeaderEntry::~HttpHeaderEntry()
1511 {
1512     assert_eid(id);
1513     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1514     /* clean name if needed */
1515
1516     if (id == HDR_OTHER)
1517         name.clean();
1518
1519     value.clean();
1520
1521     assert(Headers[id].stat.aliveCount);
1522
1523     -- Headers[id].stat.aliveCount;
1524
1525     id = HDR_BAD_HDR;
1526 }
1527
1528 /* parses and inits header entry, returns true/false */
1529 HttpHeaderEntry *
1530 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1531 {
1532     /* note: name_start == field_start */
1533     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1534     int name_len = name_end ? name_end - field_start :0;
1535     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1536     /* note: value_end == field_end */
1537
1538     ++ HeaderEntryParsedCount;
1539
1540     /* do we have a valid field name within this field? */
1541
1542     if (!name_len || name_end > field_end)
1543         return NULL;
1544
1545     if (name_len > 65534) {
1546         /* String must be LESS THAN 64K and it adds a terminating NULL */
1547         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1548         return NULL;
1549     }
1550
1551     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1552         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1553                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1554
1555         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1556             --name_len;
1557
1558         if (!name_len)
1559             return NULL;
1560     }
1561
1562     /* now we know we can parse it */
1563
1564     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1565
1566     /* is it a "known" field? */
1567     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1568
1569     String name;
1570
1571     String value;
1572
1573     if (id < 0)
1574         id = HDR_OTHER;
1575
1576     assert_eid(id);
1577
1578     /* set field name */
1579     if (id == HDR_OTHER)
1580         name.limitInit(field_start, name_len);
1581     else
1582         name = Headers[id].name;
1583
1584     /* trim field value */
1585     while (value_start < field_end && xisspace(*value_start))
1586         ++value_start;
1587
1588     while (value_start < field_end && xisspace(field_end[-1]))
1589         --field_end;
1590
1591     if (field_end - value_start > 65534) {
1592         /* String must be LESS THAN 64K and it adds a terminating NULL */
1593         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1594
1595         if (id == HDR_OTHER)
1596             name.clean();
1597
1598         return NULL;
1599     }
1600
1601     /* set field value */
1602     value.limitInit(value_start, field_end - value_start);
1603
1604     ++ Headers[id].stat.seenCount;
1605
1606     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1607
1608     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1609 }
1610
1611 HttpHeaderEntry *
1612 HttpHeaderEntry::clone() const
1613 {
1614     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1615 }
1616
1617 void
1618 HttpHeaderEntry::packInto(Packer * p) const
1619 {
1620     assert(p);
1621     packerAppend(p, name.rawBuf(), name.size());
1622     packerAppend(p, ": ", 2);
1623     packerAppend(p, value.rawBuf(), value.size());
1624     packerAppend(p, "\r\n", 2);
1625 }
1626
1627 int
1628 HttpHeaderEntry::getInt() const
1629 {
1630     assert_eid (id);
1631     assert (Headers[id].type == ftInt);
1632     int val = -1;
1633     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1634     httpHeaderNoteParsedEntry(id, value, !ok);
1635     /* XXX: Should we check ok - ie
1636      * return ok ? -1 : value;
1637      */
1638     return val;
1639 }
1640
1641 int64_t
1642 HttpHeaderEntry::getInt64() const
1643 {
1644     assert_eid (id);
1645     assert (Headers[id].type == ftInt64);
1646     int64_t val = -1;
1647     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1648     httpHeaderNoteParsedEntry(id, value, !ok);
1649     /* XXX: Should we check ok - ie
1650      * return ok ? -1 : value;
1651      */
1652     return val;
1653 }
1654
1655 static void
1656 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1657 {
1658     ++ Headers[id].stat.parsCount;
1659
1660     if (error) {
1661         ++ Headers[id].stat.errCount;
1662         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name << ": " << context << "'");
1663     }
1664 }
1665
1666 /*
1667  * Reports
1668  */
1669
1670 /* tmp variable used to pass stat info to dumpers */
1671 extern const HttpHeaderStat *dump_stat;         /* argh! */
1672 const HttpHeaderStat *dump_stat = NULL;
1673
1674 void
1675 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1676 {
1677     const int id = (int) val;
1678     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1679     const char *name = valid_id ? Headers[id].name.termedBuf() : "INVALID";
1680     int visible = count > 0;
1681     /* for entries with zero count, list only those that belong to current type of message */
1682
1683     if (!visible && valid_id && dump_stat->owner_mask)
1684         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1685
1686     if (visible)
1687         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1688                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1689 }
1690
1691 static void
1692 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1693 {
1694     if (count)
1695         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1696                           idx, (int) val, count,
1697                           xpercent(count, dump_stat->destroyedCount));
1698 }
1699
1700 static void
1701 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1702 {
1703     assert(hs && e);
1704
1705     dump_stat = hs;
1706     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1707     storeAppendPrintf(e, "\nField type distribution\n");
1708     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1709                       "id", "name", "count", "#/header");
1710     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1711     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1712     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1713                       "id", "name", "count", "#/cc_field");
1714     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1715     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1716     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1717                       "id", "name", "count", "#/sc_field");
1718     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1719     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1720     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1721                       "id", "#flds", "count", "%total");
1722     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1723     dump_stat = NULL;
1724 }
1725
1726 void
1727 httpHeaderStoreReport(StoreEntry * e)
1728 {
1729     int i;
1730     http_hdr_type ht;
1731     assert(e);
1732
1733     HttpHeaderStats[0].parsedCount =
1734         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1735     HttpHeaderStats[0].ccParsedCount =
1736         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1737     HttpHeaderStats[0].destroyedCount =
1738         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1739     HttpHeaderStats[0].busyDestroyedCount =
1740         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1741
1742     for (i = 1; i < HttpHeaderStatCount; ++i) {
1743         httpHeaderStatDump(HttpHeaderStats + i, e);
1744         storeAppendPrintf(e, "%s\n", "<br>");
1745     }
1746
1747     /* field stats for all messages */
1748     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1749
1750     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1751                       "id", "name", "#alive", "%err", "%repeat");
1752
1753     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1754         HttpHeaderFieldInfo *f = Headers + ht;
1755         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1756                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1757                           xpercent(f->stat.errCount, f->stat.parsCount),
1758                           xpercent(f->stat.repCount, f->stat.seenCount));
1759     }
1760
1761     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1762                       HttpHeaderStats[hoRequest].parsedCount,
1763                       HttpHeaderStats[hoReply].parsedCount,
1764                       HttpHeaderStats[0].parsedCount);
1765     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1766 }
1767
1768 http_hdr_type
1769 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1770 {
1771     if (name_len > 0) {
1772         for (int i = 0; i < end; ++i) {
1773             if (name_len != info[i].name.size())
1774                 continue;
1775
1776             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1777                 return info[i].id;
1778         }
1779     }
1780
1781     return HDR_BAD_HDR;
1782 }
1783
1784 http_hdr_type
1785 httpHeaderIdByNameDef(const char *name, int name_len)
1786 {
1787     if (!Headers)
1788         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1789
1790     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1791 }
1792
1793 const char *
1794 httpHeaderNameById(int id)
1795 {
1796     if (!Headers)
1797         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1798
1799     assert(id >= 0 && id < HDR_ENUM_END);
1800
1801     return Headers[id].name.termedBuf();
1802 }
1803
1804 int
1805 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1806 {
1807     int result = 0;
1808     const char *pos = NULL;
1809     const char *item;
1810     int ilen;
1811     int mlen = strlen(member);
1812
1813     assert(id >= 0);
1814
1815     String header (getStrOrList(id));
1816
1817     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1818         if (strncasecmp(item, member, mlen) == 0
1819                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1820             result = 1;
1821             break;
1822         }
1823     }
1824
1825     return result;
1826 }
1827
1828 int
1829 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1830 {
1831     int result = 0;
1832     const char *pos = NULL;
1833     const char *item;
1834     int ilen;
1835     int mlen = strlen(member);
1836
1837     assert(name);
1838
1839     String header (getByName(name));
1840
1841     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1842         if (strncasecmp(item, member, mlen) == 0
1843                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1844             result = 1;
1845             break;
1846         }
1847     }
1848
1849     return result;
1850 }
1851
1852 void
1853 HttpHeader::removeHopByHopEntries()
1854 {
1855     removeConnectionHeaderEntries();
1856
1857     const HttpHeaderEntry *e;
1858     HttpHeaderPos pos = HttpHeaderInitPos;
1859     int headers_deleted = 0;
1860     while ((e = getEntry(&pos))) {
1861         int id = e->id;
1862         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1863             delAt(pos, headers_deleted);
1864             CBIT_CLR(mask, id);
1865         }
1866     }
1867 }
1868
1869 void
1870 HttpHeader::removeConnectionHeaderEntries()
1871 {
1872     if (has(HDR_CONNECTION)) {
1873         /* anything that matches Connection list member will be deleted */
1874         String strConnection;
1875
1876         (void) getList(HDR_CONNECTION, &strConnection);
1877         const HttpHeaderEntry *e;
1878         HttpHeaderPos pos = HttpHeaderInitPos;
1879         /*
1880          * think: on-average-best nesting of the two loops (hdrEntry
1881          * and strListItem) @?@
1882          */
1883         /*
1884          * maybe we should delete standard stuff ("keep-alive","close")
1885          * from strConnection first?
1886          */
1887
1888         int headers_deleted = 0;
1889         while ((e = getEntry(&pos))) {
1890             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1891                 delAt(pos, headers_deleted);
1892         }
1893         if (headers_deleted)
1894             refreshMask();
1895     }
1896 }