src/HttpHeader.cc

   1 /*
   2  * DEBUG: section 55    HTTP Header
   3  * AUTHOR: Alex Rousskov
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  */
  31
  32 #include "squid.h"
  33 #include "base64.h"
  34 #include "globals.h"
  35 #include "HttpHdrCc.h"
  36 #include "HttpHdrContRange.h"
  37 #include "HttpHdrSc.h"
  38 #include "HttpHeader.h"
  39 #include "HttpHeaderFieldInfo.h"
  40 #include "HttpHeaderStat.h"
  41 #include "HttpHeaderTools.h"
  42 #include "MemBuf.h"
  43 #include "mgr/Registration.h"
  44 #include "profiler/Profiler.h"
  45 #include "rfc1123.h"
  46 #include "StatHist.h"
  47 #include "Store.h"
  48 #include "StrList.h"
  49 #include "SquidConfig.h"
  50 #include "SquidString.h"
  51 #include "TimeOrTag.h"
  52
  53 /*
  54  * On naming conventions:
  55  *
  56  * HTTP/1.1 defines message-header as
  57  *
  58  * message-header = field-name ":" [ field-value ] CRLF
  59  * field-name     = token
  60  * field-value    = *( field-content | LWS )
  61  *
  62  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  63  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  64  *
  65  * HttpHeader is an object that represents all message-headers in a message.
  66  * HttpHeader does not manage start-line.
  67  *
  68  * HttpHeader is implemented as a collection of header "entries".
  69  * An entry is a (field_id, field_name, field_value) triplet.
  70  */
  71
  72 /*
  73  * local constants and vars
  74  */
  75
  76 /*
  77  * A table with major attributes for every known field.
  78  * We calculate name lengths and reorganize this array on start up.
  79  * After reorganization, field id can be used as an index to the table.
  80  */
  81 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  82     {"Accept", HDR_ACCEPT, ftStr},
  83
  84     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  85     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  86     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  87     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  88     {"Age", HDR_AGE, ftInt},
  89     {"Allow", HDR_ALLOW, ftStr},
  90     {"Authorization", HDR_AUTHORIZATION, ftStr},        /* for now */
  91     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  92     {"Connection", HDR_CONNECTION, ftStr},
  93     {"Content-Base", HDR_CONTENT_BASE, ftStr},
  94     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
  95     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  96     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  97     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  98     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  99     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
 100     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
 101     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
 102     {"Cookie", HDR_COOKIE, ftStr},
 103     {"Cookie2", HDR_COOKIE2, ftStr},
 104     {"Date", HDR_DATE, ftDate_1123},
 105     {"ETag", HDR_ETAG, ftETag},
 106     {"Expect", HDR_EXPECT, ftStr},
 107     {"Expires", HDR_EXPIRES, ftDate_1123},
 108     {"From", HDR_FROM, ftStr},
 109     {"Host", HDR_HOST, ftStr},
 110     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 111     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 112     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},        /* for now */
 113     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 114     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 115     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 116     {"Link", HDR_LINK, ftStr},
 117     {"Location", HDR_LOCATION, ftStr},
 118     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 119     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 120     {"Negotiate", HDR_NEGOTIATE, ftStr},
 121     {"Origin", HDR_ORIGIN, ftStr},
 122     {"Pragma", HDR_PRAGMA, ftStr},
 123     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 124     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 125     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 126     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 127     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 128     {"Public", HDR_PUBLIC, ftStr},
 129     {"Range", HDR_RANGE, ftPRange},
 130     {"Referer", HDR_REFERER, ftStr},
 131     {"Request-Range", HDR_REQUEST_RANGE, ftPRange},     /* usually matches HDR_RANGE */
 132     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!) */
 133     {"Server", HDR_SERVER, ftStr},
 134     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 135     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 136     {"TE", HDR_TE, ftStr},
 137     {"Title", HDR_TITLE, ftStr},
 138     {"Trailer", HDR_TRAILER, ftStr},
 139     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 140     {"Translate", HDR_TRANSLATE, ftStr},        /* for now. may need to crop */
 141     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 142     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 143     {"User-Agent", HDR_USER_AGENT, ftStr},
 144     {"Vary", HDR_VARY, ftStr},  /* for now */
 145     {"Via", HDR_VIA, ftStr},    /* for now */
 146     {"Warning", HDR_WARNING, ftStr},    /* for now */
 147     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 148     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 149     {"X-Cache", HDR_X_CACHE, ftStr},
 150     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 151     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 152     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 153     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 154 #if X_ACCELERATOR_VARY
 155     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 156 #endif
 157 #if USE_ADAPTATION
 158     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 159 #endif
 160     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 161     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 162     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 163     {"Other:", HDR_OTHER, ftStr}        /* ':' will not allow matches */
 164 };
 165
 166 static HttpHeaderFieldInfo *Headers = NULL;
 167
 168 http_hdr_type &operator++ (http_hdr_type &aHeader)
 169 {
 170     int tmp = (int)aHeader;
 171     aHeader = (http_hdr_type)(++tmp);
 172     return aHeader;
 173 }
 174
 175 /*
 176  * headers with field values defined as #(values) in HTTP/1.1
 177  * Headers that are currently not recognized, are commented out.
 178  */
 179 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 180 static http_hdr_type ListHeadersArr[] = {
 181     HDR_ACCEPT,
 182     HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 183     HDR_ACCEPT_RANGES, HDR_ALLOW,
 184     HDR_CACHE_CONTROL,
 185     HDR_CONTENT_ENCODING,
 186     HDR_CONTENT_LANGUAGE,
 187     HDR_CONNECTION,
 188     HDR_EXPECT,
 189     HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 190     HDR_LINK, HDR_PRAGMA,
 191     HDR_PROXY_CONNECTION,
 192     HDR_PROXY_SUPPORT,
 193     HDR_TRANSFER_ENCODING,
 194     HDR_UPGRADE,
 195     HDR_VARY,
 196     HDR_VIA,
 197     HDR_WARNING,
 198     HDR_WWW_AUTHENTICATE,
 199     HDR_AUTHENTICATION_INFO,
 200     HDR_PROXY_AUTHENTICATION_INFO,
 201     /* HDR_TE, HDR_TRAILER */
 202 #if X_ACCELERATOR_VARY
 203     HDR_X_ACCELERATOR_VARY,
 204 #endif
 205 #if USE_ADAPTATION
 206     HDR_X_NEXT_SERVICES,
 207 #endif
 208     HDR_SURROGATE_CAPABILITY,
 209     HDR_SURROGATE_CONTROL,
 210     HDR_X_FORWARDED_FOR
 211 };
 212
 213 /* general-headers */
 214 static http_hdr_type GeneralHeadersArr[] = {
 215     HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 216     HDR_TRANSFER_ENCODING,
 217     HDR_UPGRADE,
 218     /* HDR_TRAILER, */
 219     HDR_VIA,
 220 };
 221
 222 /* entity-headers */
 223 static http_hdr_type EntityHeadersArr[] = {
 224     HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 225     HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 226     HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 227     HDR_OTHER
 228 };
 229
 230 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 231 static http_hdr_type ReplyHeadersArr[] = {
 232     HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 233     HDR_ACCEPT_RANGES, HDR_AGE,
 234     HDR_LOCATION, HDR_MAX_FORWARDS,
 235     HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_SET_COOKIE2,
 236     HDR_ORIGIN,
 237     HDR_VARY,
 238     HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 239     HDR_X_CACHE_LOOKUP,
 240     HDR_X_REQUEST_URI,
 241 #if X_ACCELERATOR_VARY
 242     HDR_X_ACCELERATOR_VARY,
 243 #endif
 244 #if USE_ADAPTATION
 245     HDR_X_NEXT_SERVICES,
 246 #endif
 247     HDR_X_SQUID_ERROR,
 248     HDR_SURROGATE_CONTROL
 249 };
 250
 251 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 252 static http_hdr_type RequestHeadersArr[] = {
 253     HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 254     HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 255     HDR_IF_RANGE, HDR_MAX_FORWARDS,
 256     HDR_ORIGIN,
 257     HDR_PROXY_CONNECTION,
 258     HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 259     HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 260 };
 261
 262 static HttpHeaderMask HopByHopHeadersMask;
 263 static http_hdr_type HopByHopHeadersArr[] = {
 264     HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 265     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 266 };
 267
 268 /* header accounting */
 269 static HttpHeaderStat HttpHeaderStats[] = {
 270     {"all"},
 271 #if USE_HTCP
 272     {"HTCP reply"},
 273 #endif
 274     {"request"},
 275     {"reply"}
 276 };
 277 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 278
 279 static int HeaderEntryParsedCount = 0;
 280
 281 /*
 282  * forward declarations and local routines
 283  */
 284
 285 class StoreEntry;
 286 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 287
 288 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 289
 290 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 291 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 292
 293 /** store report about current header usage and other stats */
 294 static void httpHeaderStoreReport(StoreEntry * e);
 295
 296 /*
 297  * Module initialization routines
 298  */
 299
 300 static void
 301 httpHeaderRegisterWithCacheManager(void)
 302 {
 303     Mgr::RegisterAction("http_headers",
 304                         "HTTP Header Statistics",
 305                         httpHeaderStoreReport, 0, 1);
 306 }
 307
 308 void
 309 httpHeaderInitModule(void)
 310 {
 311     int i;
 312     /* check that we have enough space for masks */
 313     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 314     /* all headers must be described */
 315     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 316
 317     if (!Headers)
 318         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 319
 320     /* create masks */
 321     httpHeaderMaskInit(&ListHeadersMask, 0);
 322
 323     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 324
 325     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 326
 327     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 328
 329     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 330
 331     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 332
 333     httpHeaderMaskInit(&RequestHeadersMask, 0);
 334
 335     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 336
 337     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 338
 339     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 340
 341     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 342
 343     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 344
 345     /* init header stats */
 346     assert(HttpHeaderStatCount == hoReply + 1);
 347
 348     for (i = 0; i < HttpHeaderStatCount; ++i)
 349         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 350
 351     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 352
 353     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 354
 355 #if USE_HTCP
 356
 357     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 358
 359 #endif
 360     /* init dependent modules */
 361     httpHdrCcInitModule();
 362
 363     httpHdrScInitModule();
 364
 365     httpHeaderRegisterWithCacheManager();
 366 }
 367
 368 void
 369 httpHeaderCleanModule(void)
 370 {
 371     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 372     Headers = NULL;
 373     httpHdrCcCleanModule();
 374     httpHdrScCleanModule();
 375 }
 376
 377 static void
 378 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 379 {
 380     assert(hs);
 381     assert(label);
 382     memset(hs, 0, sizeof(HttpHeaderStat));
 383     hs->label = label;
 384     hs->hdrUCountDistr.enumInit(32);    /* not a real enum */
 385     hs->fieldTypeDistr.enumInit(HDR_ENUM_END);
 386     hs->ccTypeDistr.enumInit(CC_ENUM_END);
 387     hs->scTypeDistr.enumInit(SC_ENUM_END);
 388 }
 389
 390 /*
 391  * HttpHeader Implementation
 392  */
 393
 394 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 395 {
 396     httpHeaderMaskInit(&mask, 0);
 397 }
 398
 399 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 400 {
 401     assert(anOwner > hoNone && anOwner < hoEnd);
 402     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 403     httpHeaderMaskInit(&mask, 0);
 404 }
 405
 406 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 407 {
 408     httpHeaderMaskInit(&mask, 0);
 409     update(&other, NULL); // will update the mask as well
 410 }
 411
 412 HttpHeader::~HttpHeader()
 413 {
 414     clean();
 415 }
 416
 417 HttpHeader &
 418 HttpHeader::operator =(const HttpHeader &other)
 419 {
 420     if (this != &other) {
 421         // we do not really care, but the caller probably does
 422         assert(owner == other.owner);
 423         clean();
 424         update(&other, NULL); // will update the mask as well
 425         len = other.len;
 426     }
 427     return *this;
 428 }
 429
 430 void
 431 HttpHeader::clean()
 432 {
 433     HttpHeaderPos pos = HttpHeaderInitPos;
 434     HttpHeaderEntry *e;
 435
 436     assert(owner > hoNone && owner < hoEnd);
 437     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 438
 439     PROF_start(HttpHeaderClean);
 440
 441     /*
 442      * An unfortunate bug.  The entries array is initialized
 443      * such that count is set to zero.  httpHeaderClean() seems to
 444      * be called both when 'hdr' is created, and destroyed.  Thus,
 445      * we accumulate a large number of zero counts for 'hdr' before
 446      * it is ever used.  Can't think of a good way to fix it, except
 447      * adding a state variable that indicates whether or not 'hdr'
 448      * has been used.  As a hack, just never count zero-sized header
 449      * arrays.
 450      */
 451
 452     if (owner <= hoReply) {
 453         if (0 != entries.count)
 454             HttpHeaderStats[owner].hdrUCountDistr.count(entries.count);
 455
 456         ++ HttpHeaderStats[owner].destroyedCount;
 457
 458         HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 459
 460         while ((e = getEntry(&pos))) {
 461             /* tmp hack to try to avoid coredumps */
 462
 463             if (e->id < 0 || e->id >= HDR_ENUM_END) {
 464                 debugs(55, DBG_CRITICAL, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 465             } else {
 466                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 467                 /* yes, this deletion leaves us in an inconsistent state */
 468                 delete e;
 469             }
 470         }
 471     } // if (owner <= hoReply)
 472     entries.clean();
 473     httpHeaderMaskInit(&mask, 0);
 474     len = 0;
 475     PROF_stop(HttpHeaderClean);
 476 }
 477
 478 /* append entries (also see httpHeaderUpdate) */
 479 void
 480 HttpHeader::append(const HttpHeader * src)
 481 {
 482     const HttpHeaderEntry *e;
 483     HttpHeaderPos pos = HttpHeaderInitPos;
 484     assert(src);
 485     assert(src != this);
 486     debugs(55, 7, "appending hdr: " << this << " += " << src);
 487
 488     while ((e = src->getEntry(&pos))) {
 489         addEntry(e->clone());
 490     }
 491 }
 492
 493 /* use fresh entries to replace old ones */
 494 void
 495 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 496 {
 497     assert (old);
 498     old->update (fresh, denied_mask);
 499 }
 500
 501 void
 502 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 503 {
 504     const HttpHeaderEntry *e;
 505     HttpHeaderPos pos = HttpHeaderInitPos;
 506     assert(fresh);
 507     assert(this != fresh);
 508
 509     while ((e = fresh->getEntry(&pos))) {
 510         /* deny bad guys (ok to check for HDR_OTHER) here */
 511
 512         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 513             continue;
 514
 515         if (e->id != HDR_OTHER)
 516             delById(e->id);
 517         else
 518             delByName(e->name.termedBuf());
 519     }
 520
 521     pos = HttpHeaderInitPos;
 522     while ((e = fresh->getEntry(&pos))) {
 523         /* deny bad guys (ok to check for HDR_OTHER) here */
 524
 525         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 526             continue;
 527
 528         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 529
 530         addEntry(e->clone());
 531     }
 532 }
 533
 534 /* just handy in parsing: resets and returns false */
 535 int
 536 HttpHeader::reset()
 537 {
 538     clean();
 539     return 0;
 540 }
 541
 542 int
 543 HttpHeader::parse(const char *header_start, const char *header_end)
 544 {
 545     const char *field_ptr = header_start;
 546     HttpHeaderEntry *e, *e2;
 547
 548     PROF_start(HttpHeaderParse);
 549
 550     assert(header_start && header_end);
 551     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 552     ++ HttpHeaderStats[owner].parsedCount;
 553
 554     char *nulpos;
 555     if ((nulpos = (char*)memchr(header_start, '\0', header_end - header_start))) {
 556         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 557                getStringPrefix(header_start, nulpos) << "}\nNULL\n{" << getStringPrefix(nulpos+1, header_end));
 558         goto reset;
 559     }
 560
 561     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 562      * continuation lines start with a (single) space or tab */
 563     while (field_ptr < header_end) {
 564         const char *field_start = field_ptr;
 565         const char *field_end;
 566
 567         do {
 568             const char *this_line = field_ptr;
 569             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 570
 571             if (!field_ptr)
 572                 goto reset;     /* missing <LF> */
 573
 574             field_end = field_ptr;
 575
 576             ++field_ptr;        /* Move to next line */
 577
 578             if (field_end > this_line && field_end[-1] == '\r') {
 579                 --field_end;    /* Ignore CR LF */
 580
 581                 if (owner == hoRequest && field_end > this_line) {
 582                     bool cr_only = true;
 583                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 584                         if (*p != '\r')
 585                             cr_only = false;
 586                     }
 587                     if (cr_only) {
 588                         debugs(55, DBG_IMPORTANT, "WARNING: Rejecting HTTP request with a CR+ "
 589                                "header field to prevent request smuggling attacks: {" <<
 590                                getStringPrefix(header_start, header_end) << "}");
 591                         goto reset;
 592                     }
 593                 }
 594             }
 595
 596             /* Barf on stray CR characters */
 597             if (memchr(this_line, '\r', field_end - this_line)) {
 598                 debugs(55, DBG_IMPORTANT, "WARNING: suspicious CR characters in HTTP header {" <<
 599                        getStringPrefix(field_start, field_end) << "}");
 600
 601                 if (Config.onoff.relaxed_header_parser) {
 602                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 603
 604                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 605                         *p = ' ';
 606                         ++p;
 607                     }
 608                 } else
 609                     goto reset;
 610             }
 611
 612             if (this_line + 1 == field_end && this_line > field_start) {
 613                 debugs(55, DBG_IMPORTANT, "WARNING: Blank continuation line in HTTP header {" <<
 614                        getStringPrefix(header_start, header_end) << "}");
 615                 goto reset;
 616             }
 617         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 618
 619         if (field_start == field_end) {
 620             if (field_ptr < header_end) {
 621                 debugs(55, DBG_IMPORTANT, "WARNING: unparseable HTTP header field near {" <<
 622                        getStringPrefix(field_start, header_end) << "}");
 623                 goto reset;
 624             }
 625
 626             break;              /* terminating blank line */
 627         }
 628
 629         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 630             debugs(55, DBG_IMPORTANT, "WARNING: unparseable HTTP header field {" <<
 631                    getStringPrefix(field_start, field_end) << "}");
 632             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 633                    " in {" << getStringPrefix(header_start, header_end) << "}");
 634
 635             if (Config.onoff.relaxed_header_parser)
 636                 continue;
 637             else
 638                 goto reset;
 639         }
 640
 641         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 642 //            if (e->value.cmp(e2->value.termedBuf()) != 0) {
 643             if (e->value != e2->value) {
 644                 int64_t l1, l2;
 645                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 646                        "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 647
 648                 if (!Config.onoff.relaxed_header_parser) {
 649                     delete e;
 650                     goto reset;
 651                 }
 652
 653                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 654                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e->value << "'");
 655                     delete e;
 656                     continue;
 657                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 658                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e2->value << "'");
 659                     delById(e2->id);
 660                 } else if (l1 > l2) {
 661                     delById(e2->id);
 662                 } else {
 663                     delete e;
 664                     continue;
 665                 }
 666             } else {
 667                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 668                        "NOTICE: found double content-length header");
 669
 670                 if (Config.onoff.relaxed_header_parser) {
 671                     delete e;
 672                     continue;
 673                 } else {
 674                     delete e;
 675                     goto reset;
 676                 }
 677             }
 678         }
 679
 680         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 681             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 682                    "WARNING: found whitespace in HTTP header name {" <<
 683                    getStringPrefix(field_start, field_end) << "}");
 684
 685             if (!Config.onoff.relaxed_header_parser) {
 686                 delete e;
 687                 goto reset;
 688             }
 689         }
 690
 691         addEntry(e);
 692     }
 693
 694     if (chunked()) {
 695         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 696         delById(HDR_CONTENT_LENGTH);
 697     }
 698
 699     PROF_stop(HttpHeaderParse);
 700     return 1;                   /* even if no fields where found, it is a valid header */
 701 reset:
 702     PROF_stop(HttpHeaderParse);
 703     return reset();
 704 }
 705
 706 /* packs all the entries using supplied packer */
 707 void
 708 HttpHeader::packInto(Packer * p, bool mask_sensitive_info) const
 709 {
 710     HttpHeaderPos pos = HttpHeaderInitPos;
 711     const HttpHeaderEntry *e;
 712     assert(p);
 713     debugs(55, 7, "packing hdr: (" << this << ")");
 714     /* pack all entries one by one */
 715     while ((e = getEntry(&pos))) {
 716         if (!mask_sensitive_info) {
 717             e->packInto(p);
 718             continue;
 719         }
 720         switch (e->id) {
 721         case HDR_AUTHORIZATION:
 722         case HDR_PROXY_AUTHORIZATION:
 723             packerAppend(p, e->name.rawBuf(), e->name.size());
 724             packerAppend(p, ": ** NOT DISPLAYED **\r\n", 23);
 725             break;
 726         default:
 727             e->packInto(p);
 728             break;
 729         }
 730     }
 731     /* Pack in the "special" entries */
 732
 733     /* Cache-Control */
 734 }
 735
 736 /* returns next valid entry */
 737 HttpHeaderEntry *
 738 HttpHeader::getEntry(HttpHeaderPos * pos) const
 739 {
 740     assert(pos);
 741     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 742
 743     for (++(*pos); *pos < (ssize_t)entries.count; ++(*pos)) {
 744         if (entries.items[*pos])
 745             return (HttpHeaderEntry*)entries.items[*pos];
 746     }
 747
 748     return NULL;
 749 }
 750
 751 /*
 752  * returns a pointer to a specified entry if any
 753  * note that we return one entry so it does not make much sense to ask for
 754  * "list" headers
 755  */
 756 HttpHeaderEntry *
 757 HttpHeader::findEntry(http_hdr_type id) const
 758 {
 759     HttpHeaderPos pos = HttpHeaderInitPos;
 760     HttpHeaderEntry *e;
 761     assert_eid(id);
 762     assert(!CBIT_TEST(ListHeadersMask, id));
 763
 764     /* check mask first */
 765
 766     if (!CBIT_TEST(mask, id))
 767         return NULL;
 768
 769     /* looks like we must have it, do linear search */
 770     while ((e = getEntry(&pos))) {
 771         if (e->id == id)
 772             return e;
 773     }
 774
 775     /* hm.. we thought it was there, but it was not found */
 776     assert(0);
 777
 778     return NULL;                /* not reached */
 779 }
 780
 781 /*
 782  * same as httpHeaderFindEntry
 783  */
 784 HttpHeaderEntry *
 785 HttpHeader::findLastEntry(http_hdr_type id) const
 786 {
 787     HttpHeaderPos pos = HttpHeaderInitPos;
 788     HttpHeaderEntry *e;
 789     HttpHeaderEntry *result = NULL;
 790     assert_eid(id);
 791     assert(!CBIT_TEST(ListHeadersMask, id));
 792
 793     /* check mask first */
 794
 795     if (!CBIT_TEST(mask, id))
 796         return NULL;
 797
 798     /* looks like we must have it, do linear search */
 799     while ((e = getEntry(&pos))) {
 800         if (e->id == id)
 801             result = e;
 802     }
 803
 804     assert(result);             /* must be there! */
 805     return result;
 806 }
 807
 808 /*
 809  * deletes all fields with a given name if any, returns #fields deleted;
 810  */
 811 int
 812 HttpHeader::delByName(const char *name)
 813 {
 814     int count = 0;
 815     HttpHeaderPos pos = HttpHeaderInitPos;
 816     HttpHeaderEntry *e;
 817     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 818     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 819
 820     while ((e = getEntry(&pos))) {
 821         if (!e->name.caseCmp(name))
 822             delAt(pos, count);
 823         else
 824             CBIT_SET(mask, e->id);
 825     }
 826
 827     return count;
 828 }
 829
 830 /* deletes all entries with a given id, returns the #entries deleted */
 831 int
 832 HttpHeader::delById(http_hdr_type id)
 833 {
 834     int count = 0;
 835     HttpHeaderPos pos = HttpHeaderInitPos;
 836     HttpHeaderEntry *e;
 837     debugs(55, 8, this << " del-by-id " << id);
 838     assert_eid(id);
 839     assert(id != HDR_OTHER);            /* does not make sense */
 840
 841     if (!CBIT_TEST(mask, id))
 842         return 0;
 843
 844     while ((e = getEntry(&pos))) {
 845         if (e->id == id)
 846             delAt(pos, count);
 847     }
 848
 849     CBIT_CLR(mask, id);
 850     assert(count);
 851     return count;
 852 }
 853
 854 /*
 855  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 856  * possible to iterate(search) and delete fields at the same time
 857  * NOTE: Does not update the header mask. Caller must follow up with
 858  * a call to refreshMask() if headers_deleted was incremented.
 859  */
 860 void
 861 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 862 {
 863     HttpHeaderEntry *e;
 864     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 865     e = (HttpHeaderEntry*)entries.items[pos];
 866     entries.items[pos] = NULL;
 867     /* decrement header length, allow for ": " and crlf */
 868     len -= e->name.size() + 2 + e->value.size() + 2;
 869     assert(len >= 0);
 870     delete e;
 871     ++headers_deleted;
 872 }
 873
 874 /*
 875  * Compacts the header storage
 876  */
 877 void
 878 HttpHeader::compact()
 879 {
 880     entries.prune(NULL);
 881 }
 882
 883 /*
 884  * Refreshes the header mask. Required after delAt() calls.
 885  */
 886 void
 887 HttpHeader::refreshMask()
 888 {
 889     httpHeaderMaskInit(&mask, 0);
 890     debugs(55, 7, "refreshing the mask in hdr " << this);
 891     HttpHeaderPos pos = HttpHeaderInitPos;
 892     while (HttpHeaderEntry *e = getEntry(&pos)) {
 893         CBIT_SET(mask, e->id);
 894     }
 895 }
 896
 897 /* appends an entry;
 898  * does not call e->clone() so one should not reuse "*e"
 899  */
 900 void
 901 HttpHeader::addEntry(HttpHeaderEntry * e)
 902 {
 903     assert(e);
 904     assert_eid(e->id);
 905     assert(e->name.size());
 906
 907     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 908
 909     if (CBIT_TEST(mask, e->id))
 910         ++ Headers[e->id].stat.repCount;
 911     else
 912         CBIT_SET(mask, e->id);
 913
 914     entries.push_back(e);
 915
 916     /* increment header length, allow for ": " and crlf */
 917     len += e->name.size() + 2 + e->value.size() + 2;
 918 }
 919
 920 /* inserts an entry;
 921  * does not call e->clone() so one should not reuse "*e"
 922  */
 923 void
 924 HttpHeader::insertEntry(HttpHeaderEntry * e)
 925 {
 926     assert(e);
 927     assert_eid(e->id);
 928
 929     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 930
 931     if (CBIT_TEST(mask, e->id))
 932         ++ Headers[e->id].stat.repCount;
 933     else
 934         CBIT_SET(mask, e->id);
 935
 936     entries.insert(e);
 937
 938     /* increment header length, allow for ": " and crlf */
 939     len += e->name.size() + 2 + e->value.size() + 2;
 940 }
 941
 942 bool
 943 HttpHeader::getList(http_hdr_type id, String *s) const
 944 {
 945     HttpHeaderEntry *e;
 946     HttpHeaderPos pos = HttpHeaderInitPos;
 947     debugs(55, 9, this << " joining for id " << id);
 948     /* only fields from ListHeaders array can be "listed" */
 949     assert(CBIT_TEST(ListHeadersMask, id));
 950
 951     if (!CBIT_TEST(mask, id))
 952         return false;
 953
 954     while ((e = getEntry(&pos))) {
 955         if (e->id == id)
 956             strListAdd(s, e->value.termedBuf(), ',');
 957     }
 958
 959     /*
 960      * note: we might get an empty (size==0) string if there was an "empty"
 961      * header. This results in an empty length String, which may have a NULL
 962      * buffer.
 963      */
 964     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 965     if (!s->size())
 966         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 967     else
 968         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 969
 970     return true;
 971 }
 972
 973 /* return a list of entries with the same id separated by ',' and ws */
 974 String
 975 HttpHeader::getList(http_hdr_type id) const
 976 {
 977     HttpHeaderEntry *e;
 978     HttpHeaderPos pos = HttpHeaderInitPos;
 979     debugs(55, 9, this << "joining for id " << id);
 980     /* only fields from ListHeaders array can be "listed" */
 981     assert(CBIT_TEST(ListHeadersMask, id));
 982
 983     if (!CBIT_TEST(mask, id))
 984         return String();
 985
 986     String s;
 987
 988     while ((e = getEntry(&pos))) {
 989         if (e->id == id)
 990             strListAdd(&s, e->value.termedBuf(), ',');
 991     }
 992
 993     /*
 994      * note: we might get an empty (size==0) string if there was an "empty"
 995      * header. This results in an empty length String, which may have a NULL
 996      * buffer.
 997      */
 998     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 999     if (!s.size())
1000         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
1001     else
1002         debugs(55, 6, this << ": joined for id " << id << ": " << s);
1003
1004     return s;
1005 }
1006
1007 /* return a string or list of entries with the same id separated by ',' and ws */
1008 String
1009 HttpHeader::getStrOrList(http_hdr_type id) const
1010 {
1011     HttpHeaderEntry *e;
1012
1013     if (CBIT_TEST(ListHeadersMask, id))
1014         return getList(id);
1015
1016     if ((e = findEntry(id)))
1017         return e->value;
1018
1019     return String();
1020 }
1021
1022 /*
1023  * Returns the value of the specified header and/or an undefined String.
1024  */
1025 String
1026 HttpHeader::getByName(const char *name) const
1027 {
1028     String result;
1029     // ignore presence: return undefined string if an empty header is present
1030     (void)getByNameIfPresent(name, result);
1031     return result;
1032 }
1033
1034 bool
1035 HttpHeader::getByNameIfPresent(const char *name, String &result) const
1036 {
1037     http_hdr_type id;
1038     HttpHeaderPos pos = HttpHeaderInitPos;
1039     HttpHeaderEntry *e;
1040
1041     assert(name);
1042
1043     /* First try the quick path */
1044     id = httpHeaderIdByNameDef(name, strlen(name));
1045
1046     if (id != -1) {
1047         if (!has(id))
1048             return false;
1049         result = getStrOrList(id);
1050         return true;
1051     }
1052
1053     /* Sorry, an unknown header name. Do linear search */
1054     bool found = false;
1055     while ((e = getEntry(&pos))) {
1056         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1057             found = true;
1058             strListAdd(&result, e->value.termedBuf(), ',');
1059         }
1060     }
1061
1062     return found;
1063 }
1064
1065 /*
1066  * Returns a the value of the specified list member, if any.
1067  */
1068 String
1069 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1070 {
1071     String header;
1072     const char *pos = NULL;
1073     const char *item;
1074     int ilen;
1075     int mlen = strlen(member);
1076
1077     assert(name);
1078
1079     header = getByName(name);
1080
1081     String result;
1082
1083     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1084         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1085             result.append(item + mlen + 1, ilen - mlen - 1);
1086             break;
1087         }
1088     }
1089
1090     return result;
1091 }
1092
1093 /*
1094  * returns a the value of the specified list member, if any.
1095  */
1096 String
1097 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1098 {
1099     String header;
1100     const char *pos = NULL;
1101     const char *item;
1102     int ilen;
1103     int mlen = strlen(member);
1104
1105     assert(id >= 0);
1106
1107     header = getStrOrList(id);
1108     String result;
1109
1110     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1111         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1112             result.append(item + mlen + 1, ilen - mlen - 1);
1113             break;
1114         }
1115     }
1116
1117     header.clean();
1118     return result;
1119 }
1120
1121 /* test if a field is present */
1122 int
1123 HttpHeader::has(http_hdr_type id) const
1124 {
1125     assert_eid(id);
1126     assert(id != HDR_OTHER);
1127     debugs(55, 9, this << " lookup for " << id);
1128     return CBIT_TEST(mask, id);
1129 }
1130
1131 void
1132 HttpHeader::putInt(http_hdr_type id, int number)
1133 {
1134     assert_eid(id);
1135     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1136     assert(number >= 0);
1137     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1138 }
1139
1140 void
1141 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1142 {
1143     assert_eid(id);
1144     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1145     assert(number >= 0);
1146     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1147 }
1148
1149 void
1150 HttpHeader::putTime(http_hdr_type id, time_t htime)
1151 {
1152     assert_eid(id);
1153     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1154     assert(htime >= 0);
1155     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1156 }
1157
1158 void
1159 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1160 {
1161     assert_eid(id);
1162     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1163     assert(htime >= 0);
1164     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1165 }
1166
1167 void
1168 HttpHeader::putStr(http_hdr_type id, const char *str)
1169 {
1170     assert_eid(id);
1171     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1172     assert(str);
1173     addEntry(new HttpHeaderEntry(id, NULL, str));
1174 }
1175
1176 void
1177 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1178 {
1179     assert(auth_scheme && realm);
1180     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1181 }
1182
1183 void
1184 HttpHeader::putCc(const HttpHdrCc * cc)
1185 {
1186     MemBuf mb;
1187     Packer p;
1188     assert(cc);
1189     /* remove old directives if any */
1190     delById(HDR_CACHE_CONTROL);
1191     /* pack into mb */
1192     mb.init();
1193     packerToMemInit(&p, &mb);
1194     cc->packInto(&p);
1195     /* put */
1196     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1197     /* cleanup */
1198     packerClean(&p);
1199     mb.clean();
1200 }
1201
1202 void
1203 HttpHeader::putContRange(const HttpHdrContRange * cr)
1204 {
1205     MemBuf mb;
1206     Packer p;
1207     assert(cr);
1208     /* remove old directives if any */
1209     delById(HDR_CONTENT_RANGE);
1210     /* pack into mb */
1211     mb.init();
1212     packerToMemInit(&p, &mb);
1213     httpHdrContRangePackInto(cr, &p);
1214     /* put */
1215     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1216     /* cleanup */
1217     packerClean(&p);
1218     mb.clean();
1219 }
1220
1221 void
1222 HttpHeader::putRange(const HttpHdrRange * range)
1223 {
1224     MemBuf mb;
1225     Packer p;
1226     assert(range);
1227     /* remove old directives if any */
1228     delById(HDR_RANGE);
1229     /* pack into mb */
1230     mb.init();
1231     packerToMemInit(&p, &mb);
1232     range->packInto(&p);
1233     /* put */
1234     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1235     /* cleanup */
1236     packerClean(&p);
1237     mb.clean();
1238 }
1239
1240 void
1241 HttpHeader::putSc(HttpHdrSc *sc)
1242 {
1243     MemBuf mb;
1244     Packer p;
1245     assert(sc);
1246     /* remove old directives if any */
1247     delById(HDR_SURROGATE_CONTROL);
1248     /* pack into mb */
1249     mb.init();
1250     packerToMemInit(&p, &mb);
1251     sc->packInto(&p);
1252     /* put */
1253     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1254     /* cleanup */
1255     packerClean(&p);
1256     mb.clean();
1257 }
1258
1259 void
1260 HttpHeader::putWarning(const int code, const char *const text)
1261 {
1262     char buf[512];
1263     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1264     putStr(HDR_WARNING, buf);
1265 }
1266
1267 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1268 void
1269 HttpHeader::putExt(const char *name, const char *value)
1270 {
1271     assert(name && value);
1272     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1273     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1274 }
1275
1276 int
1277 HttpHeader::getInt(http_hdr_type id) const
1278 {
1279     assert_eid(id);
1280     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1281     HttpHeaderEntry *e;
1282
1283     if ((e = findEntry(id)))
1284         return e->getInt();
1285
1286     return -1;
1287 }
1288
1289 int64_t
1290 HttpHeader::getInt64(http_hdr_type id) const
1291 {
1292     assert_eid(id);
1293     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1294     HttpHeaderEntry *e;
1295
1296     if ((e = findEntry(id)))
1297         return e->getInt64();
1298
1299     return -1;
1300 }
1301
1302 time_t
1303 HttpHeader::getTime(http_hdr_type id) const
1304 {
1305     HttpHeaderEntry *e;
1306     time_t value = -1;
1307     assert_eid(id);
1308     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1309
1310     if ((e = findEntry(id))) {
1311         value = parse_rfc1123(e->value.termedBuf());
1312         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1313     }
1314
1315     return value;
1316 }
1317
1318 /* sync with httpHeaderGetLastStr */
1319 const char *
1320 HttpHeader::getStr(http_hdr_type id) const
1321 {
1322     HttpHeaderEntry *e;
1323     assert_eid(id);
1324     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1325
1326     if ((e = findEntry(id))) {
1327         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1328         return e->value.termedBuf();
1329     }
1330
1331     return NULL;
1332 }
1333
1334 /* unusual */
1335 const char *
1336 HttpHeader::getLastStr(http_hdr_type id) const
1337 {
1338     HttpHeaderEntry *e;
1339     assert_eid(id);
1340     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1341
1342     if ((e = findLastEntry(id))) {
1343         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1344         return e->value.termedBuf();
1345     }
1346
1347     return NULL;
1348 }
1349
1350 HttpHdrCc *
1351 HttpHeader::getCc() const
1352 {
1353     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1354         return NULL;
1355     PROF_start(HttpHeader_getCc);
1356
1357     String s;
1358     getList(HDR_CACHE_CONTROL, &s);
1359
1360     HttpHdrCc *cc=new HttpHdrCc();
1361
1362     if (!cc->parse(s)) {
1363         delete cc;
1364         cc = NULL;
1365     }
1366
1367     ++ HttpHeaderStats[owner].ccParsedCount;
1368
1369     if (cc)
1370         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1371
1372     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1373
1374     PROF_stop(HttpHeader_getCc);
1375
1376     return cc;
1377 }
1378
1379 HttpHdrRange *
1380 HttpHeader::getRange() const
1381 {
1382     HttpHdrRange *r = NULL;
1383     HttpHeaderEntry *e;
1384     /* some clients will send "Request-Range" _and_ *matching* "Range"
1385      * who knows, some clients might send Request-Range only;
1386      * this "if" should work correctly in both cases;
1387      * hopefully no clients send mismatched headers! */
1388
1389     if ((e = findEntry(HDR_RANGE)) ||
1390             (e = findEntry(HDR_REQUEST_RANGE))) {
1391         r = HttpHdrRange::ParseCreate(&e->value);
1392         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1393     }
1394
1395     return r;
1396 }
1397
1398 HttpHdrSc *
1399 HttpHeader::getSc() const
1400 {
1401     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1402         return NULL;
1403
1404     String s;
1405
1406     (void) getList(HDR_SURROGATE_CONTROL, &s);
1407
1408     HttpHdrSc *sc = httpHdrScParseCreate(s);
1409
1410     ++ HttpHeaderStats[owner].ccParsedCount;
1411
1412     if (sc)
1413         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1414
1415     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1416
1417     return sc;
1418 }
1419
1420 HttpHdrContRange *
1421 HttpHeader::getContRange() const
1422 {
1423     HttpHdrContRange *cr = NULL;
1424     HttpHeaderEntry *e;
1425
1426     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1427         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1428         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1429     }
1430
1431     return cr;
1432 }
1433
1434 const char *
1435 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1436 {
1437     const char *field;
1438     int l;
1439     assert(auth_scheme);
1440     field = getStr(id);
1441
1442     if (!field)                 /* no authorization field */
1443         return NULL;
1444
1445     l = strlen(auth_scheme);
1446
1447     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1448         return NULL;
1449
1450     field += l;
1451
1452     if (!xisspace(*field))      /* wrong scheme */
1453         return NULL;
1454
1455     /* skip white space */
1456     for (; field && xisspace(*field); ++field);
1457
1458     if (!*field)                /* no authorization cookie */
1459         return NULL;
1460
1461     static char decodedAuthToken[8192];
1462     const int decodedLen = base64_decode(decodedAuthToken, sizeof(decodedAuthToken)-1, field);
1463     decodedAuthToken[decodedLen] = '\0';
1464     return decodedAuthToken;
1465 }
1466
1467 ETag
1468 HttpHeader::getETag(http_hdr_type id) const
1469 {
1470     ETag etag = {NULL, -1};
1471     HttpHeaderEntry *e;
1472     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1473
1474     if ((e = findEntry(id)))
1475         etagParseInit(&etag, e->value.termedBuf());
1476
1477     return etag;
1478 }
1479
1480 TimeOrTag
1481 HttpHeader::getTimeOrTag(http_hdr_type id) const
1482 {
1483     TimeOrTag tot;
1484     HttpHeaderEntry *e;
1485     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1486     memset(&tot, 0, sizeof(tot));
1487
1488     if ((e = findEntry(id))) {
1489         const char *str = e->value.termedBuf();
1490         /* try as an ETag */
1491
1492         if (etagParseInit(&tot.tag, str)) {
1493             tot.valid = tot.tag.str != NULL;
1494             tot.time = -1;
1495         } else {
1496             /* or maybe it is time? */
1497             tot.time = parse_rfc1123(str);
1498             tot.valid = tot.time >= 0;
1499             tot.tag.str = NULL;
1500         }
1501     }
1502
1503     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1504     return tot;
1505 }
1506
1507 /*
1508  * HttpHeaderEntry
1509  */
1510
1511 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1512 {
1513     assert_eid(anId);
1514     id = anId;
1515
1516     if (id != HDR_OTHER)
1517         name = Headers[id].name;
1518     else
1519         name = aName;
1520
1521     value = aValue;
1522
1523     ++ Headers[id].stat.aliveCount;
1524
1525     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1526 }
1527
1528 HttpHeaderEntry::~HttpHeaderEntry()
1529 {
1530     assert_eid(id);
1531     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1532     /* clean name if needed */
1533
1534     if (id == HDR_OTHER)
1535         name.clean();
1536
1537     value.clean();
1538
1539     assert(Headers[id].stat.aliveCount);
1540
1541     -- Headers[id].stat.aliveCount;
1542
1543     id = HDR_BAD_HDR;
1544 }
1545
1546 /* parses and inits header entry, returns true/false */
1547 HttpHeaderEntry *
1548 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1549 {
1550     /* note: name_start == field_start */
1551     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1552     int name_len = name_end ? name_end - field_start :0;
1553     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1554     /* note: value_end == field_end */
1555
1556     ++ HeaderEntryParsedCount;
1557
1558     /* do we have a valid field name within this field? */
1559
1560     if (!name_len || name_end > field_end)
1561         return NULL;
1562
1563     if (name_len > 65534) {
1564         /* String must be LESS THAN 64K and it adds a terminating NULL */
1565         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1566         return NULL;
1567     }
1568
1569     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1570         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1571                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1572
1573         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1574             --name_len;
1575
1576         if (!name_len)
1577             return NULL;
1578     }
1579
1580     /* now we know we can parse it */
1581
1582     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1583
1584     /* is it a "known" field? */
1585     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1586
1587     String name;
1588
1589     String value;
1590
1591     if (id < 0)
1592         id = HDR_OTHER;
1593
1594     assert_eid(id);
1595
1596     /* set field name */
1597     if (id == HDR_OTHER)
1598         name.limitInit(field_start, name_len);
1599     else
1600         name = Headers[id].name;
1601
1602     /* trim field value */
1603     while (value_start < field_end && xisspace(*value_start))
1604         ++value_start;
1605
1606     while (value_start < field_end && xisspace(field_end[-1]))
1607         --field_end;
1608
1609     if (field_end - value_start > 65534) {
1610         /* String must be LESS THAN 64K and it adds a terminating NULL */
1611         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1612
1613         if (id == HDR_OTHER)
1614             name.clean();
1615
1616         return NULL;
1617     }
1618
1619     /* set field value */
1620     value.limitInit(value_start, field_end - value_start);
1621
1622     ++ Headers[id].stat.seenCount;
1623
1624     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1625
1626     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1627 }
1628
1629 HttpHeaderEntry *
1630 HttpHeaderEntry::clone() const
1631 {
1632     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1633 }
1634
1635 void
1636 HttpHeaderEntry::packInto(Packer * p) const
1637 {
1638     assert(p);
1639     packerAppend(p, name.rawBuf(), name.size());
1640     packerAppend(p, ": ", 2);
1641     packerAppend(p, value.rawBuf(), value.size());
1642     packerAppend(p, "\r\n", 2);
1643 }
1644
1645 int
1646 HttpHeaderEntry::getInt() const
1647 {
1648     assert_eid (id);
1649     assert (Headers[id].type == ftInt);
1650     int val = -1;
1651     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1652     httpHeaderNoteParsedEntry(id, value, !ok);
1653     /* XXX: Should we check ok - ie
1654      * return ok ? -1 : value;
1655      */
1656     return val;
1657 }
1658
1659 int64_t
1660 HttpHeaderEntry::getInt64() const
1661 {
1662     assert_eid (id);
1663     assert (Headers[id].type == ftInt64);
1664     int64_t val = -1;
1665     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1666     httpHeaderNoteParsedEntry(id, value, !ok);
1667     /* XXX: Should we check ok - ie
1668      * return ok ? -1 : value;
1669      */
1670     return val;
1671 }
1672
1673 static void
1674 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1675 {
1676     ++ Headers[id].stat.parsCount;
1677
1678     if (error) {
1679         ++ Headers[id].stat.errCount;
1680         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name << ": " << context << "'");
1681     }
1682 }
1683
1684 /*
1685  * Reports
1686  */
1687
1688 /* tmp variable used to pass stat info to dumpers */
1689 extern const HttpHeaderStat *dump_stat;         /* argh! */
1690 const HttpHeaderStat *dump_stat = NULL;
1691
1692 void
1693 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1694 {
1695     const int id = (int) val;
1696     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1697     const char *name = valid_id ? Headers[id].name.termedBuf() : "INVALID";
1698     int visible = count > 0;
1699     /* for entries with zero count, list only those that belong to current type of message */
1700
1701     if (!visible && valid_id && dump_stat->owner_mask)
1702         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1703
1704     if (visible)
1705         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1706                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1707 }
1708
1709 static void
1710 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1711 {
1712     if (count)
1713         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1714                           idx, (int) val, count,
1715                           xpercent(count, dump_stat->destroyedCount));
1716 }
1717
1718 static void
1719 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1720 {
1721     assert(hs && e);
1722
1723     dump_stat = hs;
1724     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1725     storeAppendPrintf(e, "\nField type distribution\n");
1726     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1727                       "id", "name", "count", "#/header");
1728     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1729     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1730     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1731                       "id", "name", "count", "#/cc_field");
1732     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1733     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1734     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1735                       "id", "name", "count", "#/sc_field");
1736     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1737     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1738     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1739                       "id", "#flds", "count", "%total");
1740     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1741     dump_stat = NULL;
1742 }
1743
1744 void
1745 httpHeaderStoreReport(StoreEntry * e)
1746 {
1747     int i;
1748     http_hdr_type ht;
1749     assert(e);
1750
1751     HttpHeaderStats[0].parsedCount =
1752         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1753     HttpHeaderStats[0].ccParsedCount =
1754         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1755     HttpHeaderStats[0].destroyedCount =
1756         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1757     HttpHeaderStats[0].busyDestroyedCount =
1758         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1759
1760     for (i = 1; i < HttpHeaderStatCount; ++i) {
1761         httpHeaderStatDump(HttpHeaderStats + i, e);
1762         storeAppendPrintf(e, "%s\n", "<br>");
1763     }
1764
1765     /* field stats for all messages */
1766     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1767
1768     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1769                       "id", "name", "#alive", "%err", "%repeat");
1770
1771     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1772         HttpHeaderFieldInfo *f = Headers + ht;
1773         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1774                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1775                           xpercent(f->stat.errCount, f->stat.parsCount),
1776                           xpercent(f->stat.repCount, f->stat.seenCount));
1777     }
1778
1779     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1780                       HttpHeaderStats[hoRequest].parsedCount,
1781                       HttpHeaderStats[hoReply].parsedCount,
1782                       HttpHeaderStats[0].parsedCount);
1783     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1784 }
1785
1786 http_hdr_type
1787 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1788 {
1789     if (name_len > 0) {
1790         for (int i = 0; i < end; ++i) {
1791             if (name_len != info[i].name.size())
1792                 continue;
1793
1794             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1795                 return info[i].id;
1796         }
1797     }
1798
1799     return HDR_BAD_HDR;
1800 }
1801
1802 http_hdr_type
1803 httpHeaderIdByNameDef(const char *name, int name_len)
1804 {
1805     if (!Headers)
1806         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1807
1808     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1809 }
1810
1811 const char *
1812 httpHeaderNameById(int id)
1813 {
1814     if (!Headers)
1815         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1816
1817     assert(id >= 0 && id < HDR_ENUM_END);
1818
1819     return Headers[id].name.termedBuf();
1820 }
1821
1822 int
1823 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1824 {
1825     int result = 0;
1826     const char *pos = NULL;
1827     const char *item;
1828     int ilen;
1829     int mlen = strlen(member);
1830
1831     assert(id >= 0);
1832
1833     String header (getStrOrList(id));
1834
1835     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1836         if (strncasecmp(item, member, mlen) == 0
1837                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1838             result = 1;
1839             break;
1840         }
1841     }
1842
1843     return result;
1844 }
1845
1846 int
1847 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1848 {
1849     int result = 0;
1850     const char *pos = NULL;
1851     const char *item;
1852     int ilen;
1853     int mlen = strlen(member);
1854
1855     assert(name);
1856
1857     String header (getByName(name));
1858
1859     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1860         if (strncasecmp(item, member, mlen) == 0
1861                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1862             result = 1;
1863             break;
1864         }
1865     }
1866
1867     return result;
1868 }
1869
1870 void
1871 HttpHeader::removeHopByHopEntries()
1872 {
1873     removeConnectionHeaderEntries();
1874
1875     const HttpHeaderEntry *e;
1876     HttpHeaderPos pos = HttpHeaderInitPos;
1877     int headers_deleted = 0;
1878     while ((e = getEntry(&pos))) {
1879         int id = e->id;
1880         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1881             delAt(pos, headers_deleted);
1882             CBIT_CLR(mask, id);
1883         }
1884     }
1885 }
1886
1887 void
1888 HttpHeader::removeConnectionHeaderEntries()
1889 {
1890     if (has(HDR_CONNECTION)) {
1891         /* anything that matches Connection list member will be deleted */
1892         String strConnection;
1893
1894         (void) getList(HDR_CONNECTION, &strConnection);
1895         const HttpHeaderEntry *e;
1896         HttpHeaderPos pos = HttpHeaderInitPos;
1897         /*
1898          * think: on-average-best nesting of the two loops (hdrEntry
1899          * and strListItem) @?@
1900          */
1901         /*
1902          * maybe we should delete standard stuff ("keep-alive","close")
1903          * from strConnection first?
1904          */
1905
1906         int headers_deleted = 0;
1907         while ((e = getEntry(&pos))) {
1908             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1909                 delAt(pos, headers_deleted);
1910         }
1911         if (headers_deleted)
1912             refreshMask();
1913     }
1914 }