src/HttpHeader.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 55    HTTP Header
   6  * AUTHOR: Alex Rousskov
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36 #include "squid.h"
  37 #include "base64.h"
  38 #include "HttpHdrContRange.h"
  39 #include "HttpHdrCc.h"
  40 #include "HttpHdrSc.h"
  41 #include "HttpHeader.h"
  42 #include "MemBuf.h"
  43 #include "mgr/Registration.h"
  44 #include "rfc1123.h"
  45 #include "StatHist.h"
  46 #include "Store.h"
  47 #include "TimeOrTag.h"
  48
  49 /*
  50  * On naming conventions:
  51  *
  52  * HTTP/1.1 defines message-header as
  53  *
  54  * message-header = field-name ":" [ field-value ] CRLF
  55  * field-name     = token
  56  * field-value    = *( field-content | LWS )
  57  *
  58  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  59  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  60  *
  61  * HttpHeader is an object that represents all message-headers in a message.
  62  * HttpHeader does not manage start-line.
  63  *
  64  * HttpHeader is implemented as a collection of header "entries".
  65  * An entry is a (field_id, field_name, field_value) triplet.
  66  */
  67
  68
  69 /*
  70  * local constants and vars
  71  */
  72
  73 /*
  74  * A table with major attributes for every known field.
  75  * We calculate name lengths and reorganize this array on start up.
  76  * After reorganization, field id can be used as an index to the table.
  77  */
  78 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  79     {"Accept", HDR_ACCEPT, ftStr},
  80
  81     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
  82     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
  83     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
  84     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
  85     {"Age", HDR_AGE, ftInt},
  86     {"Allow", HDR_ALLOW, ftStr},
  87     {"Authorization", HDR_AUTHORIZATION, ftStr},        /* for now */
  88     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
  89     {"Connection", HDR_CONNECTION, ftStr},
  90     {"Content-Base", HDR_CONTENT_BASE, ftStr},
  91     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
  92     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
  93     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
  94     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
  95     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
  96     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
  97     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
  98     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
  99     {"Cookie", HDR_COOKIE, ftStr},
 100     {"Cookie2", HDR_COOKIE2, ftStr},
 101     {"Date", HDR_DATE, ftDate_1123},
 102     {"ETag", HDR_ETAG, ftETag},
 103     {"Expect", HDR_EXPECT, ftStr},
 104     {"Expires", HDR_EXPIRES, ftDate_1123},
 105     {"From", HDR_FROM, ftStr},
 106     {"Host", HDR_HOST, ftStr},
 107     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 108     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 109     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},        /* for now */
 110     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 111     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 112     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 113     {"Link", HDR_LINK, ftStr},
 114     {"Location", HDR_LOCATION, ftStr},
 115     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 116     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 117     {"Negotiate", HDR_NEGOTIATE, ftStr},
 118     {"Pragma", HDR_PRAGMA, ftStr},
 119     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 120     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 121     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 122     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 123     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 124     {"Public", HDR_PUBLIC, ftStr},
 125     {"Range", HDR_RANGE, ftPRange},
 126     {"Referer", HDR_REFERER, ftStr},
 127     {"Request-Range", HDR_REQUEST_RANGE, ftPRange},     /* usually matches HDR_RANGE */
 128     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!) */
 129     {"Server", HDR_SERVER, ftStr},
 130     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 131     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 132     {"TE", HDR_TE, ftStr},
 133     {"Title", HDR_TITLE, ftStr},
 134     {"Trailer", HDR_TRAILER, ftStr},
 135     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 136     {"Translate", HDR_TRANSLATE, ftStr},        /* for now. may need to crop */
 137     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 138     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 139     {"User-Agent", HDR_USER_AGENT, ftStr},
 140     {"Vary", HDR_VARY, ftStr},  /* for now */
 141     {"Via", HDR_VIA, ftStr},    /* for now */
 142     {"Warning", HDR_WARNING, ftStr},    /* for now */
 143     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 144     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 145     {"X-Cache", HDR_X_CACHE, ftStr},
 146     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 147     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 148     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 149     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 150 #if X_ACCELERATOR_VARY
 151     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 152 #endif
 153 #if USE_ADAPTATION
 154     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 155 #endif
 156     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 157     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 158     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 159     {"Other:", HDR_OTHER, ftStr}        /* ':' will not allow matches */
 160 };
 161
 162 static HttpHeaderFieldInfo *Headers = NULL;
 163
 164 http_hdr_type &operator++ (http_hdr_type &aHeader)
 165 {
 166     int tmp = (int)aHeader;
 167     aHeader = (http_hdr_type)(++tmp);
 168     return aHeader;
 169 }
 170
 171
 172 /*
 173  * headers with field values defined as #(values) in HTTP/1.1
 174  * Headers that are currently not recognized, are commented out.
 175  */
 176 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 177 static http_hdr_type ListHeadersArr[] = {
 178     HDR_ACCEPT,
 179     HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 180     HDR_ACCEPT_RANGES, HDR_ALLOW,
 181     HDR_CACHE_CONTROL,
 182     HDR_CONTENT_ENCODING,
 183     HDR_CONTENT_LANGUAGE,
 184     HDR_CONNECTION,
 185     HDR_EXPECT,
 186     HDR_IF_MATCH, HDR_IF_NONE_MATCH,
 187     HDR_LINK, HDR_PRAGMA,
 188     HDR_PROXY_CONNECTION,
 189     HDR_PROXY_SUPPORT,
 190     HDR_TRANSFER_ENCODING,
 191     HDR_UPGRADE,
 192     HDR_VARY,
 193     HDR_VIA,
 194     HDR_WARNING,
 195     HDR_WWW_AUTHENTICATE,
 196     HDR_AUTHENTICATION_INFO,
 197     HDR_PROXY_AUTHENTICATION_INFO,
 198     /* HDR_TE, HDR_TRAILER */
 199 #if X_ACCELERATOR_VARY
 200     HDR_X_ACCELERATOR_VARY,
 201 #endif
 202 #if USE_ADAPTATION
 203     HDR_X_NEXT_SERVICES,
 204 #endif
 205     HDR_SURROGATE_CAPABILITY,
 206     HDR_SURROGATE_CONTROL,
 207     HDR_X_FORWARDED_FOR
 208 };
 209
 210 /* general-headers */
 211 static http_hdr_type GeneralHeadersArr[] = {
 212     HDR_CACHE_CONTROL, HDR_CONNECTION, HDR_DATE, HDR_PRAGMA,
 213     HDR_TRANSFER_ENCODING,
 214     HDR_UPGRADE,
 215     /* HDR_TRAILER, */
 216     HDR_VIA,
 217 };
 218
 219 /* entity-headers */
 220 static http_hdr_type EntityHeadersArr[] = {
 221     HDR_ALLOW, HDR_CONTENT_BASE, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE,
 222     HDR_CONTENT_LENGTH, HDR_CONTENT_LOCATION, HDR_CONTENT_MD5,
 223     HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_ETAG, HDR_EXPIRES, HDR_LAST_MODIFIED, HDR_LINK,
 224     HDR_OTHER
 225 };
 226
 227 static HttpHeaderMask ReplyHeadersMask;         /* set run-time using ReplyHeaders */
 228 static http_hdr_type ReplyHeadersArr[] = {
 229     HDR_ACCEPT, HDR_ACCEPT_CHARSET, HDR_ACCEPT_ENCODING, HDR_ACCEPT_LANGUAGE,
 230     HDR_ACCEPT_RANGES, HDR_AGE,
 231     HDR_LOCATION, HDR_MAX_FORWARDS,
 232     HDR_MIME_VERSION, HDR_PUBLIC, HDR_RETRY_AFTER, HDR_SERVER, HDR_SET_COOKIE, HDR_SET_COOKIE2,
 233     HDR_VARY,
 234     HDR_WARNING, HDR_PROXY_CONNECTION, HDR_X_CACHE,
 235     HDR_X_CACHE_LOOKUP,
 236     HDR_X_REQUEST_URI,
 237 #if X_ACCELERATOR_VARY
 238     HDR_X_ACCELERATOR_VARY,
 239 #endif
 240 #if USE_ADAPTATION
 241     HDR_X_NEXT_SERVICES,
 242 #endif
 243     HDR_X_SQUID_ERROR,
 244     HDR_SURROGATE_CONTROL
 245 };
 246
 247 static HttpHeaderMask RequestHeadersMask;       /* set run-time using RequestHeaders */
 248 static http_hdr_type RequestHeadersArr[] = {
 249     HDR_AUTHORIZATION, HDR_FROM, HDR_HOST,
 250     HDR_IF_MATCH, HDR_IF_MODIFIED_SINCE, HDR_IF_NONE_MATCH,
 251     HDR_IF_RANGE, HDR_MAX_FORWARDS, HDR_PROXY_CONNECTION,
 252     HDR_PROXY_AUTHORIZATION, HDR_RANGE, HDR_REFERER, HDR_REQUEST_RANGE,
 253     HDR_USER_AGENT, HDR_X_FORWARDED_FOR, HDR_SURROGATE_CAPABILITY
 254 };
 255
 256 static HttpHeaderMask HopByHopHeadersMask;
 257 static http_hdr_type HopByHopHeadersArr[] = {
 258     HDR_CONNECTION, HDR_KEEP_ALIVE, /*HDR_PROXY_AUTHENTICATE,*/ HDR_PROXY_AUTHORIZATION,
 259     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE, HDR_PROXY_CONNECTION
 260 };
 261
 262 /* header accounting */
 263 static HttpHeaderStat HttpHeaderStats[] = {
 264     {"all"},
 265 #if USE_HTCP
 266     {"HTCP reply"},
 267 #endif
 268     {"request"},
 269     {"reply"}
 270 };
 271 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 272
 273 static int HeaderEntryParsedCount = 0;
 274
 275 /*
 276  * local routines
 277  */
 278
 279 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 280
 281 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 282
 283 static void httpHeaderStatInit(HttpHeaderStat * hs, const char *label);
 284 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 285
 286 /*
 287  * Module initialization routines
 288  */
 289
 290 static void
 291 httpHeaderRegisterWithCacheManager(void)
 292 {
 293     Mgr::RegisterAction("http_headers",
 294                         "HTTP Header Statistics",
 295                         httpHeaderStoreReport, 0, 1);
 296 }
 297
 298 void
 299 httpHeaderInitModule(void)
 300 {
 301     int i;
 302     /* check that we have enough space for masks */
 303     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 304     /* all headers must be described */
 305     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 306
 307     if (!Headers)
 308         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 309
 310     /* create masks */
 311     httpHeaderMaskInit(&ListHeadersMask, 0);
 312
 313     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 314
 315     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 316
 317     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 318
 319     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 320
 321     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 322
 323     httpHeaderMaskInit(&RequestHeadersMask, 0);
 324
 325     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 326
 327     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 328
 329     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 330
 331     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 332
 333     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 334
 335     /* init header stats */
 336     assert(HttpHeaderStatCount == hoReply + 1);
 337
 338     for (i = 0; i < HttpHeaderStatCount; i++)
 339         httpHeaderStatInit(HttpHeaderStats + i, HttpHeaderStats[i].label);
 340
 341     HttpHeaderStats[hoRequest].owner_mask = &RequestHeadersMask;
 342
 343     HttpHeaderStats[hoReply].owner_mask = &ReplyHeadersMask;
 344
 345 #if USE_HTCP
 346
 347     HttpHeaderStats[hoHtcpReply].owner_mask = &ReplyHeadersMask;
 348
 349 #endif
 350     /* init dependent modules */
 351     httpHdrCcInitModule();
 352
 353     httpHdrScInitModule();
 354
 355     httpHeaderRegisterWithCacheManager();
 356 }
 357
 358 void
 359 httpHeaderCleanModule(void)
 360 {
 361     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 362     Headers = NULL;
 363     httpHdrCcCleanModule();
 364     httpHdrScCleanModule();
 365 }
 366
 367 static void
 368 httpHeaderStatInit(HttpHeaderStat * hs, const char *label)
 369 {
 370     assert(hs);
 371     assert(label);
 372     memset(hs, 0, sizeof(HttpHeaderStat));
 373     hs->label = label;
 374     statHistEnumInit(&hs->hdrUCountDistr, 32);  /* not a real enum */
 375     statHistEnumInit(&hs->fieldTypeDistr, HDR_ENUM_END);
 376     statHistEnumInit(&hs->ccTypeDistr, CC_ENUM_END);
 377     statHistEnumInit(&hs->scTypeDistr, SC_ENUM_END);
 378 }
 379
 380 /*
 381  * HttpHeader Implementation
 382  */
 383
 384 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 385 {
 386     httpHeaderMaskInit(&mask, 0);
 387 }
 388
 389 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 390 {
 391     assert(anOwner > hoNone && anOwner < hoEnd);
 392     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 393     httpHeaderMaskInit(&mask, 0);
 394 }
 395
 396 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 397 {
 398     httpHeaderMaskInit(&mask, 0);
 399     update(&other, NULL); // will update the mask as well
 400 }
 401
 402 HttpHeader::~HttpHeader()
 403 {
 404     clean();
 405 }
 406
 407 HttpHeader &
 408 HttpHeader::operator =(const HttpHeader &other)
 409 {
 410     if (this != &other) {
 411         // we do not really care, but the caller probably does
 412         assert(owner == other.owner);
 413         clean();
 414         update(&other, NULL); // will update the mask as well
 415         len = other.len;
 416     }
 417     return *this;
 418 }
 419
 420 void
 421 HttpHeader::clean()
 422 {
 423     HttpHeaderPos pos = HttpHeaderInitPos;
 424     HttpHeaderEntry *e;
 425
 426     assert(owner > hoNone && owner < hoEnd);
 427     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 428
 429     PROF_start(HttpHeaderClean);
 430
 431     /*
 432      * An unfortunate bug.  The entries array is initialized
 433      * such that count is set to zero.  httpHeaderClean() seems to
 434      * be called both when 'hdr' is created, and destroyed.  Thus,
 435      * we accumulate a large number of zero counts for 'hdr' before
 436      * it is ever used.  Can't think of a good way to fix it, except
 437      * adding a state variable that indicates whether or not 'hdr'
 438      * has been used.  As a hack, just never count zero-sized header
 439      * arrays.
 440      */
 441
 442     if (owner <= hoReply) {
 443         if (0 != entries.count)
 444             HttpHeaderStats[owner].hdrUCountDistr.count(entries.count);
 445
 446         HttpHeaderStats[owner].destroyedCount++;
 447
 448         HttpHeaderStats[owner].busyDestroyedCount += entries.count > 0;
 449
 450         while ((e = getEntry(&pos))) {
 451             /* tmp hack to try to avoid coredumps */
 452
 453             if (e->id < 0 || e->id >= HDR_ENUM_END) {
 454                 debugs(55, 0, "HttpHeader::clean BUG: entry[" << pos << "] is invalid (" << e->id << "). Ignored.");
 455             } else {
 456                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 457                 /* yes, this deletion leaves us in an inconsistent state */
 458                 delete e;
 459             }
 460         }
 461     } // if (owner <= hoReply)
 462     entries.clean();
 463     httpHeaderMaskInit(&mask, 0);
 464     len = 0;
 465     PROF_stop(HttpHeaderClean);
 466 }
 467
 468 /* append entries (also see httpHeaderUpdate) */
 469 void
 470 HttpHeader::append(const HttpHeader * src)
 471 {
 472     const HttpHeaderEntry *e;
 473     HttpHeaderPos pos = HttpHeaderInitPos;
 474     assert(src);
 475     assert(src != this);
 476     debugs(55, 7, "appending hdr: " << this << " += " << src);
 477
 478     while ((e = src->getEntry(&pos))) {
 479         addEntry(e->clone());
 480     }
 481 }
 482
 483 /* use fresh entries to replace old ones */
 484 void
 485 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 486 {
 487     assert (old);
 488     old->update (fresh, denied_mask);
 489 }
 490
 491 void
 492 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 493 {
 494     const HttpHeaderEntry *e;
 495     HttpHeaderPos pos = HttpHeaderInitPos;
 496     assert(fresh);
 497     assert(this != fresh);
 498
 499     while ((e = fresh->getEntry(&pos))) {
 500         /* deny bad guys (ok to check for HDR_OTHER) here */
 501
 502         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 503             continue;
 504
 505         if (e->id != HDR_OTHER)
 506             delById(e->id);
 507         else
 508             delByName(e->name.termedBuf());
 509     }
 510
 511     pos = HttpHeaderInitPos;
 512     while ((e = fresh->getEntry(&pos))) {
 513         /* deny bad guys (ok to check for HDR_OTHER) here */
 514
 515         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 516             continue;
 517
 518         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 519
 520         addEntry(e->clone());
 521     }
 522 }
 523
 524 /* just handy in parsing: resets and returns false */
 525 int
 526 HttpHeader::reset()
 527 {
 528     clean();
 529     return 0;
 530 }
 531
 532 int
 533 HttpHeader::parse(const char *header_start, const char *header_end)
 534 {
 535     const char *field_ptr = header_start;
 536     HttpHeaderEntry *e, *e2;
 537
 538     PROF_start(HttpHeaderParse);
 539
 540     assert(header_start && header_end);
 541     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, header_end));
 542     HttpHeaderStats[owner].parsedCount++;
 543
 544     char *nulpos;
 545     if ((nulpos = (char*)memchr(header_start, '\0', header_end - header_start))) {
 546         debugs(55, 1, "WARNING: HTTP header contains NULL characters {" <<
 547                getStringPrefix(header_start, nulpos) << "}\nNULL\n{" << getStringPrefix(nulpos+1, header_end));
 548         goto reset;
 549     }
 550
 551     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 552      * continuation lines start with a (single) space or tab */
 553     while (field_ptr < header_end) {
 554         const char *field_start = field_ptr;
 555         const char *field_end;
 556
 557         do {
 558             const char *this_line = field_ptr;
 559             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 560
 561             if (!field_ptr)
 562                 goto reset;     /* missing <LF> */
 563
 564             field_end = field_ptr;
 565
 566             field_ptr++;        /* Move to next line */
 567
 568             if (field_end > this_line && field_end[-1] == '\r') {
 569                 field_end--;    /* Ignore CR LF */
 570
 571                 if (owner == hoRequest && field_end > this_line) {
 572                     bool cr_only = true;
 573                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 574                         if (*p != '\r')
 575                             cr_only = false;
 576                     }
 577                     if (cr_only) {
 578                         debugs(55, 1, "WARNING: Rejecting HTTP request with a CR+ "
 579                                "header field to prevent request smuggling attacks: {" <<
 580                                getStringPrefix(header_start, header_end) << "}");
 581                         goto reset;
 582                     }
 583                 }
 584             }
 585
 586             /* Barf on stray CR characters */
 587             if (memchr(this_line, '\r', field_end - this_line)) {
 588                 debugs(55, 1, "WARNING: suspicious CR characters in HTTP header {" <<
 589                        getStringPrefix(field_start, field_end) << "}");
 590
 591                 if (Config.onoff.relaxed_header_parser) {
 592                     char *p = (char *) this_line;       /* XXX Warning! This destroys original header content and violates specifications somewhat */
 593
 594                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL)
 595                         *p++ = ' ';
 596                 } else
 597                     goto reset;
 598             }
 599
 600             if (this_line + 1 == field_end && this_line > field_start) {
 601                 debugs(55, 1, "WARNING: Blank continuation line in HTTP header {" <<
 602                        getStringPrefix(header_start, header_end) << "}");
 603                 goto reset;
 604             }
 605         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 606
 607         if (field_start == field_end) {
 608             if (field_ptr < header_end) {
 609                 debugs(55, 1, "WARNING: unparseable HTTP header field near {" <<
 610                        getStringPrefix(field_start, header_end) << "}");
 611                 goto reset;
 612             }
 613
 614             break;              /* terminating blank line */
 615         }
 616
 617         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 618             debugs(55, 1, "WARNING: unparseable HTTP header field {" <<
 619                    getStringPrefix(field_start, field_end) << "}");
 620             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 621                    " in {" << getStringPrefix(header_start, header_end) << "}");
 622
 623             if (Config.onoff.relaxed_header_parser)
 624                 continue;
 625             else
 626                 goto reset;
 627         }
 628
 629         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 630 //            if (e->value.cmp(e2->value.termedBuf()) != 0) {
 631             if (e->value != e2->value) {
 632                 int64_t l1, l2;
 633                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 634                        "WARNING: found two conflicting content-length headers in {" << getStringPrefix(header_start, header_end) << "}");
 635
 636                 if (!Config.onoff.relaxed_header_parser) {
 637                     delete e;
 638                     goto reset;
 639                 }
 640
 641                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 642                     debugs(55, 1, "WARNING: Unparseable content-length '" << e->value << "'");
 643                     delete e;
 644                     continue;
 645                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 646                     debugs(55, 1, "WARNING: Unparseable content-length '" << e2->value << "'");
 647                     delById(e2->id);
 648                 } else if (l1 > l2) {
 649                     delById(e2->id);
 650                 } else {
 651                     delete e;
 652                     continue;
 653                 }
 654             } else {
 655                 debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 656                        "NOTICE: found double content-length header");
 657
 658                 if (Config.onoff.relaxed_header_parser) {
 659                     delete e;
 660                     continue;
 661                 } else {
 662                     delete e;
 663                     goto reset;
 664                 }
 665             }
 666         }
 667
 668         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 669             debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
 670                    "WARNING: found whitespace in HTTP header name {" <<
 671                    getStringPrefix(field_start, field_end) << "}");
 672
 673             if (!Config.onoff.relaxed_header_parser) {
 674                 delete e;
 675                 goto reset;
 676             }
 677         }
 678
 679         addEntry(e);
 680     }
 681
 682     if (chunked()) {
 683         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 684         delById(HDR_CONTENT_LENGTH);
 685     }
 686
 687     PROF_stop(HttpHeaderParse);
 688     return 1;                   /* even if no fields where found, it is a valid header */
 689 reset:
 690     PROF_stop(HttpHeaderParse);
 691     return reset();
 692 }
 693
 694 /* packs all the entries using supplied packer */
 695 void
 696 HttpHeader::packInto(Packer * p) const
 697 {
 698     HttpHeaderPos pos = HttpHeaderInitPos;
 699     const HttpHeaderEntry *e;
 700     assert(p);
 701     debugs(55, 7, "packing hdr: (" << this << ")");
 702     /* pack all entries one by one */
 703     while ((e = getEntry(&pos)))
 704         e->packInto(p);
 705
 706     /* Pack in the "special" entries */
 707
 708     /* Cache-Control */
 709 }
 710
 711 /* returns next valid entry */
 712 HttpHeaderEntry *
 713 HttpHeader::getEntry(HttpHeaderPos * pos) const
 714 {
 715     assert(pos);
 716     assert(*pos >= HttpHeaderInitPos && *pos < (ssize_t)entries.count);
 717
 718     for ((*pos)++; *pos < (ssize_t)entries.count; (*pos)++) {
 719         if (entries.items[*pos])
 720             return (HttpHeaderEntry*)entries.items[*pos];
 721     }
 722
 723     return NULL;
 724 }
 725
 726 /*
 727  * returns a pointer to a specified entry if any
 728  * note that we return one entry so it does not make much sense to ask for
 729  * "list" headers
 730  */
 731 HttpHeaderEntry *
 732 HttpHeader::findEntry(http_hdr_type id) const
 733 {
 734     HttpHeaderPos pos = HttpHeaderInitPos;
 735     HttpHeaderEntry *e;
 736     assert_eid(id);
 737     assert(!CBIT_TEST(ListHeadersMask, id));
 738
 739     /* check mask first */
 740
 741     if (!CBIT_TEST(mask, id))
 742         return NULL;
 743
 744     /* looks like we must have it, do linear search */
 745     while ((e = getEntry(&pos))) {
 746         if (e->id == id)
 747             return e;
 748     }
 749
 750     /* hm.. we thought it was there, but it was not found */
 751     assert(0);
 752
 753     return NULL;                /* not reached */
 754 }
 755
 756 /*
 757  * same as httpHeaderFindEntry
 758  */
 759 HttpHeaderEntry *
 760 HttpHeader::findLastEntry(http_hdr_type id) const
 761 {
 762     HttpHeaderPos pos = HttpHeaderInitPos;
 763     HttpHeaderEntry *e;
 764     HttpHeaderEntry *result = NULL;
 765     assert_eid(id);
 766     assert(!CBIT_TEST(ListHeadersMask, id));
 767
 768     /* check mask first */
 769
 770     if (!CBIT_TEST(mask, id))
 771         return NULL;
 772
 773     /* looks like we must have it, do linear search */
 774     while ((e = getEntry(&pos))) {
 775         if (e->id == id)
 776             result = e;
 777     }
 778
 779     assert(result);             /* must be there! */
 780     return result;
 781 }
 782
 783 /*
 784  * deletes all fields with a given name if any, returns #fields deleted;
 785  */
 786 int
 787 HttpHeader::delByName(const char *name)
 788 {
 789     int count = 0;
 790     HttpHeaderPos pos = HttpHeaderInitPos;
 791     HttpHeaderEntry *e;
 792     httpHeaderMaskInit(&mask, 0);       /* temporal inconsistency */
 793     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 794
 795     while ((e = getEntry(&pos))) {
 796         if (!e->name.caseCmp(name))
 797             delAt(pos, count);
 798         else
 799             CBIT_SET(mask, e->id);
 800     }
 801
 802     return count;
 803 }
 804
 805 /* deletes all entries with a given id, returns the #entries deleted */
 806 int
 807 HttpHeader::delById(http_hdr_type id)
 808 {
 809     int count = 0;
 810     HttpHeaderPos pos = HttpHeaderInitPos;
 811     HttpHeaderEntry *e;
 812     debugs(55, 8, this << " del-by-id " << id);
 813     assert_eid(id);
 814     assert(id != HDR_OTHER);            /* does not make sense */
 815
 816     if (!CBIT_TEST(mask, id))
 817         return 0;
 818
 819     while ((e = getEntry(&pos))) {
 820         if (e->id == id)
 821             delAt(pos, count);
 822     }
 823
 824     CBIT_CLR(mask, id);
 825     assert(count);
 826     return count;
 827 }
 828
 829 /*
 830  * deletes an entry at pos and leaves a gap; leaving a gap makes it
 831  * possible to iterate(search) and delete fields at the same time
 832  * NOTE: Does not update the header mask. Caller must follow up with
 833  * a call to refreshMask() if headers_deleted was incremented.
 834  */
 835 void
 836 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
 837 {
 838     HttpHeaderEntry *e;
 839     assert(pos >= HttpHeaderInitPos && pos < (ssize_t)entries.count);
 840     e = (HttpHeaderEntry*)entries.items[pos];
 841     entries.items[pos] = NULL;
 842     /* decrement header length, allow for ": " and crlf */
 843     len -= e->name.size() + 2 + e->value.size() + 2;
 844     assert(len >= 0);
 845     delete e;
 846     ++headers_deleted;
 847 }
 848
 849 /*
 850  * Compacts the header storage
 851  */
 852 void
 853 HttpHeader::compact()
 854 {
 855     entries.prune(NULL);
 856 }
 857
 858 /*
 859  * Refreshes the header mask. Required after delAt() calls.
 860  */
 861 void
 862 HttpHeader::refreshMask()
 863 {
 864     httpHeaderMaskInit(&mask, 0);
 865     debugs(55, 7, "refreshing the mask in hdr " << this);
 866     HttpHeaderPos pos = HttpHeaderInitPos;
 867     while (HttpHeaderEntry *e = getEntry(&pos)) {
 868         CBIT_SET(mask, e->id);
 869     }
 870 }
 871
 872 /* appends an entry;
 873  * does not call e->clone() so one should not reuse "*e"
 874  */
 875 void
 876 HttpHeader::addEntry(HttpHeaderEntry * e)
 877 {
 878     assert(e);
 879     assert_eid(e->id);
 880     assert(e->name.size());
 881
 882     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 883
 884     if (CBIT_TEST(mask, e->id))
 885         Headers[e->id].stat.repCount++;
 886     else
 887         CBIT_SET(mask, e->id);
 888
 889     entries.push_back(e);
 890
 891     /* increment header length, allow for ": " and crlf */
 892     len += e->name.size() + 2 + e->value.size() + 2;
 893 }
 894
 895 /* inserts an entry;
 896  * does not call e->clone() so one should not reuse "*e"
 897  */
 898 void
 899 HttpHeader::insertEntry(HttpHeaderEntry * e)
 900 {
 901     assert(e);
 902     assert_eid(e->id);
 903
 904     debugs(55, 7, HERE << this << " adding entry: " << e->id << " at " << entries.count);
 905
 906     if (CBIT_TEST(mask, e->id))
 907         Headers[e->id].stat.repCount++;
 908     else
 909         CBIT_SET(mask, e->id);
 910
 911     entries.insert(e);
 912
 913     /* increment header length, allow for ": " and crlf */
 914     len += e->name.size() + 2 + e->value.size() + 2;
 915 }
 916
 917 bool
 918 HttpHeader::getList(http_hdr_type id, String *s) const
 919 {
 920     HttpHeaderEntry *e;
 921     HttpHeaderPos pos = HttpHeaderInitPos;
 922     debugs(55, 9, this << " joining for id " << id);
 923     /* only fields from ListHeaders array can be "listed" */
 924     assert(CBIT_TEST(ListHeadersMask, id));
 925
 926     if (!CBIT_TEST(mask, id))
 927         return false;
 928
 929     while ((e = getEntry(&pos))) {
 930         if (e->id == id)
 931             strListAdd(s, e->value.termedBuf(), ',');
 932     }
 933
 934     /*
 935      * note: we might get an empty (size==0) string if there was an "empty"
 936      * header. This results in an empty length String, which may have a NULL
 937      * buffer.
 938      */
 939     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 940     if (!s->size())
 941         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 942     else
 943         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 944
 945     return true;
 946 }
 947
 948 /* return a list of entries with the same id separated by ',' and ws */
 949 String
 950 HttpHeader::getList(http_hdr_type id) const
 951 {
 952     HttpHeaderEntry *e;
 953     HttpHeaderPos pos = HttpHeaderInitPos;
 954     debugs(55, 9, this << "joining for id " << id);
 955     /* only fields from ListHeaders array can be "listed" */
 956     assert(CBIT_TEST(ListHeadersMask, id));
 957
 958     if (!CBIT_TEST(mask, id))
 959         return String();
 960
 961     String s;
 962
 963     while ((e = getEntry(&pos))) {
 964         if (e->id == id)
 965             strListAdd(&s, e->value.termedBuf(), ',');
 966     }
 967
 968     /*
 969      * note: we might get an empty (size==0) string if there was an "empty"
 970      * header. This results in an empty length String, which may have a NULL
 971      * buffer.
 972      */
 973     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
 974     if (!s.size())
 975         debugs(55, 3, "empty list header: " << Headers[id].name << "(" << id << ")");
 976     else
 977         debugs(55, 6, this << ": joined for id " << id << ": " << s);
 978
 979     return s;
 980 }
 981
 982 /* return a string or list of entries with the same id separated by ',' and ws */
 983 String
 984 HttpHeader::getStrOrList(http_hdr_type id) const
 985 {
 986     HttpHeaderEntry *e;
 987
 988     if (CBIT_TEST(ListHeadersMask, id))
 989         return getList(id);
 990
 991     if ((e = findEntry(id)))
 992         return e->value;
 993
 994     return String();
 995 }
 996
 997 /*
 998  * Returns the value of the specified header.
 999  */
1000 String
1001 HttpHeader::getByName(const char *name) const
1002 {
1003     http_hdr_type id;
1004     HttpHeaderPos pos = HttpHeaderInitPos;
1005     HttpHeaderEntry *e;
1006
1007     assert(name);
1008
1009     /* First try the quick path */
1010     id = httpHeaderIdByNameDef(name, strlen(name));
1011
1012     if (id != -1)
1013         return getStrOrList(id);
1014
1015     String result;
1016
1017     /* Sorry, an unknown header name. Do linear search */
1018     while ((e = getEntry(&pos))) {
1019         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1020             strListAdd(&result, e->value.termedBuf(), ',');
1021         }
1022     }
1023
1024     return result;
1025 }
1026
1027 /*
1028  * Returns a the value of the specified list member, if any.
1029  */
1030 String
1031 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1032 {
1033     String header;
1034     const char *pos = NULL;
1035     const char *item;
1036     int ilen;
1037     int mlen = strlen(member);
1038
1039     assert(name);
1040
1041     header = getByName(name);
1042
1043     String result;
1044
1045     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1046         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1047             result.append(item + mlen + 1, ilen - mlen - 1);
1048             break;
1049         }
1050     }
1051
1052     return result;
1053 }
1054
1055 /*
1056  * returns a the value of the specified list member, if any.
1057  */
1058 String
1059 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1060 {
1061     String header;
1062     const char *pos = NULL;
1063     const char *item;
1064     int ilen;
1065     int mlen = strlen(member);
1066
1067     assert(id >= 0);
1068
1069     header = getStrOrList(id);
1070     String result;
1071
1072     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1073         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1074             result.append(item + mlen + 1, ilen - mlen - 1);
1075             break;
1076         }
1077     }
1078
1079     header.clean();
1080     return result;
1081 }
1082
1083 /* test if a field is present */
1084 int
1085 HttpHeader::has(http_hdr_type id) const
1086 {
1087     assert_eid(id);
1088     assert(id != HDR_OTHER);
1089     debugs(55, 9, this << " lookup for " << id);
1090     return CBIT_TEST(mask, id);
1091 }
1092
1093 void
1094 HttpHeader::putInt(http_hdr_type id, int number)
1095 {
1096     assert_eid(id);
1097     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1098     assert(number >= 0);
1099     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1100 }
1101
1102 void
1103 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1104 {
1105     assert_eid(id);
1106     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1107     assert(number >= 0);
1108     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1109 }
1110
1111 void
1112 HttpHeader::putTime(http_hdr_type id, time_t htime)
1113 {
1114     assert_eid(id);
1115     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1116     assert(htime >= 0);
1117     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1118 }
1119
1120 void
1121 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1122 {
1123     assert_eid(id);
1124     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1125     assert(htime >= 0);
1126     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1127 }
1128
1129 void
1130 HttpHeader::putStr(http_hdr_type id, const char *str)
1131 {
1132     assert_eid(id);
1133     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1134     assert(str);
1135     addEntry(new HttpHeaderEntry(id, NULL, str));
1136 }
1137
1138 void
1139 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1140 {
1141     assert(auth_scheme && realm);
1142     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1143 }
1144
1145 void
1146 HttpHeader::putCc(const HttpHdrCc * cc)
1147 {
1148     MemBuf mb;
1149     Packer p;
1150     assert(cc);
1151     /* remove old directives if any */
1152     delById(HDR_CACHE_CONTROL);
1153     /* pack into mb */
1154     mb.init();
1155     packerToMemInit(&p, &mb);
1156     cc->packInto(&p);
1157     /* put */
1158     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1159     /* cleanup */
1160     packerClean(&p);
1161     mb.clean();
1162 }
1163
1164 void
1165 HttpHeader::putContRange(const HttpHdrContRange * cr)
1166 {
1167     MemBuf mb;
1168     Packer p;
1169     assert(cr);
1170     /* remove old directives if any */
1171     delById(HDR_CONTENT_RANGE);
1172     /* pack into mb */
1173     mb.init();
1174     packerToMemInit(&p, &mb);
1175     httpHdrContRangePackInto(cr, &p);
1176     /* put */
1177     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1178     /* cleanup */
1179     packerClean(&p);
1180     mb.clean();
1181 }
1182
1183 void
1184 HttpHeader::putRange(const HttpHdrRange * range)
1185 {
1186     MemBuf mb;
1187     Packer p;
1188     assert(range);
1189     /* remove old directives if any */
1190     delById(HDR_RANGE);
1191     /* pack into mb */
1192     mb.init();
1193     packerToMemInit(&p, &mb);
1194     range->packInto(&p);
1195     /* put */
1196     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1197     /* cleanup */
1198     packerClean(&p);
1199     mb.clean();
1200 }
1201
1202 void
1203 HttpHeader::putSc(HttpHdrSc *sc)
1204 {
1205     MemBuf mb;
1206     Packer p;
1207     assert(sc);
1208     /* remove old directives if any */
1209     delById(HDR_SURROGATE_CONTROL);
1210     /* pack into mb */
1211     mb.init();
1212     packerToMemInit(&p, &mb);
1213     sc->packInto(&p);
1214     /* put */
1215     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1216     /* cleanup */
1217     packerClean(&p);
1218     mb.clean();
1219 }
1220
1221 void
1222 HttpHeader::putWarning(const int code, const char *const text)
1223 {
1224     char buf[512];
1225     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1226     putStr(HDR_WARNING, buf);
1227 }
1228
1229 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1230 void
1231 HttpHeader::putExt(const char *name, const char *value)
1232 {
1233     assert(name && value);
1234     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1235     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1236 }
1237
1238 int
1239 HttpHeader::getInt(http_hdr_type id) const
1240 {
1241     assert_eid(id);
1242     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1243     HttpHeaderEntry *e;
1244
1245     if ((e = findEntry(id)))
1246         return e->getInt();
1247
1248     return -1;
1249 }
1250
1251 int64_t
1252 HttpHeader::getInt64(http_hdr_type id) const
1253 {
1254     assert_eid(id);
1255     assert(Headers[id].type == ftInt64);        /* must be of an appropriate type */
1256     HttpHeaderEntry *e;
1257
1258     if ((e = findEntry(id)))
1259         return e->getInt64();
1260
1261     return -1;
1262 }
1263
1264 time_t
1265 HttpHeader::getTime(http_hdr_type id) const
1266 {
1267     HttpHeaderEntry *e;
1268     time_t value = -1;
1269     assert_eid(id);
1270     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1271
1272     if ((e = findEntry(id))) {
1273         value = parse_rfc1123(e->value.termedBuf());
1274         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1275     }
1276
1277     return value;
1278 }
1279
1280 /* sync with httpHeaderGetLastStr */
1281 const char *
1282 HttpHeader::getStr(http_hdr_type id) const
1283 {
1284     HttpHeaderEntry *e;
1285     assert_eid(id);
1286     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1287
1288     if ((e = findEntry(id))) {
1289         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1290         return e->value.termedBuf();
1291     }
1292
1293     return NULL;
1294 }
1295
1296 /* unusual */
1297 const char *
1298 HttpHeader::getLastStr(http_hdr_type id) const
1299 {
1300     HttpHeaderEntry *e;
1301     assert_eid(id);
1302     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1303
1304     if ((e = findLastEntry(id))) {
1305         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1306         return e->value.termedBuf();
1307     }
1308
1309     return NULL;
1310 }
1311
1312 HttpHdrCc *
1313 HttpHeader::getCc() const
1314 {
1315     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1316         return NULL;
1317     PROF_start(HttpHeader_getCc);
1318
1319     String s;
1320     getList(HDR_CACHE_CONTROL, &s);
1321
1322     HttpHdrCc *cc=new HttpHdrCc();
1323
1324     if (!cc->parse(s)) {
1325         delete cc;
1326         cc = NULL;
1327     }
1328
1329     HttpHeaderStats[owner].ccParsedCount++;
1330
1331     if (cc)
1332         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1333
1334     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1335
1336     PROF_stop(HttpHeader_getCc);
1337
1338     return cc;
1339 }
1340
1341 HttpHdrRange *
1342 HttpHeader::getRange() const
1343 {
1344     HttpHdrRange *r = NULL;
1345     HttpHeaderEntry *e;
1346     /* some clients will send "Request-Range" _and_ *matching* "Range"
1347      * who knows, some clients might send Request-Range only;
1348      * this "if" should work correctly in both cases;
1349      * hopefully no clients send mismatched headers! */
1350
1351     if ((e = findEntry(HDR_RANGE)) ||
1352             (e = findEntry(HDR_REQUEST_RANGE))) {
1353         r = HttpHdrRange::ParseCreate(&e->value);
1354         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1355     }
1356
1357     return r;
1358 }
1359
1360 HttpHdrSc *
1361 HttpHeader::getSc() const
1362 {
1363     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1364         return NULL;
1365
1366     String s;
1367
1368     (void) getList(HDR_SURROGATE_CONTROL, &s);
1369
1370     HttpHdrSc *sc = httpHdrScParseCreate(s);
1371
1372     ++HttpHeaderStats[owner].ccParsedCount;
1373
1374     if (sc)
1375         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1376
1377     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1378
1379     return sc;
1380 }
1381
1382 HttpHdrContRange *
1383 HttpHeader::getContRange() const
1384 {
1385     HttpHdrContRange *cr = NULL;
1386     HttpHeaderEntry *e;
1387
1388     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1389         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1390         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1391     }
1392
1393     return cr;
1394 }
1395
1396 const char *
1397 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1398 {
1399     const char *field;
1400     int l;
1401     assert(auth_scheme);
1402     field = getStr(id);
1403
1404     if (!field)                 /* no authorization field */
1405         return NULL;
1406
1407     l = strlen(auth_scheme);
1408
1409     if (!l || strncasecmp(field, auth_scheme, l))       /* wrong scheme */
1410         return NULL;
1411
1412     field += l;
1413
1414     if (!xisspace(*field))      /* wrong scheme */
1415         return NULL;
1416
1417     /* skip white space */
1418     field += xcountws(field);
1419
1420     if (!*field)                /* no authorization cookie */
1421         return NULL;
1422
1423     static char decodedAuthToken[8192];
1424     const int decodedLen = base64_decode(decodedAuthToken, sizeof(decodedAuthToken)-1, field);
1425     decodedAuthToken[decodedLen] = '\0';
1426     return decodedAuthToken;
1427 }
1428
1429 ETag
1430 HttpHeader::getETag(http_hdr_type id) const
1431 {
1432     ETag etag = {NULL, -1};
1433     HttpHeaderEntry *e;
1434     assert(Headers[id].type == ftETag);         /* must be of an appropriate type */
1435
1436     if ((e = findEntry(id)))
1437         etagParseInit(&etag, e->value.termedBuf());
1438
1439     return etag;
1440 }
1441
1442 TimeOrTag
1443 HttpHeader::getTimeOrTag(http_hdr_type id) const
1444 {
1445     TimeOrTag tot;
1446     HttpHeaderEntry *e;
1447     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1448     memset(&tot, 0, sizeof(tot));
1449
1450     if ((e = findEntry(id))) {
1451         const char *str = e->value.termedBuf();
1452         /* try as an ETag */
1453
1454         if (etagParseInit(&tot.tag, str)) {
1455             tot.valid = tot.tag.str != NULL;
1456             tot.time = -1;
1457         } else {
1458             /* or maybe it is time? */
1459             tot.time = parse_rfc1123(str);
1460             tot.valid = tot.time >= 0;
1461             tot.tag.str = NULL;
1462         }
1463     }
1464
1465     assert(tot.time < 0 || !tot.tag.str);       /* paranoid */
1466     return tot;
1467 }
1468
1469 /*
1470  * HttpHeaderEntry
1471  */
1472
1473 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1474 {
1475     assert_eid(anId);
1476     id = anId;
1477
1478     if (id != HDR_OTHER)
1479         name = Headers[id].name;
1480     else
1481         name = aName;
1482
1483     value = aValue;
1484
1485     Headers[id].stat.aliveCount++;
1486
1487     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1488 }
1489
1490 HttpHeaderEntry::~HttpHeaderEntry()
1491 {
1492     assert_eid(id);
1493     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1494     /* clean name if needed */
1495
1496     if (id == HDR_OTHER)
1497         name.clean();
1498
1499     value.clean();
1500
1501     assert(Headers[id].stat.aliveCount);
1502
1503     Headers[id].stat.aliveCount--;
1504
1505     id = HDR_BAD_HDR;
1506 }
1507
1508 /* parses and inits header entry, returns true/false */
1509 HttpHeaderEntry *
1510 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1511 {
1512     /* note: name_start == field_start */
1513     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1514     int name_len = name_end ? name_end - field_start :0;
1515     const char *value_start = field_start + name_len + 1;       /* skip ':' */
1516     /* note: value_end == field_end */
1517
1518     HeaderEntryParsedCount++;
1519
1520     /* do we have a valid field name within this field? */
1521
1522     if (!name_len || name_end > field_end)
1523         return NULL;
1524
1525     if (name_len > 65534) {
1526         /* String must be LESS THAN 64K and it adds a terminating NULL */
1527         debugs(55, 1, "WARNING: ignoring header name of " << name_len << " bytes");
1528         return NULL;
1529     }
1530
1531     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1532         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1533                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end) << "'");
1534
1535         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1536             name_len--;
1537
1538         if (!name_len)
1539             return NULL;
1540     }
1541
1542     /* now we know we can parse it */
1543
1544     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end) << "'");
1545
1546     /* is it a "known" field? */
1547     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1548
1549     String name;
1550
1551     String value;
1552
1553     if (id < 0)
1554         id = HDR_OTHER;
1555
1556     assert_eid(id);
1557
1558     /* set field name */
1559     if (id == HDR_OTHER)
1560         name.limitInit(field_start, name_len);
1561     else
1562         name = Headers[id].name;
1563
1564     /* trim field value */
1565     while (value_start < field_end && xisspace(*value_start))
1566         value_start++;
1567
1568     while (value_start < field_end && xisspace(field_end[-1]))
1569         field_end--;
1570
1571     if (field_end - value_start > 65534) {
1572         /* String must be LESS THAN 64K and it adds a terminating NULL */
1573         debugs(55, 1, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1574
1575         if (id == HDR_OTHER)
1576             name.clean();
1577
1578         return NULL;
1579     }
1580
1581     /* set field value */
1582     value.limitInit(value_start, field_end - value_start);
1583
1584     Headers[id].stat.seenCount++;
1585
1586     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1587
1588     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1589 }
1590
1591 HttpHeaderEntry *
1592 HttpHeaderEntry::clone() const
1593 {
1594     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1595 }
1596
1597 void
1598 HttpHeaderEntry::packInto(Packer * p) const
1599 {
1600     assert(p);
1601     packerAppend(p, name.rawBuf(), name.size());
1602     packerAppend(p, ": ", 2);
1603     packerAppend(p, value.rawBuf(), value.size());
1604     packerAppend(p, "\r\n", 2);
1605 }
1606
1607 int
1608 HttpHeaderEntry::getInt() const
1609 {
1610     assert_eid (id);
1611     assert (Headers[id].type == ftInt);
1612     int val = -1;
1613     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1614     httpHeaderNoteParsedEntry(id, value, !ok);
1615     /* XXX: Should we check ok - ie
1616      * return ok ? -1 : value;
1617      */
1618     return val;
1619 }
1620
1621 int64_t
1622 HttpHeaderEntry::getInt64() const
1623 {
1624     assert_eid (id);
1625     assert (Headers[id].type == ftInt64);
1626     int64_t val = -1;
1627     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1628     httpHeaderNoteParsedEntry(id, value, !ok);
1629     /* XXX: Should we check ok - ie
1630      * return ok ? -1 : value;
1631      */
1632     return val;
1633 }
1634
1635 static void
1636 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1637 {
1638     Headers[id].stat.parsCount++;
1639
1640     if (error) {
1641         Headers[id].stat.errCount++;
1642         debugs(55, 2, "cannot parse hdr field: '" << Headers[id].name << ": " << context << "'");
1643     }
1644 }
1645
1646 /*
1647  * Reports
1648  */
1649
1650 /* tmp variable used to pass stat info to dumpers */
1651 extern const HttpHeaderStat *dump_stat;         /* argh! */
1652 const HttpHeaderStat *dump_stat = NULL;
1653
1654 void
1655 httpHeaderFieldStatDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1656 {
1657     const int id = (int) val;
1658     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1659     const char *name = valid_id ? Headers[id].name.termedBuf() : "INVALID";
1660     int visible = count > 0;
1661     /* for entries with zero count, list only those that belong to current type of message */
1662
1663     if (!visible && valid_id && dump_stat->owner_mask)
1664         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1665
1666     if (visible)
1667         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1668                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1669 }
1670
1671 static void
1672 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double size, int count)
1673 {
1674     if (count)
1675         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1676                           idx, (int) val, count,
1677                           xpercent(count, dump_stat->destroyedCount));
1678 }
1679
1680
1681 static void
1682 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1683 {
1684     assert(hs && e);
1685
1686     dump_stat = hs;
1687     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1688     storeAppendPrintf(e, "\nField type distribution\n");
1689     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1690                       "id", "name", "count", "#/header");
1691     statHistDump(&hs->fieldTypeDistr, e, httpHeaderFieldStatDumper);
1692     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1693     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1694                       "id", "name", "count", "#/cc_field");
1695     statHistDump(&hs->ccTypeDistr, e, httpHdrCcStatDumper);
1696     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1697     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1698                       "id", "name", "count", "#/sc_field");
1699     statHistDump(&hs->scTypeDistr, e, httpHdrScStatDumper);
1700     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1701     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1702                       "id", "#flds", "count", "%total");
1703     statHistDump(&hs->hdrUCountDistr, e, httpHeaderFldsPerHdrDumper);
1704     dump_stat = NULL;
1705 }
1706
1707 void
1708 httpHeaderStoreReport(StoreEntry * e)
1709 {
1710     int i;
1711     http_hdr_type ht;
1712     assert(e);
1713
1714     HttpHeaderStats[0].parsedCount =
1715         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1716     HttpHeaderStats[0].ccParsedCount =
1717         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1718     HttpHeaderStats[0].destroyedCount =
1719         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1720     HttpHeaderStats[0].busyDestroyedCount =
1721         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1722
1723     for (i = 1; i < HttpHeaderStatCount; i++) {
1724         httpHeaderStatDump(HttpHeaderStats + i, e);
1725         storeAppendPrintf(e, "%s\n", "<br>");
1726     }
1727
1728     /* field stats for all messages */
1729     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1730
1731     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1732                       "id", "name", "#alive", "%err", "%repeat");
1733
1734     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1735         HttpHeaderFieldInfo *f = Headers + ht;
1736         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1737                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1738                           xpercent(f->stat.errCount, f->stat.parsCount),
1739                           xpercent(f->stat.repCount, f->stat.seenCount));
1740     }
1741
1742     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1743                       HttpHeaderStats[hoRequest].parsedCount,
1744                       HttpHeaderStats[hoReply].parsedCount,
1745                       HttpHeaderStats[0].parsedCount);
1746     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1747 }
1748
1749 http_hdr_type
1750 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1751 {
1752     if (name_len > 0) {
1753         for (int i = 0; i < end; ++i) {
1754             if (name_len != info[i].name.size())
1755                 continue;
1756
1757             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1758                 return info[i].id;
1759         }
1760     }
1761
1762     return HDR_BAD_HDR;
1763 }
1764
1765 http_hdr_type
1766 httpHeaderIdByNameDef(const char *name, int name_len)
1767 {
1768     if (!Headers)
1769         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1770
1771     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1772 }
1773
1774 const char *
1775 httpHeaderNameById(int id)
1776 {
1777     if (!Headers)
1778         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1779
1780     assert(id >= 0 && id < HDR_ENUM_END);
1781
1782     return Headers[id].name.termedBuf();
1783 }
1784
1785 int
1786 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1787 {
1788     int result = 0;
1789     const char *pos = NULL;
1790     const char *item;
1791     int ilen;
1792     int mlen = strlen(member);
1793
1794     assert(id >= 0);
1795
1796     String header (getStrOrList(id));
1797
1798     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1799         if (strncasecmp(item, member, mlen) == 0
1800                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1801             result = 1;
1802             break;
1803         }
1804     }
1805
1806     return result;
1807 }
1808
1809 int
1810 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1811 {
1812     int result = 0;
1813     const char *pos = NULL;
1814     const char *item;
1815     int ilen;
1816     int mlen = strlen(member);
1817
1818     assert(name);
1819
1820     String header (getByName(name));
1821
1822     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1823         if (strncasecmp(item, member, mlen) == 0
1824                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1825             result = 1;
1826             break;
1827         }
1828     }
1829
1830     return result;
1831 }
1832
1833 void
1834 HttpHeader::removeHopByHopEntries()
1835 {
1836     removeConnectionHeaderEntries();
1837
1838     const HttpHeaderEntry *e;
1839     HttpHeaderPos pos = HttpHeaderInitPos;
1840     int headers_deleted = 0;
1841     while ((e = getEntry(&pos))) {
1842         int id = e->id;
1843         if (CBIT_TEST(HopByHopHeadersMask, id)) {
1844             delAt(pos, headers_deleted);
1845             CBIT_CLR(mask, id);
1846         }
1847     }
1848 }
1849
1850 void
1851 HttpHeader::removeConnectionHeaderEntries()
1852 {
1853     if (has(HDR_CONNECTION)) {
1854         /* anything that matches Connection list member will be deleted */
1855         String strConnection;
1856
1857         (void) getList(HDR_CONNECTION, &strConnection);
1858         const HttpHeaderEntry *e;
1859         HttpHeaderPos pos = HttpHeaderInitPos;
1860         /*
1861          * think: on-average-best nesting of the two loops (hdrEntry
1862          * and strListItem) @?@
1863          */
1864         /*
1865          * maybe we should delete standard stuff ("keep-alive","close")
1866          * from strConnection first?
1867          */
1868
1869         int headers_deleted = 0;
1870         while ((e = getEntry(&pos))) {
1871             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
1872                 delAt(pos, headers_deleted);
1873         }
1874         if (headers_deleted)
1875             refreshMask();
1876     }
1877 }