src/HttpHeader.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 55    HTTP Header */
  10
  11 #include "squid.h"
  12 #include "base/LookupTable.h"
  13 #include "base64.h"
  14 #include "globals.h"
  15 #include "HttpHdrCc.h"
  16 #include "HttpHdrContRange.h"
  17 #include "HttpHdrSc.h"
  18 #include "HttpHeader.h"
  19 #include "HttpHeaderFieldInfo.h"
  20 #include "HttpHeaderStat.h"
  21 #include "HttpHeaderTools.h"
  22 #include "MemBuf.h"
  23 #include "mgr/Registration.h"
  24 #include "profiler/Profiler.h"
  25 #include "rfc1123.h"
  26 #include "SquidConfig.h"
  27 #include "SquidString.h"
  28 #include "StatHist.h"
  29 #include "Store.h"
  30 #include "StrList.h"
  31 #include "TimeOrTag.h"
  32 #include "util.h"
  33
  34 #include <algorithm>
  35
  36 /* XXX: the whole set of API managing the entries vector should be rethought
  37  *      after the parse4r-ng effort is complete.
  38  */
  39
  40 /*
  41  * On naming conventions:
  42  *
  43  * HTTP/1.1 defines message-header as
  44  *
  45  * message-header = field-name ":" [ field-value ] CRLF
  46  * field-name     = token
  47  * field-value    = *( field-content | LWS )
  48  *
  49  * HTTP/1.1 does not give a name name a group of all message-headers in a message.
  50  * Squid 1.1 seems to refer to that group _plus_ start-line as "headers".
  51  *
  52  * HttpHeader is an object that represents all message-headers in a message.
  53  * HttpHeader does not manage start-line.
  54  *
  55  * HttpHeader is implemented as a collection of header "entries".
  56  * An entry is a (field_id, field_name, field_value) triplet.
  57  */
  58
  59 /*
  60  * local constants and vars
  61  */
  62
  63 /*
  64  * A table with major attributes for every known field.
  65  * We calculate name lengths and reorganize this array on start up.
  66  * After reorganization, field id can be used as an index to the table.
  67  */
  68 static const HttpHeaderFieldAttrs HeadersAttrs[] = {
  69     HttpHeaderFieldAttrs("Accept", HDR_ACCEPT, ftStr),
  70     HttpHeaderFieldAttrs("Accept-Charset", HDR_ACCEPT_CHARSET, ftStr),
  71     HttpHeaderFieldAttrs("Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr),
  72     HttpHeaderFieldAttrs("Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr),
  73     HttpHeaderFieldAttrs("Accept-Ranges", HDR_ACCEPT_RANGES, ftStr),
  74     HttpHeaderFieldAttrs("Age", HDR_AGE, ftInt),
  75     HttpHeaderFieldAttrs("Allow", HDR_ALLOW, ftStr),
  76     HttpHeaderFieldAttrs("Alternate-Protocol", HDR_ALTERNATE_PROTOCOL, ftStr),
  77     HttpHeaderFieldAttrs("Authorization", HDR_AUTHORIZATION, ftStr),    /* for now */
  78     HttpHeaderFieldAttrs("Cache-Control", HDR_CACHE_CONTROL, ftPCc),
  79     HttpHeaderFieldAttrs("Connection", HDR_CONNECTION, ftStr),
  80     HttpHeaderFieldAttrs("Content-Base", HDR_CONTENT_BASE, ftStr),
  81     HttpHeaderFieldAttrs("Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr),  /* for now */
  82     HttpHeaderFieldAttrs("Content-Encoding", HDR_CONTENT_ENCODING, ftStr),
  83     HttpHeaderFieldAttrs("Content-Language", HDR_CONTENT_LANGUAGE, ftStr),
  84     HttpHeaderFieldAttrs("Content-Length", HDR_CONTENT_LENGTH, ftInt64),
  85     HttpHeaderFieldAttrs("Content-Location", HDR_CONTENT_LOCATION, ftStr),
  86     HttpHeaderFieldAttrs("Content-MD5", HDR_CONTENT_MD5, ftStr),    /* for now */
  87     HttpHeaderFieldAttrs("Content-Range", HDR_CONTENT_RANGE, ftPContRange),
  88     HttpHeaderFieldAttrs("Content-Type", HDR_CONTENT_TYPE, ftStr),
  89     HttpHeaderFieldAttrs("Cookie", HDR_COOKIE, ftStr),
  90     HttpHeaderFieldAttrs("Cookie2", HDR_COOKIE2, ftStr),
  91     HttpHeaderFieldAttrs("Date", HDR_DATE, ftDate_1123),
  92     HttpHeaderFieldAttrs("ETag", HDR_ETAG, ftETag),
  93     HttpHeaderFieldAttrs("Expect", HDR_EXPECT, ftStr),
  94     HttpHeaderFieldAttrs("Expires", HDR_EXPIRES, ftDate_1123),
  95     HttpHeaderFieldAttrs("Forwarded", HDR_FORWARDED, ftStr),
  96     HttpHeaderFieldAttrs("From", HDR_FROM, ftStr),
  97     HttpHeaderFieldAttrs("Host", HDR_HOST, ftStr),
  98     HttpHeaderFieldAttrs("HTTP2-Settings", HDR_HTTP2_SETTINGS, ftStr), /* for now */
  99     HttpHeaderFieldAttrs("If-Match", HDR_IF_MATCH, ftStr),  /* for now */
 100     HttpHeaderFieldAttrs("If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123),
 101     HttpHeaderFieldAttrs("If-None-Match", HDR_IF_NONE_MATCH, ftStr),    /* for now */
 102     HttpHeaderFieldAttrs("If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag),
 103     HttpHeaderFieldAttrs("If-Unmodified-Since", HDR_IF_UNMODIFIED_SINCE, ftDate_1123),
 104     HttpHeaderFieldAttrs("Keep-Alive", HDR_KEEP_ALIVE, ftStr),
 105     HttpHeaderFieldAttrs("Key", HDR_KEY, ftStr),
 106     HttpHeaderFieldAttrs("Last-Modified", HDR_LAST_MODIFIED, ftDate_1123),
 107     HttpHeaderFieldAttrs("Link", HDR_LINK, ftStr),
 108     HttpHeaderFieldAttrs("Location", HDR_LOCATION, ftStr),
 109     HttpHeaderFieldAttrs("Max-Forwards", HDR_MAX_FORWARDS, ftInt64),
 110     HttpHeaderFieldAttrs("Mime-Version", HDR_MIME_VERSION, ftStr),  /* for now */
 111     HttpHeaderFieldAttrs("Negotiate", HDR_NEGOTIATE, ftStr),
 112     HttpHeaderFieldAttrs("Origin", HDR_ORIGIN, ftStr),
 113     HttpHeaderFieldAttrs("Pragma", HDR_PRAGMA, ftStr),
 114     HttpHeaderFieldAttrs("Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr),
 115     HttpHeaderFieldAttrs("Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr),
 116     HttpHeaderFieldAttrs("Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr),
 117     HttpHeaderFieldAttrs("Proxy-Connection", HDR_PROXY_CONNECTION, ftStr),
 118     HttpHeaderFieldAttrs("Proxy-support", HDR_PROXY_SUPPORT, ftStr),
 119     HttpHeaderFieldAttrs("Public", HDR_PUBLIC, ftStr),
 120     HttpHeaderFieldAttrs("Range", HDR_RANGE, ftPRange),
 121     HttpHeaderFieldAttrs("Referer", HDR_REFERER, ftStr),
 122     HttpHeaderFieldAttrs("Request-Range", HDR_REQUEST_RANGE, ftPRange), /* usually matches HDR_RANGE */
 123     HttpHeaderFieldAttrs("Retry-After", HDR_RETRY_AFTER, ftStr),    /* for now (ftDate_1123 or ftInt!) */
 124     HttpHeaderFieldAttrs("Server", HDR_SERVER, ftStr),
 125     HttpHeaderFieldAttrs("Set-Cookie", HDR_SET_COOKIE, ftStr),
 126     HttpHeaderFieldAttrs("Set-Cookie2", HDR_SET_COOKIE2, ftStr),
 127     HttpHeaderFieldAttrs("TE", HDR_TE, ftStr),
 128     HttpHeaderFieldAttrs("Title", HDR_TITLE, ftStr),
 129     HttpHeaderFieldAttrs("Trailer", HDR_TRAILER, ftStr),
 130     HttpHeaderFieldAttrs("Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr),
 131     HttpHeaderFieldAttrs("Translate", HDR_TRANSLATE, ftStr),    /* for now. may need to crop */
 132     HttpHeaderFieldAttrs("Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr),  /* for now ignore. may need to crop */
 133     HttpHeaderFieldAttrs("Upgrade", HDR_UPGRADE, ftStr),    /* for now */
 134     HttpHeaderFieldAttrs("User-Agent", HDR_USER_AGENT, ftStr),
 135     HttpHeaderFieldAttrs("Vary", HDR_VARY, ftStr),  /* for now */
 136     HttpHeaderFieldAttrs("Via", HDR_VIA, ftStr),    /* for now */
 137     HttpHeaderFieldAttrs("Warning", HDR_WARNING, ftStr),    /* for now */
 138     HttpHeaderFieldAttrs("WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr),
 139     HttpHeaderFieldAttrs("Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr),
 140     HttpHeaderFieldAttrs("X-Cache", HDR_X_CACHE, ftStr),
 141     HttpHeaderFieldAttrs("X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr),
 142     HttpHeaderFieldAttrs("X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr),
 143     HttpHeaderFieldAttrs("X-Request-URI", HDR_X_REQUEST_URI, ftStr),
 144     HttpHeaderFieldAttrs("X-Squid-Error", HDR_X_SQUID_ERROR, ftStr),
 145 #if X_ACCELERATOR_VARY
 146     HttpHeaderFieldAttrs("X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr),
 147 #endif
 148 #if USE_ADAPTATION
 149     HttpHeaderFieldAttrs("X-Next-Services", HDR_X_NEXT_SERVICES, ftStr),
 150 #endif
 151     HttpHeaderFieldAttrs("Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr),
 152     HttpHeaderFieldAttrs("Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc),
 153     HttpHeaderFieldAttrs("Front-End-Https", HDR_FRONT_END_HTTPS, ftStr),
 154     HttpHeaderFieldAttrs("FTP-Command", HDR_FTP_COMMAND, ftStr),
 155     HttpHeaderFieldAttrs("FTP-Arguments", HDR_FTP_ARGUMENTS, ftStr),
 156     HttpHeaderFieldAttrs("FTP-Pre", HDR_FTP_PRE, ftStr),
 157     HttpHeaderFieldAttrs("FTP-Status", HDR_FTP_STATUS, ftInt),
 158     HttpHeaderFieldAttrs("FTP-Reason", HDR_FTP_REASON, ftStr),
 159     HttpHeaderFieldAttrs("Other:", HDR_OTHER, ftStr)    /* ':' will not allow matches */
 160 };
 161
 162 struct HeaderTableRecord {
 163     const char *name;
 164     http_hdr_type id;
 165     field_type type;
 166 };
 167 // Note: MUST be sorted by value of http_hdr_type.
 168 // invariant: for each index in headerTable, (int)headerTable[index] = index
 169 static const HeaderTableRecord headerTable[] = {
 170     {"Accept", HDR_ACCEPT, ftStr},
 171     {"Accept-Charset", HDR_ACCEPT_CHARSET, ftStr},
 172     {"Accept-Encoding", HDR_ACCEPT_ENCODING, ftStr},
 173     {"Accept-Language", HDR_ACCEPT_LANGUAGE, ftStr},
 174     {"Accept-Ranges", HDR_ACCEPT_RANGES, ftStr},
 175     {"Age", HDR_AGE, ftInt},
 176     {"Allow", HDR_ALLOW, ftStr},
 177     {"Alternate-Protocol", HDR_ALTERNATE_PROTOCOL, ftStr},
 178     {"Authorization", HDR_AUTHORIZATION, ftStr},    /* for now */
 179     {"Cache-Control", HDR_CACHE_CONTROL, ftPCc},
 180     {"Connection", HDR_CONNECTION, ftStr},
 181     {"Content-Base", HDR_CONTENT_BASE, ftStr},
 182     {"Content-Disposition", HDR_CONTENT_DISPOSITION, ftStr},  /* for now */
 183     {"Content-Encoding", HDR_CONTENT_ENCODING, ftStr},
 184     {"Content-Language", HDR_CONTENT_LANGUAGE, ftStr},
 185     {"Content-Length", HDR_CONTENT_LENGTH, ftInt64},
 186     {"Content-Location", HDR_CONTENT_LOCATION, ftStr},
 187     {"Content-MD5", HDR_CONTENT_MD5, ftStr},    /* for now */
 188     {"Content-Range", HDR_CONTENT_RANGE, ftPContRange},
 189     {"Content-Type", HDR_CONTENT_TYPE, ftStr},
 190     {"Cookie", HDR_COOKIE, ftStr},
 191     {"Cookie2", HDR_COOKIE2, ftStr},
 192     {"Date", HDR_DATE, ftDate_1123},
 193     {"ETag", HDR_ETAG, ftETag},
 194     {"Expect", HDR_EXPECT, ftStr},
 195     {"Expires", HDR_EXPIRES, ftDate_1123},
 196     {"Forwarded", HDR_FORWARDED, ftStr},
 197     {"From", HDR_FROM, ftStr},
 198     {"Host", HDR_HOST, ftStr},
 199     {"HTTP2-Settings", HDR_HTTP2_SETTINGS, ftStr}, /* for now */
 200     {"If-Match", HDR_IF_MATCH, ftStr},  /* for now */
 201     {"If-Modified-Since", HDR_IF_MODIFIED_SINCE, ftDate_1123},
 202     {"If-None-Match", HDR_IF_NONE_MATCH, ftStr},    /* for now */
 203     {"If-Range", HDR_IF_RANGE, ftDate_1123_or_ETag},
 204     {"If-Unmodified-Since", HDR_IF_UNMODIFIED_SINCE, ftDate_1123},
 205     {"Keep-Alive", HDR_KEEP_ALIVE, ftStr},
 206     {"Key", HDR_KEY, ftStr},
 207     {"Last-Modified", HDR_LAST_MODIFIED, ftDate_1123},
 208     {"Link", HDR_LINK, ftStr},
 209     {"Location", HDR_LOCATION, ftStr},
 210     {"Max-Forwards", HDR_MAX_FORWARDS, ftInt64},
 211     {"Mime-Version", HDR_MIME_VERSION, ftStr},  /* for now */
 212     {"Negotiate", HDR_NEGOTIATE, ftStr},
 213     {"Origin", HDR_ORIGIN, ftStr},
 214     {"Pragma", HDR_PRAGMA, ftStr},
 215     {"Proxy-Authenticate", HDR_PROXY_AUTHENTICATE, ftStr},
 216     {"Proxy-Authentication-Info", HDR_PROXY_AUTHENTICATION_INFO, ftStr},
 217     {"Proxy-Authorization", HDR_PROXY_AUTHORIZATION, ftStr},
 218     {"Proxy-Connection", HDR_PROXY_CONNECTION, ftStr},
 219     {"Proxy-support", HDR_PROXY_SUPPORT, ftStr},
 220     {"Public", HDR_PUBLIC, ftStr},
 221     {"Range", HDR_RANGE, ftPRange},
 222     {"Referer", HDR_REFERER, ftStr},
 223     {"Request-Range", HDR_REQUEST_RANGE, ftPRange}, /* usually matches HDR_RANGE */
 224     {"Retry-After", HDR_RETRY_AFTER, ftStr},    /* for now (ftDate_1123 or ftInt!} */
 225     {"Server", HDR_SERVER, ftStr},
 226     {"Set-Cookie", HDR_SET_COOKIE, ftStr},
 227     {"Set-Cookie2", HDR_SET_COOKIE2, ftStr},
 228     {"TE", HDR_TE, ftStr},
 229     {"Title", HDR_TITLE, ftStr},
 230     {"Trailer", HDR_TRAILER, ftStr},
 231     {"Transfer-Encoding", HDR_TRANSFER_ENCODING, ftStr},
 232     {"Translate", HDR_TRANSLATE, ftStr},    /* for now. may need to crop */
 233     {"Unless-Modified-Since", HDR_UNLESS_MODIFIED_SINCE, ftStr},  /* for now ignore. may need to crop */
 234     {"Upgrade", HDR_UPGRADE, ftStr},    /* for now */
 235     {"User-Agent", HDR_USER_AGENT, ftStr},
 236     {"Vary", HDR_VARY, ftStr},  /* for now */
 237     {"Via", HDR_VIA, ftStr},    /* for now */
 238     {"Warning", HDR_WARNING, ftStr},    /* for now */
 239     {"WWW-Authenticate", HDR_WWW_AUTHENTICATE, ftStr},
 240     {"Authentication-Info", HDR_AUTHENTICATION_INFO, ftStr},
 241     {"X-Cache", HDR_X_CACHE, ftStr},
 242     {"X-Cache-Lookup", HDR_X_CACHE_LOOKUP, ftStr},
 243     {"X-Forwarded-For", HDR_X_FORWARDED_FOR, ftStr},
 244     {"X-Request-URI", HDR_X_REQUEST_URI, ftStr},
 245     {"X-Squid-Error", HDR_X_SQUID_ERROR, ftStr},
 246 #if X_ACCELERATOR_VARY
 247     {"X-Accelerator-Vary", HDR_X_ACCELERATOR_VARY, ftStr},
 248 #endif
 249 #if USE_ADAPTATION
 250     {"X-Next-Services", HDR_X_NEXT_SERVICES, ftStr},
 251 #endif
 252     {"Surrogate-Capability", HDR_SURROGATE_CAPABILITY, ftStr},
 253     {"Surrogate-Control", HDR_SURROGATE_CONTROL, ftPSc},
 254     {"Front-End-Https", HDR_FRONT_END_HTTPS, ftStr},
 255     {"FTP-Command", HDR_FTP_COMMAND, ftStr},
 256     {"FTP-Arguments", HDR_FTP_ARGUMENTS, ftStr},
 257     {"FTP-Pre", HDR_FTP_PRE, ftStr},
 258     {"FTP-Status", HDR_FTP_STATUS, ftInt},
 259     {"FTP-Reason", HDR_FTP_REASON, ftStr},
 260     {nullptr, HDR_OTHER}    /* ':' will not allow matches */
 261 };
 262
 263 static HttpHeaderFieldInfo *Headers = NULL;
 264 LookupTable<http_hdr_type, HeaderTableRecord> headerLookupTable(HDR_OTHER, headerTable);
 265 std::vector<HttpHeaderFieldStat> headerStatsTable(HDR_OTHER);
 266
 267 http_hdr_type &operator++ (http_hdr_type &aHeader)
 268 {
 269     int tmp = (int)aHeader;
 270     aHeader = (http_hdr_type)(++tmp);
 271     return aHeader;
 272 }
 273
 274 /*
 275  * headers with field values defined as #(values) in HTTP/1.1
 276  * Headers that are currently not recognized, are commented out.
 277  */
 278 static HttpHeaderMask ListHeadersMask;  /* set run-time using  ListHeadersArr */
 279 static http_hdr_type ListHeadersArr[] = {
 280     HDR_ACCEPT,
 281     HDR_ACCEPT_CHARSET,
 282     HDR_ACCEPT_ENCODING,
 283     HDR_ACCEPT_LANGUAGE,
 284     HDR_ACCEPT_RANGES,
 285     HDR_ALLOW,
 286     HDR_CACHE_CONTROL,
 287     HDR_CONTENT_ENCODING,
 288     HDR_CONTENT_LANGUAGE,
 289     HDR_CONNECTION,
 290     HDR_EXPECT,
 291     HDR_IF_MATCH,
 292     HDR_IF_NONE_MATCH,
 293     HDR_KEY,
 294     HDR_LINK,
 295     HDR_PRAGMA,
 296     HDR_PROXY_CONNECTION,
 297     HDR_PROXY_SUPPORT,
 298     HDR_TRANSFER_ENCODING,
 299     HDR_UPGRADE,
 300     HDR_VARY,
 301     HDR_VIA,
 302     HDR_WARNING,
 303     HDR_WWW_AUTHENTICATE,
 304     HDR_AUTHENTICATION_INFO,
 305     HDR_PROXY_AUTHENTICATION_INFO,
 306     /* HDR_TE, HDR_TRAILER */
 307 #if X_ACCELERATOR_VARY
 308     HDR_X_ACCELERATOR_VARY,
 309 #endif
 310 #if USE_ADAPTATION
 311     HDR_X_NEXT_SERVICES,
 312 #endif
 313     HDR_SURROGATE_CAPABILITY,
 314     HDR_SURROGATE_CONTROL,
 315     HDR_FORWARDED,
 316     HDR_X_FORWARDED_FOR
 317 };
 318
 319 /* general-headers */
 320 static http_hdr_type GeneralHeadersArr[] = {
 321     HDR_CACHE_CONTROL,
 322     HDR_CONNECTION,
 323     HDR_DATE,
 324     HDR_FORWARDED,
 325     HDR_X_FORWARDED_FOR,
 326     HDR_MIME_VERSION,
 327     HDR_PRAGMA,
 328     HDR_PROXY_CONNECTION,
 329     HDR_TRANSFER_ENCODING,
 330     HDR_UPGRADE,
 331     /* HDR_TRAILER, */
 332     HDR_VIA,
 333 };
 334
 335 /* entity-headers */
 336 static http_hdr_type EntityHeadersArr[] = {
 337     HDR_ALLOW,
 338     HDR_CONTENT_BASE,
 339     HDR_CONTENT_ENCODING,
 340     HDR_CONTENT_LANGUAGE,
 341     HDR_CONTENT_LENGTH,
 342     HDR_CONTENT_LOCATION,
 343     HDR_CONTENT_MD5,
 344     HDR_CONTENT_RANGE,
 345     HDR_CONTENT_TYPE,
 346     HDR_ETAG,
 347     HDR_EXPIRES,
 348     HDR_LAST_MODIFIED,
 349     HDR_LINK,
 350     HDR_OTHER
 351 };
 352
 353 /* request-only headers */
 354 static HttpHeaderMask RequestHeadersMask;   /* set run-time using RequestHeaders */
 355 static http_hdr_type RequestHeadersArr[] = {
 356     HDR_ACCEPT,
 357     HDR_ACCEPT_CHARSET,
 358     HDR_ACCEPT_ENCODING,
 359     HDR_ACCEPT_LANGUAGE,
 360     HDR_AUTHORIZATION,
 361     HDR_EXPECT,
 362     HDR_FROM,
 363     HDR_HOST,
 364     HDR_HTTP2_SETTINGS,
 365     HDR_IF_MATCH,
 366     HDR_IF_MODIFIED_SINCE,
 367     HDR_IF_NONE_MATCH,
 368     HDR_IF_RANGE,
 369     HDR_IF_UNMODIFIED_SINCE,
 370     HDR_MAX_FORWARDS,
 371     HDR_ORIGIN,
 372     HDR_PROXY_AUTHORIZATION,
 373     HDR_RANGE,
 374     HDR_REFERER,
 375     HDR_REQUEST_RANGE,
 376     HDR_TE,
 377     HDR_USER_AGENT,
 378     HDR_SURROGATE_CAPABILITY
 379 };
 380
 381 /* reply-only headers */
 382 static HttpHeaderMask ReplyHeadersMask;     /* set run-time using ReplyHeaders */
 383 static http_hdr_type ReplyHeadersArr[] = {
 384     HDR_ACCEPT_ENCODING,
 385     HDR_ACCEPT_RANGES,
 386     HDR_AGE,
 387     HDR_KEY,
 388     HDR_LOCATION,
 389     HDR_PROXY_AUTHENTICATE,
 390     HDR_PUBLIC,
 391     HDR_RETRY_AFTER,
 392     HDR_SERVER,
 393     HDR_SET_COOKIE,
 394     HDR_SET_COOKIE2,
 395     HDR_VARY,
 396     HDR_WARNING,
 397     HDR_WWW_AUTHENTICATE,
 398     HDR_X_CACHE,
 399     HDR_X_CACHE_LOOKUP,
 400     HDR_X_REQUEST_URI,
 401 #if X_ACCELERATOR_VARY
 402     HDR_X_ACCELERATOR_VARY,
 403 #endif
 404 #if USE_ADAPTATION
 405     HDR_X_NEXT_SERVICES,
 406 #endif
 407     HDR_X_SQUID_ERROR,
 408     HDR_SURROGATE_CONTROL
 409 };
 410
 411 /* hop-by-hop headers */
 412 static HttpHeaderMask HopByHopHeadersMask;
 413 static http_hdr_type HopByHopHeadersArr[] = {
 414     HDR_ALTERNATE_PROTOCOL,
 415     HDR_CONNECTION,
 416     HDR_HTTP2_SETTINGS,
 417     HDR_KEEP_ALIVE,
 418     /*HDR_PROXY_AUTHENTICATE, // removal handled specially for peer login */
 419     HDR_PROXY_AUTHORIZATION,
 420     HDR_TE,
 421     HDR_TRAILER,
 422     HDR_TRANSFER_ENCODING,
 423     HDR_UPGRADE,
 424     HDR_PROXY_CONNECTION
 425 };
 426
 427 /* header accounting */
 428 // NP: keep in sync with enum http_hdr_owner_type
 429 static HttpHeaderStat HttpHeaderStats[] = {
 430     HttpHeaderStat(/*hoNone*/ "all", NULL),
 431 #if USE_HTCP
 432     HttpHeaderStat(/*hoHtcpReply*/ "HTCP reply", &ReplyHeadersMask),
 433 #endif
 434     HttpHeaderStat(/*hoRequest*/ "request", &RequestHeadersMask),
 435     HttpHeaderStat(/*hoReply*/ "reply", &ReplyHeadersMask)
 436 #if USE_OPENSSL
 437     /* hoErrorDetail */
 438 #endif
 439     /* hoEnd */
 440 };
 441 static int HttpHeaderStatCount = countof(HttpHeaderStats);
 442
 443 static int HeaderEntryParsedCount = 0;
 444
 445 /*
 446  * forward declarations and local routines
 447  */
 448
 449 class StoreEntry;
 450 #define assert_eid(id) assert((id) >= 0 && (id) < HDR_ENUM_END)
 451
 452 static void httpHeaderNoteParsedEntry(http_hdr_type id, String const &value, int error);
 453
 454 static void httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e);
 455
 456 /** store report about current header usage and other stats */
 457 static void httpHeaderStoreReport(StoreEntry * e);
 458
 459 /*
 460  * Module initialization routines
 461  */
 462
 463 static void
 464 httpHeaderRegisterWithCacheManager(void)
 465 {
 466     Mgr::RegisterAction("http_headers",
 467                         "HTTP Header Statistics",
 468                         httpHeaderStoreReport, 0, 1);
 469 }
 470
 471 void
 472 httpHeaderInitModule(void)
 473 {
 474     /* check that we have enough space for masks */
 475     assert(8 * sizeof(HttpHeaderMask) >= HDR_ENUM_END);
 476     /* all headers must be described */
 477     assert(countof(HeadersAttrs) == HDR_ENUM_END);
 478
 479     if (!Headers)
 480         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
 481     // use headerLookupTable in place of Headers
 482
 483     /* create masks */
 484     httpHeaderMaskInit(&ListHeadersMask, 0);
 485     httpHeaderCalcMask(&ListHeadersMask, ListHeadersArr, countof(ListHeadersArr));
 486
 487     httpHeaderMaskInit(&ReplyHeadersMask, 0);
 488     httpHeaderCalcMask(&ReplyHeadersMask, ReplyHeadersArr, countof(ReplyHeadersArr));
 489     httpHeaderCalcMask(&ReplyHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 490     httpHeaderCalcMask(&ReplyHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 491
 492     httpHeaderMaskInit(&RequestHeadersMask, 0);
 493     httpHeaderCalcMask(&RequestHeadersMask, RequestHeadersArr, countof(RequestHeadersArr));
 494     httpHeaderCalcMask(&RequestHeadersMask, GeneralHeadersArr, countof(GeneralHeadersArr));
 495     httpHeaderCalcMask(&RequestHeadersMask, EntityHeadersArr, countof(EntityHeadersArr));
 496
 497     httpHeaderMaskInit(&HopByHopHeadersMask, 0);
 498     httpHeaderCalcMask(&HopByHopHeadersMask, HopByHopHeadersArr, countof(HopByHopHeadersArr));
 499
 500     /* header stats initialized by class constructor */
 501     assert(HttpHeaderStatCount == hoReply + 1);
 502
 503     /* init dependent modules */
 504     httpHdrCcInitModule();
 505     httpHdrScInitModule();
 506
 507     httpHeaderRegisterWithCacheManager();
 508 }
 509
 510 void
 511 httpHeaderCleanModule(void)
 512 {
 513     httpHeaderDestroyFieldsInfo(Headers, HDR_ENUM_END);
 514     Headers = NULL;
 515     httpHdrCcCleanModule();
 516     httpHdrScCleanModule();
 517 }
 518
 519 /*
 520  * HttpHeader Implementation
 521  */
 522
 523 HttpHeader::HttpHeader() : owner (hoNone), len (0)
 524 {
 525     httpHeaderMaskInit(&mask, 0);
 526 }
 527
 528 HttpHeader::HttpHeader(const http_hdr_owner_type anOwner): owner(anOwner), len(0)
 529 {
 530     assert(anOwner > hoNone && anOwner < hoEnd);
 531     debugs(55, 7, "init-ing hdr: " << this << " owner: " << owner);
 532     httpHeaderMaskInit(&mask, 0);
 533 }
 534
 535 HttpHeader::HttpHeader(const HttpHeader &other): owner(other.owner), len(other.len)
 536 {
 537     httpHeaderMaskInit(&mask, 0);
 538     update(&other, NULL); // will update the mask as well
 539 }
 540
 541 HttpHeader::~HttpHeader()
 542 {
 543     clean();
 544 }
 545
 546 HttpHeader &
 547 HttpHeader::operator =(const HttpHeader &other)
 548 {
 549     if (this != &other) {
 550         // we do not really care, but the caller probably does
 551         assert(owner == other.owner);
 552         clean();
 553         update(&other, NULL); // will update the mask as well
 554         len = other.len;
 555     }
 556     return *this;
 557 }
 558
 559 void
 560 HttpHeader::clean()
 561 {
 562
 563     assert(owner > hoNone && owner < hoEnd);
 564     debugs(55, 7, "cleaning hdr: " << this << " owner: " << owner);
 565
 566     PROF_start(HttpHeaderClean);
 567
 568     if (owner <= hoReply) {
 569         /*
 570          * An unfortunate bug.  The entries array is initialized
 571          * such that count is set to zero.  httpHeaderClean() seems to
 572          * be called both when 'hdr' is created, and destroyed.  Thus,
 573          * we accumulate a large number of zero counts for 'hdr' before
 574          * it is ever used.  Can't think of a good way to fix it, except
 575          * adding a state variable that indicates whether or not 'hdr'
 576          * has been used.  As a hack, just never count zero-sized header
 577          * arrays.
 578          */
 579         if (!entries.empty())
 580             HttpHeaderStats[owner].hdrUCountDistr.count(entries.size());
 581
 582         ++ HttpHeaderStats[owner].destroyedCount;
 583
 584         HttpHeaderStats[owner].busyDestroyedCount += entries.size() > 0;
 585     } // if (owner <= hoReply)
 586
 587     for (std::vector<HttpHeaderEntry *>::iterator i = entries.begin(); i != entries.end(); ++i) {
 588         HttpHeaderEntry *e = *i;
 589         if (e == NULL)
 590             continue;
 591         if (e->id < 0 || e->id >= HDR_ENUM_END) {
 592             debugs(55, DBG_CRITICAL, "BUG: invalid entry (" << e->id << "). Ignored.");
 593         } else {
 594             if (owner <= hoReply)
 595                 HttpHeaderStats[owner].fieldTypeDistr.count(e->id);
 596             delete e;
 597         }
 598     }
 599
 600     entries.clear();
 601     httpHeaderMaskInit(&mask, 0);
 602     len = 0;
 603     PROF_stop(HttpHeaderClean);
 604 }
 605
 606 /* append entries (also see httpHeaderUpdate) */
 607 void
 608 HttpHeader::append(const HttpHeader * src)
 609 {
 610     const HttpHeaderEntry *e;
 611     HttpHeaderPos pos = HttpHeaderInitPos;
 612     assert(src);
 613     assert(src != this);
 614     debugs(55, 7, "appending hdr: " << this << " += " << src);
 615
 616     while ((e = src->getEntry(&pos))) {
 617         addEntry(e->clone());
 618     }
 619 }
 620
 621 /* use fresh entries to replace old ones */
 622 void
 623 httpHeaderUpdate(HttpHeader * old, const HttpHeader * fresh, const HttpHeaderMask * denied_mask)
 624 {
 625     assert (old);
 626     old->update (fresh, denied_mask);
 627 }
 628
 629 void
 630 HttpHeader::update (HttpHeader const *fresh, HttpHeaderMask const *denied_mask)
 631 {
 632     const HttpHeaderEntry *e;
 633     HttpHeaderPos pos = HttpHeaderInitPos;
 634     assert(fresh);
 635     assert(this != fresh);
 636
 637     while ((e = fresh->getEntry(&pos))) {
 638         /* deny bad guys (ok to check for HDR_OTHER) here */
 639
 640         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 641             continue;
 642
 643         if (e->id != HDR_OTHER)
 644             delById(e->id);
 645         else
 646             delByName(e->name.termedBuf());
 647     }
 648
 649     pos = HttpHeaderInitPos;
 650     while ((e = fresh->getEntry(&pos))) {
 651         /* deny bad guys (ok to check for HDR_OTHER) here */
 652
 653         if (denied_mask && CBIT_TEST(*denied_mask, e->id))
 654             continue;
 655
 656         debugs(55, 7, "Updating header '" << HeadersAttrs[e->id].name << "' in cached entry");
 657
 658         addEntry(e->clone());
 659     }
 660 }
 661
 662 /* just handy in parsing: resets and returns false */
 663 int
 664 HttpHeader::reset()
 665 {
 666     clean();
 667     return 0;
 668 }
 669
 670 int
 671 HttpHeader::parse(const char *header_start, size_t hdrLen)
 672 {
 673     const char *field_ptr = header_start;
 674     const char *header_end = header_start + hdrLen; // XXX: remove
 675     HttpHeaderEntry *e, *e2;
 676     int warnOnError = (Config.onoff.relaxed_header_parser <= 0 ? DBG_IMPORTANT : 2);
 677
 678     PROF_start(HttpHeaderParse);
 679
 680     assert(header_start && header_end);
 681     debugs(55, 7, "parsing hdr: (" << this << ")" << std::endl << getStringPrefix(header_start, hdrLen));
 682     ++ HttpHeaderStats[owner].parsedCount;
 683
 684     char *nulpos;
 685     if ((nulpos = (char*)memchr(header_start, '\0', hdrLen))) {
 686         debugs(55, DBG_IMPORTANT, "WARNING: HTTP header contains NULL characters {" <<
 687                getStringPrefix(header_start, nulpos-header_start) << "}\nNULL\n{" << getStringPrefix(nulpos+1, hdrLen-(nulpos-header_start)-1));
 688         PROF_stop(HttpHeaderParse);
 689         return reset();
 690     }
 691
 692     /* common format headers are "<name>:[ws]<value>" lines delimited by <CRLF>.
 693      * continuation lines start with a (single) space or tab */
 694     while (field_ptr < header_end) {
 695         const char *field_start = field_ptr;
 696         const char *field_end;
 697
 698         do {
 699             const char *this_line = field_ptr;
 700             field_ptr = (const char *)memchr(field_ptr, '\n', header_end - field_ptr);
 701
 702             if (!field_ptr) {
 703                 // missing <LF>
 704                 PROF_stop(HttpHeaderParse);
 705                 return reset();
 706             }
 707
 708             field_end = field_ptr;
 709
 710             ++field_ptr;    /* Move to next line */
 711
 712             if (field_end > this_line && field_end[-1] == '\r') {
 713                 --field_end;    /* Ignore CR LF */
 714
 715                 if (owner == hoRequest && field_end > this_line) {
 716                     bool cr_only = true;
 717                     for (const char *p = this_line; p < field_end && cr_only; ++p) {
 718                         if (*p != '\r')
 719                             cr_only = false;
 720                     }
 721                     if (cr_only) {
 722                         debugs(55, DBG_IMPORTANT, "SECURITY WARNING: Rejecting HTTP request with a CR+ "
 723                                "header field to prevent request smuggling attacks: {" <<
 724                                getStringPrefix(header_start, hdrLen) << "}");
 725                         PROF_stop(HttpHeaderParse);
 726                         return reset();
 727                     }
 728                 }
 729             }
 730
 731             /* Barf on stray CR characters */
 732             if (memchr(this_line, '\r', field_end - this_line)) {
 733                 debugs(55, warnOnError, "WARNING: suspicious CR characters in HTTP header {" <<
 734                        getStringPrefix(field_start, field_end-field_start) << "}");
 735
 736                 if (Config.onoff.relaxed_header_parser) {
 737                     char *p = (char *) this_line;   /* XXX Warning! This destroys original header content and violates specifications somewhat */
 738
 739                     while ((p = (char *)memchr(p, '\r', field_end - p)) != NULL) {
 740                         *p = ' ';
 741                         ++p;
 742                     }
 743                 } else {
 744                     PROF_stop(HttpHeaderParse);
 745                     return reset();
 746                 }
 747             }
 748
 749             if (this_line + 1 == field_end && this_line > field_start) {
 750                 debugs(55, warnOnError, "WARNING: Blank continuation line in HTTP header {" <<
 751                        getStringPrefix(header_start, hdrLen) << "}");
 752                 PROF_stop(HttpHeaderParse);
 753                 return reset();
 754             }
 755         } while (field_ptr < header_end && (*field_ptr == ' ' || *field_ptr == '\t'));
 756
 757         if (field_start == field_end) {
 758             if (field_ptr < header_end) {
 759                 debugs(55, warnOnError, "WARNING: unparseable HTTP header field near {" <<
 760                        getStringPrefix(field_start, hdrLen-(field_start-header_start)) << "}");
 761                 PROF_stop(HttpHeaderParse);
 762                 return reset();
 763             }
 764
 765             break;      /* terminating blank line */
 766         }
 767
 768         if ((e = HttpHeaderEntry::parse(field_start, field_end)) == NULL) {
 769             debugs(55, warnOnError, "WARNING: unparseable HTTP header field {" <<
 770                    getStringPrefix(field_start, field_end-field_start) << "}");
 771             debugs(55, warnOnError, " in {" << getStringPrefix(header_start, hdrLen) << "}");
 772
 773             if (Config.onoff.relaxed_header_parser)
 774                 continue;
 775
 776             PROF_stop(HttpHeaderParse);
 777             return reset();
 778         }
 779
 780         if (e->id == HDR_CONTENT_LENGTH && (e2 = findEntry(e->id)) != NULL) {
 781             if (e->value != e2->value) {
 782                 int64_t l1, l2;
 783                 debugs(55, warnOnError, "WARNING: found two conflicting content-length headers in {" <<
 784                        getStringPrefix(header_start, hdrLen) << "}");
 785
 786                 if (!Config.onoff.relaxed_header_parser) {
 787                     delete e;
 788                     PROF_stop(HttpHeaderParse);
 789                     return reset();
 790                 }
 791
 792                 if (!httpHeaderParseOffset(e->value.termedBuf(), &l1)) {
 793                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e->value << "'");
 794                     delete e;
 795                     continue;
 796                 } else if (!httpHeaderParseOffset(e2->value.termedBuf(), &l2)) {
 797                     debugs(55, DBG_IMPORTANT, "WARNING: Unparseable content-length '" << e2->value << "'");
 798                     delById(e2->id);
 799                 } else if (l1 > l2) {
 800                     delById(e2->id);
 801                 } else {
 802                     delete e;
 803                     continue;
 804                 }
 805             } else {
 806                 debugs(55, warnOnError, "NOTICE: found double content-length header");
 807                 delete e;
 808
 809                 if (Config.onoff.relaxed_header_parser)
 810                     continue;
 811
 812                 PROF_stop(HttpHeaderParse);
 813                 return reset();
 814             }
 815         }
 816
 817         if (e->id == HDR_OTHER && stringHasWhitespace(e->name.termedBuf())) {
 818             debugs(55, warnOnError, "WARNING: found whitespace in HTTP header name {" <<
 819                    getStringPrefix(field_start, field_end-field_start) << "}");
 820
 821             if (!Config.onoff.relaxed_header_parser) {
 822                 delete e;
 823                 PROF_stop(HttpHeaderParse);
 824                 return reset();
 825             }
 826         }
 827
 828         addEntry(e);
 829     }
 830
 831     if (chunked()) {
 832         // RFC 2616 section 4.4: ignore Content-Length with Transfer-Encoding
 833         delById(HDR_CONTENT_LENGTH);
 834     }
 835
 836     PROF_stop(HttpHeaderParse);
 837     return 1;           /* even if no fields where found, it is a valid header */
 838 }
 839
 840 /* packs all the entries using supplied packer */
 841 void
 842 HttpHeader::packInto(Packable * p, bool mask_sensitive_info) const
 843 {
 844     HttpHeaderPos pos = HttpHeaderInitPos;
 845     const HttpHeaderEntry *e;
 846     assert(p);
 847     debugs(55, 7, this << " into " << p <<
 848            (mask_sensitive_info ? " while masking" : ""));
 849     /* pack all entries one by one */
 850     while ((e = getEntry(&pos))) {
 851         if (!mask_sensitive_info) {
 852             e->packInto(p);
 853             continue;
 854         }
 855
 856         bool maskThisEntry = false;
 857         switch (e->id) {
 858         case HDR_AUTHORIZATION:
 859         case HDR_PROXY_AUTHORIZATION:
 860             maskThisEntry = true;
 861             break;
 862
 863         case HDR_FTP_ARGUMENTS:
 864             if (const HttpHeaderEntry *cmd = findEntry(HDR_FTP_COMMAND))
 865                 maskThisEntry = (cmd->value == "PASS");
 866             break;
 867
 868         default:
 869             break;
 870         }
 871         if (maskThisEntry) {
 872             p->append(e->name.rawBuf(), e->name.size());
 873             p->append(": ** NOT DISPLAYED **\r\n", 23);
 874         } else {
 875             e->packInto(p);
 876         }
 877
 878     }
 879     /* Pack in the "special" entries */
 880
 881     /* Cache-Control */
 882 }
 883
 884 /* returns next valid entry */
 885 HttpHeaderEntry *
 886 HttpHeader::getEntry(HttpHeaderPos * pos) const
 887 {
 888     assert(pos);
 889     assert(*pos >= HttpHeaderInitPos && *pos < static_cast<ssize_t>(entries.size()));
 890
 891     for (++(*pos); *pos < static_cast<ssize_t>(entries.size()); ++(*pos)) {
 892         if (entries[*pos])
 893             return static_cast<HttpHeaderEntry*>(entries[*pos]);
 894     }
 895
 896     return NULL;
 897 }
 898
 899 /*
 900  * returns a pointer to a specified entry if any
 901  * note that we return one entry so it does not make much sense to ask for
 902  * "list" headers
 903  */
 904 HttpHeaderEntry *
 905 HttpHeader::findEntry(http_hdr_type id) const
 906 {
 907     HttpHeaderPos pos = HttpHeaderInitPos;
 908     HttpHeaderEntry *e;
 909     assert_eid(id);
 910     assert(!CBIT_TEST(ListHeadersMask, id));
 911
 912     /* check mask first */
 913
 914     if (!CBIT_TEST(mask, id))
 915         return NULL;
 916
 917     /* looks like we must have it, do linear search */
 918     while ((e = getEntry(&pos))) {
 919         if (e->id == id)
 920             return e;
 921     }
 922
 923     /* hm.. we thought it was there, but it was not found */
 924     assert(0);
 925
 926     return NULL;        /* not reached */
 927 }
 928
 929 /*
 930  * same as httpHeaderFindEntry
 931  */
 932 HttpHeaderEntry *
 933 HttpHeader::findLastEntry(http_hdr_type id) const
 934 {
 935     HttpHeaderPos pos = HttpHeaderInitPos;
 936     HttpHeaderEntry *e;
 937     HttpHeaderEntry *result = NULL;
 938     assert_eid(id);
 939     assert(!CBIT_TEST(ListHeadersMask, id));
 940
 941     /* check mask first */
 942
 943     if (!CBIT_TEST(mask, id))
 944         return NULL;
 945
 946     /* looks like we must have it, do linear search */
 947     while ((e = getEntry(&pos))) {
 948         if (e->id == id)
 949             result = e;
 950     }
 951
 952     assert(result);     /* must be there! */
 953     return result;
 954 }
 955
 956 /*
 957  * deletes all fields with a given name if any, returns #fields deleted;
 958  */
 959 int
 960 HttpHeader::delByName(const char *name)
 961 {
 962     int count = 0;
 963     HttpHeaderPos pos = HttpHeaderInitPos;
 964     HttpHeaderEntry *e;
 965     httpHeaderMaskInit(&mask, 0);   /* temporal inconsistency */
 966     debugs(55, 9, "deleting '" << name << "' fields in hdr " << this);
 967
 968     while ((e = getEntry(&pos))) {
 969         if (!e->name.caseCmp(name))
 970             delAt(pos, count);
 971         else
 972             CBIT_SET(mask, e->id);
 973     }
 974
 975     return count;
 976 }
 977
 978 /* deletes all entries with a given id, returns the #entries deleted */
 979 int
 980 HttpHeader::delById(http_hdr_type id)
 981 {
 982     int count = 0;
 983     HttpHeaderPos pos = HttpHeaderInitPos;
 984     HttpHeaderEntry *e;
 985     debugs(55, 8, this << " del-by-id " << id);
 986     assert_eid(id);
 987     assert(id != HDR_OTHER);        /* does not make sense */
 988
 989     if (!CBIT_TEST(mask, id))
 990         return 0;
 991
 992     while ((e = getEntry(&pos))) {
 993         if (e->id == id)
 994             delAt(pos, count);
 995     }
 996
 997     CBIT_CLR(mask, id);
 998     assert(count);
 999     return count;
1000 }
1001
1002 /*
1003  * deletes an entry at pos and leaves a gap; leaving a gap makes it
1004  * possible to iterate(search) and delete fields at the same time
1005  * NOTE: Does not update the header mask. Caller must follow up with
1006  * a call to refreshMask() if headers_deleted was incremented.
1007  */
1008 void
1009 HttpHeader::delAt(HttpHeaderPos pos, int &headers_deleted)
1010 {
1011     HttpHeaderEntry *e;
1012     assert(pos >= HttpHeaderInitPos && pos < static_cast<ssize_t>(entries.size()));
1013     e = static_cast<HttpHeaderEntry*>(entries[pos]);
1014     entries[pos] = NULL;
1015     /* decrement header length, allow for ": " and crlf */
1016     len -= e->name.size() + 2 + e->value.size() + 2;
1017     assert(len >= 0);
1018     delete e;
1019     ++headers_deleted;
1020 }
1021
1022 /*
1023  * Compacts the header storage
1024  */
1025 void
1026 HttpHeader::compact()
1027 {
1028     // TODO: optimize removal, or possibly make it so that's not needed.
1029     std::vector<HttpHeaderEntry *>::iterator newend;
1030     newend = std::remove(entries.begin(), entries.end(), static_cast<HttpHeaderEntry *>(NULL));
1031     entries.resize(newend-entries.begin());
1032 }
1033
1034 /*
1035  * Refreshes the header mask. Required after delAt() calls.
1036  */
1037 void
1038 HttpHeader::refreshMask()
1039 {
1040     httpHeaderMaskInit(&mask, 0);
1041     debugs(55, 7, "refreshing the mask in hdr " << this);
1042     HttpHeaderPos pos = HttpHeaderInitPos;
1043     while (HttpHeaderEntry *e = getEntry(&pos)) {
1044         CBIT_SET(mask, e->id);
1045     }
1046 }
1047
1048 /* appends an entry;
1049  * does not call e->clone() so one should not reuse "*e"
1050  */
1051 void
1052 HttpHeader::addEntry(HttpHeaderEntry * e)
1053 {
1054     assert(e);
1055     assert_eid(e->id);
1056     assert(e->name.size());
1057
1058     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
1059
1060     if (CBIT_TEST(mask, e->id)) {
1061         ++ Headers[e->id].stat.repCount;
1062         ++ headerStatsTable.at(e->id).repCount; //TODO: use operator[]
1063     } else {
1064         CBIT_SET(mask, e->id);
1065     }
1066
1067     entries.push_back(e);
1068
1069     /* increment header length, allow for ": " and crlf */
1070     len += e->name.size() + 2 + e->value.size() + 2;
1071 }
1072
1073 /* inserts an entry;
1074  * does not call e->clone() so one should not reuse "*e"
1075  */
1076 void
1077 HttpHeader::insertEntry(HttpHeaderEntry * e)
1078 {
1079     assert(e);
1080     assert_eid(e->id);
1081
1082     debugs(55, 7, this << " adding entry: " << e->id << " at " << entries.size());
1083
1084     if (CBIT_TEST(mask, e->id)) {
1085         ++ Headers[e->id].stat.repCount;
1086         ++ headerStatsTable.at(e->id).repCount;
1087     } else {
1088         CBIT_SET(mask, e->id);
1089     }
1090
1091     entries.insert(entries.begin(),e);
1092
1093     /* increment header length, allow for ": " and crlf */
1094     len += e->name.size() + 2 + e->value.size() + 2;
1095 }
1096
1097 bool
1098 HttpHeader::getList(http_hdr_type id, String *s) const
1099 {
1100     HttpHeaderEntry *e;
1101     HttpHeaderPos pos = HttpHeaderInitPos;
1102     debugs(55, 9, this << " joining for id " << id);
1103     /* only fields from ListHeaders array can be "listed" */
1104     assert(CBIT_TEST(ListHeadersMask, id));
1105
1106     if (!CBIT_TEST(mask, id))
1107         return false;
1108
1109     while ((e = getEntry(&pos))) {
1110         if (e->id == id)
1111             strListAdd(s, e->value.termedBuf(), ',');
1112     }
1113
1114     /*
1115      * note: we might get an empty (size==0) string if there was an "empty"
1116      * header. This results in an empty length String, which may have a NULL
1117      * buffer.
1118      */
1119     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
1120     if (!s->size())
1121         debugs(55, 3, "empty list header: " << headerTable[id].name << "(" << id << ")");
1122     else
1123         debugs(55, 6, this << ": joined for id " << id << ": " << s);
1124
1125     return true;
1126 }
1127
1128 /* return a list of entries with the same id separated by ',' and ws */
1129 String
1130 HttpHeader::getList(http_hdr_type id) const
1131 {
1132     HttpHeaderEntry *e;
1133     HttpHeaderPos pos = HttpHeaderInitPos;
1134     debugs(55, 9, this << "joining for id " << id);
1135     /* only fields from ListHeaders array can be "listed" */
1136     assert(CBIT_TEST(ListHeadersMask, id));
1137
1138     if (!CBIT_TEST(mask, id))
1139         return String();
1140
1141     String s;
1142
1143     while ((e = getEntry(&pos))) {
1144         if (e->id == id)
1145             strListAdd(&s, e->value.termedBuf(), ',');
1146     }
1147
1148     /*
1149      * note: we might get an empty (size==0) string if there was an "empty"
1150      * header. This results in an empty length String, which may have a NULL
1151      * buffer.
1152      */
1153     /* temporary warning: remove it? (Is it useful for diagnostics ?) */
1154     if (!s.size())
1155         debugs(55, 3, "empty list header: " << headerTable[id].name << "(" << id << ")");
1156     else
1157         debugs(55, 6, this << ": joined for id " << id << ": " << s);
1158
1159     return s;
1160 }
1161
1162 /* return a string or list of entries with the same id separated by ',' and ws */
1163 String
1164 HttpHeader::getStrOrList(http_hdr_type id) const
1165 {
1166     HttpHeaderEntry *e;
1167
1168     if (CBIT_TEST(ListHeadersMask, id))
1169         return getList(id);
1170
1171     if ((e = findEntry(id)))
1172         return e->value;
1173
1174     return String();
1175 }
1176
1177 /*
1178  * Returns the value of the specified header and/or an undefined String.
1179  */
1180 String
1181 HttpHeader::getByName(const char *name) const
1182 {
1183     String result;
1184     // ignore presence: return undefined string if an empty header is present
1185     (void)getByNameIfPresent(name, result);
1186     return result;
1187 }
1188
1189 bool
1190 HttpHeader::getByNameIfPresent(const char *name, String &result) const
1191 {
1192     http_hdr_type id;
1193     HttpHeaderPos pos = HttpHeaderInitPos;
1194     HttpHeaderEntry *e;
1195
1196     assert(name);
1197
1198     /* First try the quick path */
1199     id = httpHeaderIdByNameDef(name, strlen(name));
1200
1201     if (id != -1) {
1202         if (!has(id))
1203             return false;
1204         result = getStrOrList(id);
1205         return true;
1206     }
1207
1208     /* Sorry, an unknown header name. Do linear search */
1209     bool found = false;
1210     while ((e = getEntry(&pos))) {
1211         if (e->id == HDR_OTHER && e->name.caseCmp(name) == 0) {
1212             found = true;
1213             strListAdd(&result, e->value.termedBuf(), ',');
1214         }
1215     }
1216
1217     return found;
1218 }
1219
1220 /*
1221  * Returns a the value of the specified list member, if any.
1222  */
1223 String
1224 HttpHeader::getByNameListMember(const char *name, const char *member, const char separator) const
1225 {
1226     String header;
1227     const char *pos = NULL;
1228     const char *item;
1229     int ilen;
1230     int mlen = strlen(member);
1231
1232     assert(name);
1233
1234     header = getByName(name);
1235
1236     String result;
1237
1238     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1239         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1240             result.append(item + mlen + 1, ilen - mlen - 1);
1241             break;
1242         }
1243     }
1244
1245     return result;
1246 }
1247
1248 /*
1249  * returns a the value of the specified list member, if any.
1250  */
1251 String
1252 HttpHeader::getListMember(http_hdr_type id, const char *member, const char separator) const
1253 {
1254     String header;
1255     const char *pos = NULL;
1256     const char *item;
1257     int ilen;
1258     int mlen = strlen(member);
1259
1260     assert(id >= 0);
1261
1262     header = getStrOrList(id);
1263     String result;
1264
1265     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1266         if (strncmp(item, member, mlen) == 0 && item[mlen] == '=') {
1267             result.append(item + mlen + 1, ilen - mlen - 1);
1268             break;
1269         }
1270     }
1271
1272     header.clean();
1273     return result;
1274 }
1275
1276 /* test if a field is present */
1277 int
1278 HttpHeader::has(http_hdr_type id) const
1279 {
1280     assert_eid(id);
1281     assert(id != HDR_OTHER);
1282     debugs(55, 9, this << " lookup for " << id);
1283     return CBIT_TEST(mask, id);
1284 }
1285
1286 void
1287 HttpHeader::putInt(http_hdr_type id, int number)
1288 {
1289     assert_eid(id);
1290     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1291     assert(number >= 0);
1292     addEntry(new HttpHeaderEntry(id, NULL, xitoa(number)));
1293 }
1294
1295 void
1296 HttpHeader::putInt64(http_hdr_type id, int64_t number)
1297 {
1298     assert_eid(id);
1299     assert(Headers[id].type == ftInt64);    /* must be of an appropriate type */
1300     assert(number >= 0);
1301     addEntry(new HttpHeaderEntry(id, NULL, xint64toa(number)));
1302 }
1303
1304 void
1305 HttpHeader::putTime(http_hdr_type id, time_t htime)
1306 {
1307     assert_eid(id);
1308     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1309     assert(htime >= 0);
1310     addEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1311 }
1312
1313 void
1314 HttpHeader::insertTime(http_hdr_type id, time_t htime)
1315 {
1316     assert_eid(id);
1317     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1318     assert(htime >= 0);
1319     insertEntry(new HttpHeaderEntry(id, NULL, mkrfc1123(htime)));
1320 }
1321
1322 void
1323 HttpHeader::putStr(http_hdr_type id, const char *str)
1324 {
1325     assert_eid(id);
1326     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1327     assert(str);
1328     addEntry(new HttpHeaderEntry(id, NULL, str));
1329 }
1330
1331 void
1332 HttpHeader::putAuth(const char *auth_scheme, const char *realm)
1333 {
1334     assert(auth_scheme && realm);
1335     httpHeaderPutStrf(this, HDR_WWW_AUTHENTICATE, "%s realm=\"%s\"", auth_scheme, realm);
1336 }
1337
1338 void
1339 HttpHeader::putCc(const HttpHdrCc * cc)
1340 {
1341     assert(cc);
1342     /* remove old directives if any */
1343     delById(HDR_CACHE_CONTROL);
1344     /* pack into mb */
1345     MemBuf mb;
1346     mb.init();
1347     cc->packInto(&mb);
1348     /* put */
1349     addEntry(new HttpHeaderEntry(HDR_CACHE_CONTROL, NULL, mb.buf));
1350     /* cleanup */
1351     mb.clean();
1352 }
1353
1354 void
1355 HttpHeader::putContRange(const HttpHdrContRange * cr)
1356 {
1357     assert(cr);
1358     /* remove old directives if any */
1359     delById(HDR_CONTENT_RANGE);
1360     /* pack into mb */
1361     MemBuf mb;
1362     mb.init();
1363     httpHdrContRangePackInto(cr, &mb);
1364     /* put */
1365     addEntry(new HttpHeaderEntry(HDR_CONTENT_RANGE, NULL, mb.buf));
1366     /* cleanup */
1367     mb.clean();
1368 }
1369
1370 void
1371 HttpHeader::putRange(const HttpHdrRange * range)
1372 {
1373     assert(range);
1374     /* remove old directives if any */
1375     delById(HDR_RANGE);
1376     /* pack into mb */
1377     MemBuf mb;
1378     mb.init();
1379     range->packInto(&mb);
1380     /* put */
1381     addEntry(new HttpHeaderEntry(HDR_RANGE, NULL, mb.buf));
1382     /* cleanup */
1383     mb.clean();
1384 }
1385
1386 void
1387 HttpHeader::putSc(HttpHdrSc *sc)
1388 {
1389     assert(sc);
1390     /* remove old directives if any */
1391     delById(HDR_SURROGATE_CONTROL);
1392     /* pack into mb */
1393     MemBuf mb;
1394     mb.init();
1395     sc->packInto(&mb);
1396     /* put */
1397     addEntry(new HttpHeaderEntry(HDR_SURROGATE_CONTROL, NULL, mb.buf));
1398     /* cleanup */
1399     mb.clean();
1400 }
1401
1402 void
1403 HttpHeader::putWarning(const int code, const char *const text)
1404 {
1405     char buf[512];
1406     snprintf(buf, sizeof(buf), "%i %s \"%s\"", code, visible_appname_string, text);
1407     putStr(HDR_WARNING, buf);
1408 }
1409
1410 /* add extension header (these fields are not parsed/analyzed/joined, etc.) */
1411 void
1412 HttpHeader::putExt(const char *name, const char *value)
1413 {
1414     assert(name && value);
1415     debugs(55, 8, this << " adds ext entry " << name << " : " << value);
1416     addEntry(new HttpHeaderEntry(HDR_OTHER, name, value));
1417 }
1418
1419 int
1420 HttpHeader::getInt(http_hdr_type id) const
1421 {
1422     assert_eid(id);
1423     assert(Headers[id].type == ftInt);  /* must be of an appropriate type */
1424     HttpHeaderEntry *e;
1425
1426     if ((e = findEntry(id)))
1427         return e->getInt();
1428
1429     return -1;
1430 }
1431
1432 int64_t
1433 HttpHeader::getInt64(http_hdr_type id) const
1434 {
1435     assert_eid(id);
1436     assert(Headers[id].type == ftInt64);    /* must be of an appropriate type */
1437     HttpHeaderEntry *e;
1438
1439     if ((e = findEntry(id)))
1440         return e->getInt64();
1441
1442     return -1;
1443 }
1444
1445 time_t
1446 HttpHeader::getTime(http_hdr_type id) const
1447 {
1448     HttpHeaderEntry *e;
1449     time_t value = -1;
1450     assert_eid(id);
1451     assert(Headers[id].type == ftDate_1123);    /* must be of an appropriate type */
1452
1453     if ((e = findEntry(id))) {
1454         value = parse_rfc1123(e->value.termedBuf());
1455         httpHeaderNoteParsedEntry(e->id, e->value, value < 0);
1456     }
1457
1458     return value;
1459 }
1460
1461 /* sync with httpHeaderGetLastStr */
1462 const char *
1463 HttpHeader::getStr(http_hdr_type id) const
1464 {
1465     HttpHeaderEntry *e;
1466     assert_eid(id);
1467     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1468
1469     if ((e = findEntry(id))) {
1470         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1471         return e->value.termedBuf();
1472     }
1473
1474     return NULL;
1475 }
1476
1477 /* unusual */
1478 const char *
1479 HttpHeader::getLastStr(http_hdr_type id) const
1480 {
1481     HttpHeaderEntry *e;
1482     assert_eid(id);
1483     assert(Headers[id].type == ftStr);  /* must be of an appropriate type */
1484
1485     if ((e = findLastEntry(id))) {
1486         httpHeaderNoteParsedEntry(e->id, e->value, 0);  /* no errors are possible */
1487         return e->value.termedBuf();
1488     }
1489
1490     return NULL;
1491 }
1492
1493 HttpHdrCc *
1494 HttpHeader::getCc() const
1495 {
1496     if (!CBIT_TEST(mask, HDR_CACHE_CONTROL))
1497         return NULL;
1498     PROF_start(HttpHeader_getCc);
1499
1500     String s;
1501     getList(HDR_CACHE_CONTROL, &s);
1502
1503     HttpHdrCc *cc=new HttpHdrCc();
1504
1505     if (!cc->parse(s)) {
1506         delete cc;
1507         cc = NULL;
1508     }
1509
1510     ++ HttpHeaderStats[owner].ccParsedCount;
1511
1512     if (cc)
1513         httpHdrCcUpdateStats(cc, &HttpHeaderStats[owner].ccTypeDistr);
1514
1515     httpHeaderNoteParsedEntry(HDR_CACHE_CONTROL, s, !cc);
1516
1517     PROF_stop(HttpHeader_getCc);
1518
1519     return cc;
1520 }
1521
1522 HttpHdrRange *
1523 HttpHeader::getRange() const
1524 {
1525     HttpHdrRange *r = NULL;
1526     HttpHeaderEntry *e;
1527     /* some clients will send "Request-Range" _and_ *matching* "Range"
1528      * who knows, some clients might send Request-Range only;
1529      * this "if" should work correctly in both cases;
1530      * hopefully no clients send mismatched headers! */
1531
1532     if ((e = findEntry(HDR_RANGE)) ||
1533             (e = findEntry(HDR_REQUEST_RANGE))) {
1534         r = HttpHdrRange::ParseCreate(&e->value);
1535         httpHeaderNoteParsedEntry(e->id, e->value, !r);
1536     }
1537
1538     return r;
1539 }
1540
1541 HttpHdrSc *
1542 HttpHeader::getSc() const
1543 {
1544     if (!CBIT_TEST(mask, HDR_SURROGATE_CONTROL))
1545         return NULL;
1546
1547     String s;
1548
1549     (void) getList(HDR_SURROGATE_CONTROL, &s);
1550
1551     HttpHdrSc *sc = httpHdrScParseCreate(s);
1552
1553     ++ HttpHeaderStats[owner].ccParsedCount;
1554
1555     if (sc)
1556         sc->updateStats(&HttpHeaderStats[owner].scTypeDistr);
1557
1558     httpHeaderNoteParsedEntry(HDR_SURROGATE_CONTROL, s, !sc);
1559
1560     return sc;
1561 }
1562
1563 HttpHdrContRange *
1564 HttpHeader::getContRange() const
1565 {
1566     HttpHdrContRange *cr = NULL;
1567     HttpHeaderEntry *e;
1568
1569     if ((e = findEntry(HDR_CONTENT_RANGE))) {
1570         cr = httpHdrContRangeParseCreate(e->value.termedBuf());
1571         httpHeaderNoteParsedEntry(e->id, e->value, !cr);
1572     }
1573
1574     return cr;
1575 }
1576
1577 const char *
1578 HttpHeader::getAuth(http_hdr_type id, const char *auth_scheme) const
1579 {
1580     const char *field;
1581     int l;
1582     assert(auth_scheme);
1583     field = getStr(id);
1584
1585     if (!field)         /* no authorization field */
1586         return NULL;
1587
1588     l = strlen(auth_scheme);
1589
1590     if (!l || strncasecmp(field, auth_scheme, l))   /* wrong scheme */
1591         return NULL;
1592
1593     field += l;
1594
1595     if (!xisspace(*field))  /* wrong scheme */
1596         return NULL;
1597
1598     /* skip white space */
1599     for (; field && xisspace(*field); ++field);
1600
1601     if (!*field)        /* no authorization cookie */
1602         return NULL;
1603
1604     static char decodedAuthToken[8192];
1605     struct base64_decode_ctx ctx;
1606     base64_decode_init(&ctx);
1607     size_t decodedLen = 0;
1608     if (!base64_decode_update(&ctx, &decodedLen, reinterpret_cast<uint8_t*>(decodedAuthToken), strlen(field), reinterpret_cast<const uint8_t*>(field)) ||
1609             !base64_decode_final(&ctx)) {
1610         return NULL;
1611     }
1612     decodedAuthToken[decodedLen] = '\0';
1613     return decodedAuthToken;
1614 }
1615
1616 ETag
1617 HttpHeader::getETag(http_hdr_type id) const
1618 {
1619     ETag etag = {NULL, -1};
1620     HttpHeaderEntry *e;
1621     assert(Headers[id].type == ftETag);     /* must be of an appropriate type */
1622
1623     if ((e = findEntry(id)))
1624         etagParseInit(&etag, e->value.termedBuf());
1625
1626     return etag;
1627 }
1628
1629 TimeOrTag
1630 HttpHeader::getTimeOrTag(http_hdr_type id) const
1631 {
1632     TimeOrTag tot;
1633     HttpHeaderEntry *e;
1634     assert(Headers[id].type == ftDate_1123_or_ETag);    /* must be of an appropriate type */
1635     memset(&tot, 0, sizeof(tot));
1636
1637     if ((e = findEntry(id))) {
1638         const char *str = e->value.termedBuf();
1639         /* try as an ETag */
1640
1641         if (etagParseInit(&tot.tag, str)) {
1642             tot.valid = tot.tag.str != NULL;
1643             tot.time = -1;
1644         } else {
1645             /* or maybe it is time? */
1646             tot.time = parse_rfc1123(str);
1647             tot.valid = tot.time >= 0;
1648             tot.tag.str = NULL;
1649         }
1650     }
1651
1652     assert(tot.time < 0 || !tot.tag.str);   /* paranoid */
1653     return tot;
1654 }
1655
1656 /*
1657  * HttpHeaderEntry
1658  */
1659
1660 HttpHeaderEntry::HttpHeaderEntry(http_hdr_type anId, const char *aName, const char *aValue)
1661 {
1662     assert_eid(anId);
1663     id = anId;
1664
1665     if (id != HDR_OTHER)
1666         name = headerTable[id].name;
1667     else
1668         name = aName;
1669
1670     value = aValue;
1671
1672     ++ Headers[id].stat.aliveCount;
1673     ++ headerStatsTable.at(id).aliveCount;
1674
1675     debugs(55, 9, "created HttpHeaderEntry " << this << ": '" << name << " : " << value );
1676 }
1677
1678 HttpHeaderEntry::~HttpHeaderEntry()
1679 {
1680     assert_eid(id);
1681     debugs(55, 9, "destroying entry " << this << ": '" << name << ": " << value << "'");
1682
1683     assert(Headers[id].stat.aliveCount); // is this really needed?
1684
1685     -- Headers[id].stat.aliveCount;
1686
1687     id = HDR_BAD_HDR;
1688 }
1689
1690 /* parses and inits header entry, returns true/false */
1691 HttpHeaderEntry *
1692 HttpHeaderEntry::parse(const char *field_start, const char *field_end)
1693 {
1694     /* note: name_start == field_start */
1695     const char *name_end = (const char *)memchr(field_start, ':', field_end - field_start);
1696     int name_len = name_end ? name_end - field_start :0;
1697     const char *value_start = field_start + name_len + 1;   /* skip ':' */
1698     /* note: value_end == field_end */
1699
1700     ++ HeaderEntryParsedCount;
1701
1702     /* do we have a valid field name within this field? */
1703
1704     if (!name_len || name_end > field_end)
1705         return NULL;
1706
1707     if (name_len > 65534) {
1708         /* String must be LESS THAN 64K and it adds a terminating NULL */
1709         debugs(55, DBG_IMPORTANT, "WARNING: ignoring header name of " << name_len << " bytes");
1710         return NULL;
1711     }
1712
1713     if (Config.onoff.relaxed_header_parser && xisspace(field_start[name_len - 1])) {
1714         debugs(55, Config.onoff.relaxed_header_parser <= 0 ? 1 : 2,
1715                "NOTICE: Whitespace after header name in '" << getStringPrefix(field_start, field_end-field_start) << "'");
1716
1717         while (name_len > 0 && xisspace(field_start[name_len - 1]))
1718             --name_len;
1719
1720         if (!name_len)
1721             return NULL;
1722     }
1723
1724     /* now we know we can parse it */
1725
1726     debugs(55, 9, "parsing HttpHeaderEntry: near '" <<  getStringPrefix(field_start, field_end-field_start) << "'");
1727
1728     /* is it a "known" field? */
1729     http_hdr_type id = httpHeaderIdByName(field_start, name_len, Headers, HDR_ENUM_END);
1730
1731     String name;
1732
1733     String value;
1734
1735     if (id < 0)
1736         id = HDR_OTHER;
1737
1738     assert_eid(id);
1739
1740     /* set field name */
1741     if (id == HDR_OTHER)
1742         name.limitInit(field_start, name_len);
1743     else
1744         name = headerTable[id].name;
1745
1746     /* trim field value */
1747     while (value_start < field_end && xisspace(*value_start))
1748         ++value_start;
1749
1750     while (value_start < field_end && xisspace(field_end[-1]))
1751         --field_end;
1752
1753     if (field_end - value_start > 65534) {
1754         /* String must be LESS THAN 64K and it adds a terminating NULL */
1755         debugs(55, DBG_IMPORTANT, "WARNING: ignoring '" << name << "' header of " << (field_end - value_start) << " bytes");
1756
1757         if (id == HDR_OTHER)
1758             name.clean();
1759
1760         return NULL;
1761     }
1762
1763     /* set field value */
1764     value.limitInit(value_start, field_end - value_start);
1765
1766     ++ Headers[id].stat.seenCount;
1767     ++ headerStatsTable[id].seenCount;
1768
1769     debugs(55, 9, "parsed HttpHeaderEntry: '" << name << ": " << value << "'");
1770
1771     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1772 }
1773
1774 HttpHeaderEntry *
1775 HttpHeaderEntry::clone() const
1776 {
1777     return new HttpHeaderEntry(id, name.termedBuf(), value.termedBuf());
1778 }
1779
1780 void
1781 HttpHeaderEntry::packInto(Packable * p) const
1782 {
1783     assert(p);
1784     p->append(name.rawBuf(), name.size());
1785     p->append(": ", 2);
1786     p->append(value.rawBuf(), value.size());
1787     p->append("\r\n", 2);
1788 }
1789
1790 int
1791 HttpHeaderEntry::getInt() const
1792 {
1793     assert_eid (id);
1794     assert (Headers[id].type == ftInt);
1795     int val = -1;
1796     int ok = httpHeaderParseInt(value.termedBuf(), &val);
1797     httpHeaderNoteParsedEntry(id, value, !ok);
1798     /* XXX: Should we check ok - ie
1799      * return ok ? -1 : value;
1800      */
1801     return val;
1802 }
1803
1804 int64_t
1805 HttpHeaderEntry::getInt64() const
1806 {
1807     assert_eid (id);
1808     assert (Headers[id].type == ftInt64);
1809     int64_t val = -1;
1810     int ok = httpHeaderParseOffset(value.termedBuf(), &val);
1811     httpHeaderNoteParsedEntry(id, value, !ok);
1812     /* XXX: Should we check ok - ie
1813      * return ok ? -1 : value;
1814      */
1815     return val;
1816 }
1817
1818 static void
1819 httpHeaderNoteParsedEntry(http_hdr_type id, String const &context, int error)
1820 {
1821     ++ Headers[id].stat.parsCount;
1822     ++ headerStatsTable.at(id).parsCount;
1823
1824     if (error) {
1825         ++ Headers[id].stat.errCount;
1826         ++ headerStatsTable.at(id).errCount;
1827         debugs(55, 2, "cannot parse hdr field: '" << headerTable[id].name << ": " << context << "'");
1828     }
1829 }
1830
1831 /*
1832  * Reports
1833  */
1834
1835 /* tmp variable used to pass stat info to dumpers */
1836 extern const HttpHeaderStat *dump_stat;     /* argh! */
1837 const HttpHeaderStat *dump_stat = NULL;
1838
1839 void
1840 httpHeaderFieldStatDumper(StoreEntry * sentry, int, double val, double, int count)
1841 {
1842     const int id = (int) val;
1843     const int valid_id = id >= 0 && id < HDR_ENUM_END;
1844     const char *name = valid_id ? headerTable[id].name : "INVALID";
1845     int visible = count > 0;
1846     /* for entries with zero count, list only those that belong to current type of message */
1847
1848     if (!visible && valid_id && dump_stat->owner_mask)
1849         visible = CBIT_TEST(*dump_stat->owner_mask, id);
1850
1851     if (visible)
1852         storeAppendPrintf(sentry, "%2d\t %-20s\t %5d\t %6.2f\n",
1853                           id, name, count, xdiv(count, dump_stat->busyDestroyedCount));
1854 }
1855
1856 static void
1857 httpHeaderFldsPerHdrDumper(StoreEntry * sentry, int idx, double val, double, int count)
1858 {
1859     if (count)
1860         storeAppendPrintf(sentry, "%2d\t %5d\t %5d\t %6.2f\n",
1861                           idx, (int) val, count,
1862                           xpercent(count, dump_stat->destroyedCount));
1863 }
1864
1865 static void
1866 httpHeaderStatDump(const HttpHeaderStat * hs, StoreEntry * e)
1867 {
1868     assert(hs && e);
1869
1870     dump_stat = hs;
1871     storeAppendPrintf(e, "\nHeader Stats: %s\n", hs->label);
1872     storeAppendPrintf(e, "\nField type distribution\n");
1873     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1874                       "id", "name", "count", "#/header");
1875     hs->fieldTypeDistr.dump(e, httpHeaderFieldStatDumper);
1876     storeAppendPrintf(e, "\nCache-control directives distribution\n");
1877     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1878                       "id", "name", "count", "#/cc_field");
1879     hs->ccTypeDistr.dump(e, httpHdrCcStatDumper);
1880     storeAppendPrintf(e, "\nSurrogate-control directives distribution\n");
1881     storeAppendPrintf(e, "%2s\t %-20s\t %5s\t %6s\n",
1882                       "id", "name", "count", "#/sc_field");
1883     hs->scTypeDistr.dump(e, httpHdrScStatDumper);
1884     storeAppendPrintf(e, "\nNumber of fields per header distribution\n");
1885     storeAppendPrintf(e, "%2s\t %-5s\t %5s\t %6s\n",
1886                       "id", "#flds", "count", "%total");
1887     hs->hdrUCountDistr.dump(e, httpHeaderFldsPerHdrDumper);
1888     storeAppendPrintf(e, "\n");
1889     dump_stat = NULL;
1890 }
1891
1892 void
1893 httpHeaderStoreReport(StoreEntry * e)
1894 {
1895     int i;
1896     http_hdr_type ht;
1897     assert(e);
1898
1899     HttpHeaderStats[0].parsedCount =
1900         HttpHeaderStats[hoRequest].parsedCount + HttpHeaderStats[hoReply].parsedCount;
1901     HttpHeaderStats[0].ccParsedCount =
1902         HttpHeaderStats[hoRequest].ccParsedCount + HttpHeaderStats[hoReply].ccParsedCount;
1903     HttpHeaderStats[0].destroyedCount =
1904         HttpHeaderStats[hoRequest].destroyedCount + HttpHeaderStats[hoReply].destroyedCount;
1905     HttpHeaderStats[0].busyDestroyedCount =
1906         HttpHeaderStats[hoRequest].busyDestroyedCount + HttpHeaderStats[hoReply].busyDestroyedCount;
1907
1908     for (i = 1; i < HttpHeaderStatCount; ++i) {
1909         httpHeaderStatDump(HttpHeaderStats + i, e);
1910     }
1911
1912     /* field stats for all messages */
1913     storeAppendPrintf(e, "\nHttp Fields Stats (replies and requests)\n");
1914
1915     storeAppendPrintf(e, "%2s\t %-25s\t %5s\t %6s\t %6s\n",
1916                       "id", "name", "#alive", "%err", "%repeat");
1917
1918     for (ht = (http_hdr_type)0; ht < HDR_ENUM_END; ++ht) {
1919         HttpHeaderFieldInfo *f = Headers + ht;
1920         storeAppendPrintf(e, "%2d\t %-25s\t %5d\t %6.3f\t %6.3f\n",
1921                           f->id, f->name.termedBuf(), f->stat.aliveCount,
1922                           xpercent(f->stat.errCount, f->stat.parsCount),
1923                           xpercent(f->stat.repCount, f->stat.seenCount));
1924     }
1925
1926     storeAppendPrintf(e, "Headers Parsed: %d + %d = %d\n",
1927                       HttpHeaderStats[hoRequest].parsedCount,
1928                       HttpHeaderStats[hoReply].parsedCount,
1929                       HttpHeaderStats[0].parsedCount);
1930     storeAppendPrintf(e, "Hdr Fields Parsed: %d\n", HeaderEntryParsedCount);
1931 }
1932
1933 http_hdr_type
1934 httpHeaderIdByName(const char *name, size_t name_len, const HttpHeaderFieldInfo * info, int end)
1935 {
1936     if (name_len > 0) {
1937         for (int i = 0; i < end; ++i) {
1938             if (name_len != info[i].name.size())
1939                 continue;
1940
1941             if (!strncasecmp(name, info[i].name.rawBuf(), name_len))
1942                 return info[i].id;
1943         }
1944     }
1945
1946     return HDR_BAD_HDR;
1947 }
1948
1949 http_hdr_type
1950 httpHeaderIdByNameDef(const char *name, int name_len)
1951 {
1952     if (!Headers)
1953         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1954
1955     return httpHeaderIdByName(name, name_len, Headers, HDR_ENUM_END);
1956 }
1957
1958 const char *
1959 httpHeaderNameById(int id)
1960 {
1961     if (!Headers)
1962         Headers = httpHeaderBuildFieldsInfo(HeadersAttrs, HDR_ENUM_END);
1963
1964     assert(id >= 0 && id < HDR_ENUM_END);
1965
1966     return headerTable[id].name;
1967 }
1968
1969 int
1970 HttpHeader::hasListMember(http_hdr_type id, const char *member, const char separator) const
1971 {
1972     int result = 0;
1973     const char *pos = NULL;
1974     const char *item;
1975     int ilen;
1976     int mlen = strlen(member);
1977
1978     assert(id >= 0);
1979
1980     String header (getStrOrList(id));
1981
1982     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
1983         if (strncasecmp(item, member, mlen) == 0
1984                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
1985             result = 1;
1986             break;
1987         }
1988     }
1989
1990     return result;
1991 }
1992
1993 int
1994 HttpHeader::hasByNameListMember(const char *name, const char *member, const char separator) const
1995 {
1996     int result = 0;
1997     const char *pos = NULL;
1998     const char *item;
1999     int ilen;
2000     int mlen = strlen(member);
2001
2002     assert(name);
2003
2004     String header (getByName(name));
2005
2006     while (strListGetItem(&header, separator, &item, &ilen, &pos)) {
2007         if (strncasecmp(item, member, mlen) == 0
2008                 && (item[mlen] == '=' || item[mlen] == separator || item[mlen] == ';' || item[mlen] == '\0')) {
2009             result = 1;
2010             break;
2011         }
2012     }
2013
2014     return result;
2015 }
2016
2017 void
2018 HttpHeader::removeHopByHopEntries()
2019 {
2020     removeConnectionHeaderEntries();
2021
2022     const HttpHeaderEntry *e;
2023     HttpHeaderPos pos = HttpHeaderInitPos;
2024     int headers_deleted = 0;
2025     while ((e = getEntry(&pos))) {
2026         int id = e->id;
2027         if (CBIT_TEST(HopByHopHeadersMask, id)) {
2028             delAt(pos, headers_deleted);
2029             CBIT_CLR(mask, id);
2030         }
2031     }
2032 }
2033
2034 void
2035 HttpHeader::removeConnectionHeaderEntries()
2036 {
2037     if (has(HDR_CONNECTION)) {
2038         /* anything that matches Connection list member will be deleted */
2039         String strConnection;
2040
2041         (void) getList(HDR_CONNECTION, &strConnection);
2042         const HttpHeaderEntry *e;
2043         HttpHeaderPos pos = HttpHeaderInitPos;
2044         /*
2045          * think: on-average-best nesting of the two loops (hdrEntry
2046          * and strListItem) @?@
2047          */
2048         /*
2049          * maybe we should delete standard stuff ("keep-alive","close")
2050          * from strConnection first?
2051          */
2052
2053         int headers_deleted = 0;
2054         while ((e = getEntry(&pos))) {
2055             if (strListIsMember(&strConnection, e->name.termedBuf(), ','))
2056                 delAt(pos, headers_deleted);
2057         }
2058         if (headers_deleted)
2059             refreshMask();
2060     }
2061 }
2062