src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "globals.h"
  15 #include "HttpBody.h"
  16 #include "HttpHdrCc.h"
  17 #include "HttpHdrContRange.h"
  18 #include "HttpHdrSc.h"
  19 #include "HttpReply.h"
  20 #include "HttpRequest.h"
  21 #include "MemBuf.h"
  22 #include "SquidConfig.h"
  23 #include "SquidTime.h"
  24 #include "Store.h"
  25 #include "StrList.h"
  26
  27 /* local constants */
  28
  29 /* If we receive a 304 from the origin during a cache revalidation, we must
  30  * update the headers of the existing entry. Specifically, we need to update all
  31  * end-to-end headers and not any hop-by-hop headers (rfc2616 13.5.3).
  32  *
  33  * This is not the whole story though: since it is possible for a faulty/malicious
  34  * origin server to set headers it should not in a 304, we must explicitly ignore
  35  * these too. Specifically all entity-headers except those permitted in a 304
  36  * (rfc2616 10.3.5) must be ignored.
  37  *
  38  * The list of headers we don't update is made up of:
  39  *     all hop-by-hop headers
  40  *     all entity-headers except Expires and Content-Location
  41  */
  42 static HttpHeaderMask Denied304HeadersMask;
  43 static http_hdr_type Denied304HeadersArr[] = {
  44     // hop-by-hop headers
  45     HDR_CONNECTION, HDR_KEEP_ALIVE, HDR_PROXY_AUTHENTICATE, HDR_PROXY_AUTHORIZATION,
  46     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE,
  47     // entity headers
  48     HDR_ALLOW, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH,
  49     HDR_CONTENT_MD5, HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_LAST_MODIFIED
  50 };
  51
  52 /* module initialization */
  53 void
  54 httpReplyInitModule(void)
  55 {
  56     assert(Http::scNone == 0); // HttpReply::parse() interface assumes that
  57     httpHeaderMaskInit(&Denied304HeadersMask, 0);
  58     httpHeaderCalcMask(&Denied304HeadersMask, Denied304HeadersArr, countof(Denied304HeadersArr));
  59 }
  60
  61 HttpReply::HttpReply() : HttpMsg(hoReply), date (0), last_modified (0),
  62     expires (0), surrogate_control (NULL), content_range (NULL), keep_alive (0),
  63     protoPrefix("HTTP/"), bodySizeMax(-2)
  64 {
  65     init();
  66 }
  67
  68 HttpReply::~HttpReply()
  69 {
  70     if (do_clean)
  71         clean();
  72 }
  73
  74 void
  75 HttpReply::init()
  76 {
  77     hdrCacheInit();
  78     sline.init();
  79     pstate = psReadyToParseStartLine;
  80     do_clean = true;
  81 }
  82
  83 void HttpReply::reset()
  84 {
  85
  86     // reset should not reset the protocol; could have made protoPrefix a
  87     // virtual function instead, but it is not clear whether virtual methods
  88     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  89     // conversions are not going to kill virtual tables
  90     const String pfx = protoPrefix;
  91     clean();
  92     init();
  93     protoPrefix = pfx;
  94 }
  95
  96 void
  97 HttpReply::clean()
  98 {
  99     // we used to assert that the pipe is NULL, but now the message only
 100     // points to a pipe that is owned and initiated by another object.
 101     body_pipe = NULL;
 102
 103     body.clear();
 104     hdrCacheClean();
 105     header.clean();
 106     sline.clean();
 107     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
 108 }
 109
 110 void
 111 HttpReply::packHeadersInto(Packer * p) const
 112 {
 113     sline.packInto(p);
 114     header.packInto(p);
 115     packerAppend(p, "\r\n", 2);
 116 }
 117
 118 void
 119 HttpReply::packInto(Packer * p)
 120 {
 121     packHeadersInto(p);
 122     body.packInto(p);
 123 }
 124
 125 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 126 MemBuf *
 127 HttpReply::pack()
 128 {
 129     MemBuf *mb = new MemBuf;
 130     Packer p;
 131
 132     mb->init();
 133     packerToMemInit(&p, mb);
 134     packInto(&p);
 135     packerClean(&p);
 136     return mb;
 137 }
 138
 139 HttpReply *
 140 HttpReply::make304() const
 141 {
 142     static const http_hdr_type ImsEntries[] = {HDR_DATE, HDR_CONTENT_TYPE, HDR_EXPIRES, HDR_LAST_MODIFIED, /* eof */ HDR_OTHER};
 143
 144     HttpReply *rv = new HttpReply;
 145     int t;
 146     HttpHeaderEntry *e;
 147
 148     /* rv->content_length; */
 149     rv->date = date;
 150     rv->last_modified = last_modified;
 151     rv->expires = expires;
 152     rv->content_type = content_type;
 153     /* rv->cache_control */
 154     /* rv->content_range */
 155     /* rv->keep_alive */
 156     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, NULL);
 157
 158     for (t = 0; ImsEntries[t] != HDR_OTHER; ++t)
 159         if ((e = header.findEntry(ImsEntries[t])))
 160             rv->header.addEntry(e->clone());
 161
 162     /* rv->body */
 163     return rv;
 164 }
 165
 166 MemBuf *
 167 HttpReply::packed304Reply()
 168 {
 169     /* Not as efficient as skipping the header duplication,
 170      * but easier to maintain
 171      */
 172     HttpReply *temp = make304();
 173     MemBuf *rv = temp->pack();
 174     delete temp;
 175     return rv;
 176 }
 177
 178 void
 179 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 180                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 181 {
 182     HttpHeader *hdr;
 183     sline.set(Http::ProtocolVersion(), status, reason);
 184     hdr = &header;
 185     hdr->putStr(HDR_SERVER, visible_appname_string);
 186     hdr->putStr(HDR_MIME_VERSION, "1.0");
 187     hdr->putTime(HDR_DATE, squid_curtime);
 188
 189     if (ctype) {
 190         hdr->putStr(HDR_CONTENT_TYPE, ctype);
 191         content_type = ctype;
 192     } else
 193         content_type = String();
 194
 195     if (clen >= 0)
 196         hdr->putInt64(HDR_CONTENT_LENGTH, clen);
 197
 198     if (expiresTime >= 0)
 199         hdr->putTime(HDR_EXPIRES, expiresTime);
 200
 201     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 202         hdr->putTime(HDR_LAST_MODIFIED, lmt);
 203
 204     date = squid_curtime;
 205
 206     content_length = clen;
 207
 208     expires = expiresTime;
 209
 210     last_modified = lmt;
 211 }
 212
 213 void
 214 HttpReply::redirect(Http::StatusCode status, const char *loc)
 215 {
 216     HttpHeader *hdr;
 217     sline.set(Http::ProtocolVersion(), status, NULL);
 218     hdr = &header;
 219     hdr->putStr(HDR_SERVER, APP_FULLNAME);
 220     hdr->putTime(HDR_DATE, squid_curtime);
 221     hdr->putInt64(HDR_CONTENT_LENGTH, 0);
 222     hdr->putStr(HDR_LOCATION, loc);
 223     date = squid_curtime;
 224     content_length = 0;
 225 }
 226
 227 /* compare the validators of two replies.
 228  * 1 = they match
 229  * 0 = they do not match
 230  */
 231 int
 232 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 233 {
 234     String one,two;
 235     assert (otherRep);
 236     /* Numbers first - easiest to check */
 237     /* Content-Length */
 238     /* TODO: remove -1 bypass */
 239
 240     if (content_length != otherRep->content_length
 241             && content_length > -1 &&
 242             otherRep->content_length > -1)
 243         return 0;
 244
 245     /* ETag */
 246     one = header.getStrOrList(HDR_ETAG);
 247
 248     two = otherRep->header.getStrOrList(HDR_ETAG);
 249
 250     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 251         one.clean();
 252         two.clean();
 253         return 0;
 254     }
 255
 256     if (last_modified != otherRep->last_modified)
 257         return 0;
 258
 259     /* MD5 */
 260     one = header.getStrOrList(HDR_CONTENT_MD5);
 261
 262     two = otherRep->header.getStrOrList(HDR_CONTENT_MD5);
 263
 264     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 265         one.clean();
 266         two.clean();
 267         return 0;
 268     }
 269
 270     return 1;
 271 }
 272
 273 void
 274 HttpReply::updateOnNotModified(HttpReply const * freshRep)
 275 {
 276     assert(freshRep);
 277
 278     /* clean cache */
 279     hdrCacheClean();
 280     /* update raw headers */
 281     header.update(&freshRep->header,
 282                   (const HttpHeaderMask *) &Denied304HeadersMask);
 283
 284     header.compact();
 285     /* init cache */
 286     hdrCacheInit();
 287 }
 288
 289 /* internal routines */
 290
 291 time_t
 292 HttpReply::hdrExpirationTime()
 293 {
 294     /* The s-maxage and max-age directive takes priority over Expires */
 295
 296     if (cache_control) {
 297         if (date >= 0) {
 298             if (cache_control->hasSMaxAge())
 299                 return date + cache_control->sMaxAge();
 300
 301             if (cache_control->hasMaxAge())
 302                 return date + cache_control->maxAge();
 303         } else {
 304             /*
 305              * Conservatively handle the case when we have a max-age
 306              * header, but no Date for reference?
 307              */
 308
 309             if (cache_control->hasSMaxAge())
 310                 return squid_curtime;
 311
 312             if (cache_control->hasMaxAge())
 313                 return squid_curtime;
 314         }
 315     }
 316
 317     if (Config.onoff.vary_ignore_expire &&
 318             header.has(HDR_VARY)) {
 319         const time_t d = header.getTime(HDR_DATE);
 320         const time_t e = header.getTime(HDR_EXPIRES);
 321
 322         if (d == e)
 323             return -1;
 324     }
 325
 326     if (header.has(HDR_EXPIRES)) {
 327         const time_t e = header.getTime(HDR_EXPIRES);
 328         /*
 329          * HTTP/1.0 says that robust implementations should consider
 330          * bad or malformed Expires header as equivalent to "expires
 331          * immediately."
 332          */
 333         return e < 0 ? squid_curtime : e;
 334     }
 335
 336     return -1;
 337 }
 338
 339 /* sync this routine when you update HttpReply struct */
 340 void
 341 HttpReply::hdrCacheInit()
 342 {
 343     HttpMsg::hdrCacheInit();
 344
 345     http_ver = sline.version;
 346     content_length = header.getInt64(HDR_CONTENT_LENGTH);
 347     date = header.getTime(HDR_DATE);
 348     last_modified = header.getTime(HDR_LAST_MODIFIED);
 349     surrogate_control = header.getSc();
 350     content_range = header.getContRange();
 351     keep_alive = persistent() ? 1 : 0;
 352     const char *str = header.getStr(HDR_CONTENT_TYPE);
 353
 354     if (str)
 355         content_type.limitInit(str, strcspn(str, ";\t "));
 356     else
 357         content_type = String();
 358
 359     /* be sure to set expires after date and cache-control */
 360     expires = hdrExpirationTime();
 361 }
 362
 363 /* sync this routine when you update HttpReply struct */
 364 void
 365 HttpReply::hdrCacheClean()
 366 {
 367     content_type.clean();
 368
 369     if (cache_control) {
 370         delete cache_control;
 371         cache_control = NULL;
 372     }
 373
 374     if (surrogate_control) {
 375         delete surrogate_control;
 376         surrogate_control = NULL;
 377     }
 378
 379     if (content_range) {
 380         httpHdrContRangeDestroy(content_range);
 381         content_range = NULL;
 382     }
 383 }
 384
 385 /*
 386  * Returns the body size of a HTTP response
 387  */
 388 int64_t
 389 HttpReply::bodySize(const HttpRequestMethod& method) const
 390 {
 391     if (sline.version.major < 1)
 392         return -1;
 393     else if (method.id() == Http::METHOD_HEAD)
 394         return 0;
 395     else if (sline.status() == Http::scOkay)
 396         (void) 0;       /* common case, continue */
 397     else if (sline.status() == Http::scNoContent)
 398         return 0;
 399     else if (sline.status() == Http::scNotModified)
 400         return 0;
 401     else if (sline.status() < Http::scOkay)
 402         return 0;
 403
 404     return content_length;
 405 }
 406
 407 /**
 408  * Checks the first line of an HTTP Reply is valid.
 409  * currently only checks "HTTP/" exists.
 410  *
 411  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 412  */
 413 bool
 414 HttpReply::sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, Http::StatusCode *error)
 415 {
 416     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 417
 418     // content is long enough to possibly hold a reply
 419     // 4 being magic size of a 3-digit number plus space delimiter
 420     if ( buf->contentSize() < (protoPrefix.psize() + 4) ) {
 421         if (hdr_len > 0) {
 422             debugs(58, 3, HERE << "Too small reply header (" << hdr_len << " bytes)");
 423             *error = Http::scInvalidHeader;
 424         }
 425         return false;
 426     }
 427
 428     int pos;
 429     // catch missing or mismatched protocol identifier
 430     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 431     if (strncmp(buf->content(), "ICY", 3) == 0) {
 432         protoPrefix = "ICY";
 433         pos = protoPrefix.psize();
 434     } else {
 435
 436         if (protoPrefix.cmp(buf->content(), protoPrefix.size()) != 0) {
 437             debugs(58, 3, "HttpReply::sanityCheckStartLine: missing protocol prefix (" << protoPrefix << ") in '" << buf->content() << "'");
 438             *error = Http::scInvalidHeader;
 439             return false;
 440         }
 441
 442         // catch missing or negative status value (negative '-' is not a digit)
 443         pos = protoPrefix.psize();
 444
 445         // skip arbitrary number of digits and a dot in the verion portion
 446         while ( pos <= buf->contentSize() && (*(buf->content()+pos) == '.' || xisdigit(*(buf->content()+pos)) ) ) ++pos;
 447
 448         // catch missing version info
 449         if (pos == protoPrefix.psize()) {
 450             debugs(58, 3, "HttpReply::sanityCheckStartLine: missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf->content() << "'");
 451             *error = Http::scInvalidHeader;
 452             return false;
 453         }
 454     }
 455
 456     // skip arbitrary number of spaces...
 457     while (pos <= buf->contentSize() && (char)*(buf->content()+pos) == ' ') ++pos;
 458
 459     if (pos < buf->contentSize() && !xisdigit(*(buf->content()+pos))) {
 460         debugs(58, 3, "HttpReply::sanityCheckStartLine: missing or invalid status number in '" << buf->content() << "'");
 461         *error = Http::scInvalidHeader;
 462         return false;
 463     }
 464
 465     return true;
 466 }
 467
 468 bool
 469 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 470 {
 471     return sline.parse(protoPrefix, blk_start, blk_end);
 472 }
 473
 474 /* handy: resets and returns -1 */
 475 int
 476 HttpReply::httpMsgParseError()
 477 {
 478     int result(HttpMsg::httpMsgParseError());
 479     /* indicate an error in the status line */
 480     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 481     return result;
 482 }
 483
 484 /*
 485  * Indicate whether or not we would usually expect an entity-body
 486  * along with this response
 487  */
 488 bool
 489 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 490 {
 491     bool expectBody = true;
 492
 493     if (req_method == Http::METHOD_HEAD)
 494         expectBody = false;
 495     else if (sline.status() == Http::scNoContent)
 496         expectBody = false;
 497     else if (sline.status() == Http::scNotModified)
 498         expectBody = false;
 499     else if (sline.status() < Http::scOkay)
 500         expectBody = false;
 501     else
 502         expectBody = true;
 503
 504     if (expectBody) {
 505         if (header.chunked())
 506             theSize = -1;
 507         else if (content_length >= 0)
 508             theSize = content_length;
 509         else
 510             theSize = -1;
 511     }
 512
 513     return expectBody;
 514 }
 515
 516 bool
 517 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 518 {
 519     calcMaxBodySize(request);
 520     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 521     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 522 }
 523
 524 bool
 525 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 526 {
 527     calcMaxBodySize(request);
 528     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 529
 530     if (bodySizeMax < 0) // no body size limit
 531         return false;
 532
 533     int64_t expectedSize = -1;
 534     if (!expectingBody(request.method, expectedSize))
 535         return false;
 536
 537     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 538
 539     if (expectedSize < 0) // expecting body of an unknown length
 540         return false;
 541
 542     return expectedSize > bodySizeMax;
 543 }
 544
 545 void
 546 HttpReply::calcMaxBodySize(HttpRequest& request) const
 547 {
 548     // hack: -2 is used as "we have not calculated max body size yet" state
 549     if (bodySizeMax != -2) // already tried
 550         return;
 551     bodySizeMax = -1;
 552
 553     // short-circuit ACL testing if there are none configured
 554     if (!Config.ReplyBodySize)
 555         return;
 556
 557     ACLFilledChecklist ch(NULL, &request, NULL);
 558     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 559     ch.reply = const_cast<HttpReply *>(this);
 560     HTTPMSGLOCK(ch.reply);
 561     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 562         /* if there is no ACL list or if the ACLs listed match use this size value */
 563         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 564             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 565             bodySizeMax = l->size; // may be -1
 566             break;
 567         }
 568     }
 569 }
 570
 571 // XXX: check that this is sufficient for eCAP cloning
 572 HttpReply *
 573 HttpReply::clone() const
 574 {
 575     HttpReply *rep = new HttpReply();
 576     rep->sline = sline; // used in hdrCacheInit() call below
 577     rep->header.append(&header);
 578     rep->hdrCacheInit();
 579     rep->hdr_sz = hdr_sz;
 580     rep->http_ver = http_ver;
 581     rep->pstate = pstate;
 582     rep->body_pipe = body_pipe;
 583
 584     // keep_alive is handled in hdrCacheInit()
 585     return rep;
 586 }
 587
 588 bool HttpReply::inheritProperties(const HttpMsg *aMsg)
 589 {
 590     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 591     if (!aRep)
 592         return false;
 593     keep_alive = aRep->keep_alive;
 594     return true;
 595 }
 596
 597 void HttpReply::removeStaleWarnings()
 598 {
 599     String warning;
 600     if (header.getList(HDR_WARNING, &warning)) {
 601         const String newWarning = removeStaleWarningValues(warning);
 602         if (warning.size() && warning.size() == newWarning.size())
 603             return; // some warnings are there and none changed
 604         header.delById(HDR_WARNING);
 605         if (newWarning.size()) { // some warnings left
 606             HttpHeaderEntry *const e =
 607                 new HttpHeaderEntry(HDR_WARNING, NULL, newWarning.termedBuf());
 608             header.addEntry(e);
 609         }
 610     }
 611 }
 612
 613 /**
 614  * Remove warning-values with warn-date different from Date value from
 615  * a single header entry. Returns a string with all valid warning-values.
 616  */
 617 String HttpReply::removeStaleWarningValues(const String &value)
 618 {
 619     String newValue;
 620     const char *item = 0;
 621     int len = 0;
 622     const char *pos = 0;
 623     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 624         bool keep = true;
 625         // Does warning-value have warn-date (which contains quoted date)?
 626         // We scan backwards, looking for two quoted strings.
 627         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 628         const char *p = item + len - 1;
 629
 630         while (p >= item && xisspace(*p)) --p; // skip whitespace
 631
 632         // warning-value MUST end with quote
 633         if (p >= item && *p == '"') {
 634             const char *const warnDateEnd = p;
 635             --p;
 636             while (p >= item && *p != '"') --p; // find the next quote
 637
 638             const char *warnDateBeg = p + 1;
 639             --p;
 640             while (p >= item && xisspace(*p)) --p; // skip whitespace
 641
 642             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 643                 // found warn-text
 644                 String warnDate;
 645                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 646                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 647                 keep = (time > 0 && time == date); // keep valid and matching date
 648             }
 649         }
 650
 651         if (keep) {
 652             if (newValue.size())
 653                 newValue.append(", ");
 654             newValue.append(item, len);
 655         }
 656     }
 657
 658     return newValue;
 659 }
 660