src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "base/EnumIterator.h"
  15 #include "globals.h"
  16 #include "HttpBody.h"
  17 #include "HttpHdrCc.h"
  18 #include "HttpHdrContRange.h"
  19 #include "HttpHdrSc.h"
  20 #include "HttpReply.h"
  21 #include "HttpRequest.h"
  22 #include "MemBuf.h"
  23 #include "SquidConfig.h"
  24 #include "SquidTime.h"
  25 #include "Store.h"
  26 #include "StrList.h"
  27
  28 /* local constants */
  29
  30 /* If we receive a 304 from the origin during a cache revalidation, we must
  31  * update the headers of the existing entry. Specifically, we need to update all
  32  * end-to-end headers and not any hop-by-hop headers (rfc2616 13.5.3).
  33  *
  34  * This is not the whole story though: since it is possible for a faulty/malicious
  35  * origin server to set headers it should not in a 304, we must explicitly ignore
  36  * these too. Specifically all entity-headers except those permitted in a 304
  37  * (rfc2616 10.3.5) must be ignored.
  38  *
  39  * The list of headers we don't update is made up of:
  40  *     all hop-by-hop headers
  41  *     all entity-headers except Expires and Content-Location
  42  *
  43  * These headers are now stored in RegisteredHeadersHash.gperf and accessible
  44  * as Http::HeaderLookupTable.lookup(id).denied304
  45  */
  46 static HttpHeaderMask Denied304HeadersMask;
  47
  48 /* module initialization */
  49 void
  50 httpReplyInitModule(void)
  51 {
  52     assert(Http::scNone == 0); // HttpReply::parse() interface assumes that
  53     httpHeaderMaskInit(&Denied304HeadersMask, 0);
  54
  55     for (auto id : WholeEnum<Http::HdrType>()) {
  56         if (Http::HeaderLookupTable.lookup(id).denied304)
  57             CBIT_SET(Denied304HeadersMask, id);
  58     }
  59 }
  60
  61 HttpReply::HttpReply() : HttpMsg(hoReply), date (0), last_modified (0),
  62     expires (0), surrogate_control (NULL), content_range (NULL), keep_alive (0),
  63     protoPrefix("HTTP/"), bodySizeMax(-2)
  64 {
  65     init();
  66 }
  67
  68 HttpReply::~HttpReply()
  69 {
  70     if (do_clean)
  71         clean();
  72 }
  73
  74 void
  75 HttpReply::init()
  76 {
  77     hdrCacheInit();
  78     sline.init();
  79     pstate = psReadyToParseStartLine;
  80     do_clean = true;
  81 }
  82
  83 void HttpReply::reset()
  84 {
  85
  86     // reset should not reset the protocol; could have made protoPrefix a
  87     // virtual function instead, but it is not clear whether virtual methods
  88     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  89     // conversions are not going to kill virtual tables
  90     const String pfx = protoPrefix;
  91     clean();
  92     init();
  93     protoPrefix = pfx;
  94 }
  95
  96 void
  97 HttpReply::clean()
  98 {
  99     // we used to assert that the pipe is NULL, but now the message only
 100     // points to a pipe that is owned and initiated by another object.
 101     body_pipe = NULL;
 102
 103     body.clear();
 104     hdrCacheClean();
 105     header.clean();
 106     sline.clean();
 107     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
 108 }
 109
 110 void
 111 HttpReply::packHeadersInto(Packable * p) const
 112 {
 113     sline.packInto(p);
 114     header.packInto(p);
 115     p->append("\r\n", 2);
 116 }
 117
 118 void
 119 HttpReply::packInto(Packable * p)
 120 {
 121     packHeadersInto(p);
 122     body.packInto(p);
 123 }
 124
 125 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 126 MemBuf *
 127 HttpReply::pack()
 128 {
 129     MemBuf *mb = new MemBuf;
 130     mb->init();
 131     packInto(mb);
 132     return mb;
 133 }
 134
 135 HttpReply *
 136 HttpReply::make304() const
 137 {
 138     static const Http::HdrType ImsEntries[] = {Http::HdrType::DATE, Http::HdrType::CONTENT_TYPE, Http::HdrType::EXPIRES, Http::HdrType::LAST_MODIFIED, /* eof */ Http::HdrType::OTHER};
 139
 140     HttpReply *rv = new HttpReply;
 141     int t;
 142     HttpHeaderEntry *e;
 143
 144     /* rv->content_length; */
 145     rv->date = date;
 146     rv->last_modified = last_modified;
 147     rv->expires = expires;
 148     rv->content_type = content_type;
 149     /* rv->cache_control */
 150     /* rv->content_range */
 151     /* rv->keep_alive */
 152     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, NULL);
 153
 154     for (t = 0; ImsEntries[t] != Http::HdrType::OTHER; ++t) {
 155         if ((e = header.findEntry(ImsEntries[t])))
 156             rv->header.addEntry(e->clone());
 157     }
 158
 159     /* rv->body */
 160     return rv;
 161 }
 162
 163 MemBuf *
 164 HttpReply::packed304Reply()
 165 {
 166     /* Not as efficient as skipping the header duplication,
 167      * but easier to maintain
 168      */
 169     HttpReply *temp = make304();
 170     MemBuf *rv = temp->pack();
 171     delete temp;
 172     return rv;
 173 }
 174
 175 void
 176 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 177                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 178 {
 179     HttpHeader *hdr;
 180     sline.set(Http::ProtocolVersion(), status, reason);
 181     hdr = &header;
 182     hdr->putStr(Http::HdrType::SERVER, visible_appname_string);
 183     hdr->putStr(Http::HdrType::MIME_VERSION, "1.0");
 184     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 185
 186     if (ctype) {
 187         hdr->putStr(Http::HdrType::CONTENT_TYPE, ctype);
 188         content_type = ctype;
 189     } else
 190         content_type = String();
 191
 192     if (clen >= 0)
 193         hdr->putInt64(Http::HdrType::CONTENT_LENGTH, clen);
 194
 195     if (expiresTime >= 0)
 196         hdr->putTime(Http::HdrType::EXPIRES, expiresTime);
 197
 198     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 199         hdr->putTime(Http::HdrType::LAST_MODIFIED, lmt);
 200
 201     date = squid_curtime;
 202
 203     content_length = clen;
 204
 205     expires = expiresTime;
 206
 207     last_modified = lmt;
 208 }
 209
 210 void
 211 HttpReply::redirect(Http::StatusCode status, const char *loc)
 212 {
 213     HttpHeader *hdr;
 214     sline.set(Http::ProtocolVersion(), status, NULL);
 215     hdr = &header;
 216     hdr->putStr(Http::HdrType::SERVER, APP_FULLNAME);
 217     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 218     hdr->putInt64(Http::HdrType::CONTENT_LENGTH, 0);
 219     hdr->putStr(Http::HdrType::LOCATION, loc);
 220     date = squid_curtime;
 221     content_length = 0;
 222 }
 223
 224 /* compare the validators of two replies.
 225  * 1 = they match
 226  * 0 = they do not match
 227  */
 228 int
 229 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 230 {
 231     String one,two;
 232     assert (otherRep);
 233     /* Numbers first - easiest to check */
 234     /* Content-Length */
 235     /* TODO: remove -1 bypass */
 236
 237     if (content_length != otherRep->content_length
 238             && content_length > -1 &&
 239             otherRep->content_length > -1)
 240         return 0;
 241
 242     /* ETag */
 243     one = header.getStrOrList(Http::HdrType::ETAG);
 244
 245     two = otherRep->header.getStrOrList(Http::HdrType::ETAG);
 246
 247     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 248         one.clean();
 249         two.clean();
 250         return 0;
 251     }
 252
 253     if (last_modified != otherRep->last_modified)
 254         return 0;
 255
 256     /* MD5 */
 257     one = header.getStrOrList(Http::HdrType::CONTENT_MD5);
 258
 259     two = otherRep->header.getStrOrList(Http::HdrType::CONTENT_MD5);
 260
 261     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 262         one.clean();
 263         two.clean();
 264         return 0;
 265     }
 266
 267     return 1;
 268 }
 269
 270 void
 271 HttpReply::updateOnNotModified(HttpReply const * freshRep)
 272 {
 273     assert(freshRep);
 274
 275     /* clean cache */
 276     hdrCacheClean();
 277     /* update raw headers */
 278     header.update(&freshRep->header,
 279                   (const HttpHeaderMask *) &Denied304HeadersMask);
 280
 281     header.compact();
 282     /* init cache */
 283     hdrCacheInit();
 284 }
 285
 286 /* internal routines */
 287
 288 time_t
 289 HttpReply::hdrExpirationTime()
 290 {
 291     /* The s-maxage and max-age directive takes priority over Expires */
 292
 293     if (cache_control) {
 294         if (date >= 0) {
 295             if (cache_control->hasSMaxAge())
 296                 return date + cache_control->sMaxAge();
 297
 298             if (cache_control->hasMaxAge())
 299                 return date + cache_control->maxAge();
 300         } else {
 301             /*
 302              * Conservatively handle the case when we have a max-age
 303              * header, but no Date for reference?
 304              */
 305
 306             if (cache_control->hasSMaxAge())
 307                 return squid_curtime;
 308
 309             if (cache_control->hasMaxAge())
 310                 return squid_curtime;
 311         }
 312     }
 313
 314     if (Config.onoff.vary_ignore_expire &&
 315             header.has(Http::HdrType::VARY)) {
 316         const time_t d = header.getTime(Http::HdrType::DATE);
 317         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 318
 319         if (d == e)
 320             return -1;
 321     }
 322
 323     if (header.has(Http::HdrType::EXPIRES)) {
 324         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 325         /*
 326          * HTTP/1.0 says that robust implementations should consider
 327          * bad or malformed Expires header as equivalent to "expires
 328          * immediately."
 329          */
 330         return e < 0 ? squid_curtime : e;
 331     }
 332
 333     return -1;
 334 }
 335
 336 /* sync this routine when you update HttpReply struct */
 337 void
 338 HttpReply::hdrCacheInit()
 339 {
 340     HttpMsg::hdrCacheInit();
 341
 342     http_ver = sline.version;
 343     content_length = header.getInt64(Http::HdrType::CONTENT_LENGTH);
 344     date = header.getTime(Http::HdrType::DATE);
 345     last_modified = header.getTime(Http::HdrType::LAST_MODIFIED);
 346     surrogate_control = header.getSc();
 347     content_range = header.getContRange();
 348     keep_alive = persistent() ? 1 : 0;
 349     const char *str = header.getStr(Http::HdrType::CONTENT_TYPE);
 350
 351     if (str)
 352         content_type.limitInit(str, strcspn(str, ";\t "));
 353     else
 354         content_type = String();
 355
 356     /* be sure to set expires after date and cache-control */
 357     expires = hdrExpirationTime();
 358 }
 359
 360 /* sync this routine when you update HttpReply struct */
 361 void
 362 HttpReply::hdrCacheClean()
 363 {
 364     content_type.clean();
 365
 366     if (cache_control) {
 367         delete cache_control;
 368         cache_control = NULL;
 369     }
 370
 371     if (surrogate_control) {
 372         delete surrogate_control;
 373         surrogate_control = NULL;
 374     }
 375
 376     if (content_range) {
 377         httpHdrContRangeDestroy(content_range);
 378         content_range = NULL;
 379     }
 380 }
 381
 382 /*
 383  * Returns the body size of a HTTP response
 384  */
 385 int64_t
 386 HttpReply::bodySize(const HttpRequestMethod& method) const
 387 {
 388     if (sline.version.major < 1)
 389         return -1;
 390     else if (method.id() == Http::METHOD_HEAD)
 391         return 0;
 392     else if (sline.status() == Http::scOkay)
 393         (void) 0;       /* common case, continue */
 394     else if (sline.status() == Http::scNoContent)
 395         return 0;
 396     else if (sline.status() == Http::scNotModified)
 397         return 0;
 398     else if (sline.status() < Http::scOkay)
 399         return 0;
 400
 401     return content_length;
 402 }
 403
 404 /**
 405  * Checks the first line of an HTTP Reply is valid.
 406  * currently only checks "HTTP/" exists.
 407  *
 408  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 409  */
 410 bool
 411 HttpReply::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 412 {
 413     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 414
 415     // content is long enough to possibly hold a reply
 416     // 4 being magic size of a 3-digit number plus space delimiter
 417     if (hdr_len < (size_t)(protoPrefix.psize() + 4)) {
 418         if (hdr_len > 0) {
 419             debugs(58, 3, "Too small reply header (" << hdr_len << " bytes)");
 420             *error = Http::scInvalidHeader;
 421         }
 422         return false;
 423     }
 424
 425     int pos;
 426     // catch missing or mismatched protocol identifier
 427     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 428     if (strncmp(buf, "ICY", 3) == 0) {
 429         protoPrefix = "ICY";
 430         pos = protoPrefix.psize();
 431     } else {
 432
 433         if (protoPrefix.cmp(buf, protoPrefix.size()) != 0) {
 434             debugs(58, 3, "missing protocol prefix (" << protoPrefix << ") in '" << buf << "'");
 435             *error = Http::scInvalidHeader;
 436             return false;
 437         }
 438
 439         // catch missing or negative status value (negative '-' is not a digit)
 440         pos = protoPrefix.psize();
 441
 442         // skip arbitrary number of digits and a dot in the verion portion
 443         while ((size_t)pos <= hdr_len && (*(buf+pos) == '.' || xisdigit(*(buf+pos)) ) ) ++pos;
 444
 445         // catch missing version info
 446         if (pos == protoPrefix.psize()) {
 447             debugs(58, 3, "missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf << "'");
 448             *error = Http::scInvalidHeader;
 449             return false;
 450         }
 451     }
 452
 453     // skip arbitrary number of spaces...
 454     while ((size_t)pos <= hdr_len && (char)*(buf+pos) == ' ') ++pos;
 455
 456     if ((size_t)pos < hdr_len && !xisdigit(*(buf+pos))) {
 457         debugs(58, 3, "missing or invalid status number in '" << buf << "'");
 458         *error = Http::scInvalidHeader;
 459         return false;
 460     }
 461
 462     return true;
 463 }
 464
 465 bool
 466 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 467 {
 468     return sline.parse(protoPrefix, blk_start, blk_end);
 469 }
 470
 471 /* handy: resets and returns -1 */
 472 int
 473 HttpReply::httpMsgParseError()
 474 {
 475     int result(HttpMsg::httpMsgParseError());
 476     /* indicate an error in the status line */
 477     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 478     return result;
 479 }
 480
 481 /*
 482  * Indicate whether or not we would usually expect an entity-body
 483  * along with this response
 484  */
 485 bool
 486 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 487 {
 488     bool expectBody = true;
 489
 490     if (req_method == Http::METHOD_HEAD)
 491         expectBody = false;
 492     else if (sline.status() == Http::scNoContent)
 493         expectBody = false;
 494     else if (sline.status() == Http::scNotModified)
 495         expectBody = false;
 496     else if (sline.status() < Http::scOkay)
 497         expectBody = false;
 498     else
 499         expectBody = true;
 500
 501     if (expectBody) {
 502         if (header.chunked())
 503             theSize = -1;
 504         else if (content_length >= 0)
 505             theSize = content_length;
 506         else
 507             theSize = -1;
 508     }
 509
 510     return expectBody;
 511 }
 512
 513 bool
 514 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 515 {
 516     calcMaxBodySize(request);
 517     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 518     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 519 }
 520
 521 bool
 522 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 523 {
 524     calcMaxBodySize(request);
 525     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 526
 527     if (bodySizeMax < 0) // no body size limit
 528         return false;
 529
 530     int64_t expectedSize = -1;
 531     if (!expectingBody(request.method, expectedSize))
 532         return false;
 533
 534     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 535
 536     if (expectedSize < 0) // expecting body of an unknown length
 537         return false;
 538
 539     return expectedSize > bodySizeMax;
 540 }
 541
 542 void
 543 HttpReply::calcMaxBodySize(HttpRequest& request) const
 544 {
 545     // hack: -2 is used as "we have not calculated max body size yet" state
 546     if (bodySizeMax != -2) // already tried
 547         return;
 548     bodySizeMax = -1;
 549
 550     // short-circuit ACL testing if there are none configured
 551     if (!Config.ReplyBodySize)
 552         return;
 553
 554     ACLFilledChecklist ch(NULL, &request, NULL);
 555     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 556     ch.reply = const_cast<HttpReply *>(this);
 557     HTTPMSGLOCK(ch.reply);
 558     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 559         /* if there is no ACL list or if the ACLs listed match use this size value */
 560         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 561             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 562             bodySizeMax = l->size; // may be -1
 563             break;
 564         }
 565     }
 566 }
 567
 568 // XXX: check that this is sufficient for eCAP cloning
 569 HttpReply *
 570 HttpReply::clone() const
 571 {
 572     HttpReply *rep = new HttpReply();
 573     rep->sline = sline; // used in hdrCacheInit() call below
 574     rep->header.append(&header);
 575     rep->hdrCacheInit();
 576     rep->hdr_sz = hdr_sz;
 577     rep->http_ver = http_ver;
 578     rep->pstate = pstate;
 579     rep->body_pipe = body_pipe;
 580
 581     // keep_alive is handled in hdrCacheInit()
 582     return rep;
 583 }
 584
 585 bool HttpReply::inheritProperties(const HttpMsg *aMsg)
 586 {
 587     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 588     if (!aRep)
 589         return false;
 590     keep_alive = aRep->keep_alive;
 591     return true;
 592 }
 593
 594 void HttpReply::removeStaleWarnings()
 595 {
 596     String warning;
 597     if (header.getList(Http::HdrType::WARNING, &warning)) {
 598         const String newWarning = removeStaleWarningValues(warning);
 599         if (warning.size() && warning.size() == newWarning.size())
 600             return; // some warnings are there and none changed
 601         header.delById(Http::HdrType::WARNING);
 602         if (newWarning.size()) { // some warnings left
 603             HttpHeaderEntry *const e =
 604                 new HttpHeaderEntry(Http::HdrType::WARNING, NULL, newWarning.termedBuf());
 605             header.addEntry(e);
 606         }
 607     }
 608 }
 609
 610 /**
 611  * Remove warning-values with warn-date different from Date value from
 612  * a single header entry. Returns a string with all valid warning-values.
 613  */
 614 String HttpReply::removeStaleWarningValues(const String &value)
 615 {
 616     String newValue;
 617     const char *item = 0;
 618     int len = 0;
 619     const char *pos = 0;
 620     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 621         bool keep = true;
 622         // Does warning-value have warn-date (which contains quoted date)?
 623         // We scan backwards, looking for two quoted strings.
 624         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 625         const char *p = item + len - 1;
 626
 627         while (p >= item && xisspace(*p)) --p; // skip whitespace
 628
 629         // warning-value MUST end with quote
 630         if (p >= item && *p == '"') {
 631             const char *const warnDateEnd = p;
 632             --p;
 633             while (p >= item && *p != '"') --p; // find the next quote
 634
 635             const char *warnDateBeg = p + 1;
 636             --p;
 637             while (p >= item && xisspace(*p)) --p; // skip whitespace
 638
 639             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 640                 // found warn-text
 641                 String warnDate;
 642                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 643                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 644                 keep = (time > 0 && time == date); // keep valid and matching date
 645             }
 646         }
 647
 648         if (keep) {
 649             if (newValue.size())
 650                 newValue.append(", ");
 651             newValue.append(item, len);
 652         }
 653     }
 654
 655     return newValue;
 656 }
 657