src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "base/EnumIterator.h"
  15 #include "globals.h"
  16 #include "HttpBody.h"
  17 #include "HttpHdrCc.h"
  18 #include "HttpHdrContRange.h"
  19 #include "HttpHdrSc.h"
  20 #include "HttpReply.h"
  21 #include "HttpRequest.h"
  22 #include "MemBuf.h"
  23 #include "SquidConfig.h"
  24 #include "SquidTime.h"
  25 #include "Store.h"
  26 #include "StrList.h"
  27
  28 HttpReply::HttpReply() : HttpMsg(hoReply), date (0), last_modified (0),
  29     expires (0), surrogate_control (NULL), content_range (NULL), keep_alive (0),
  30     protoPrefix("HTTP/"), bodySizeMax(-2)
  31 {
  32     init();
  33 }
  34
  35 HttpReply::~HttpReply()
  36 {
  37     if (do_clean)
  38         clean();
  39 }
  40
  41 void
  42 HttpReply::init()
  43 {
  44     hdrCacheInit();
  45     sline.init();
  46     pstate = psReadyToParseStartLine;
  47     do_clean = true;
  48 }
  49
  50 void HttpReply::reset()
  51 {
  52
  53     // reset should not reset the protocol; could have made protoPrefix a
  54     // virtual function instead, but it is not clear whether virtual methods
  55     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  56     // conversions are not going to kill virtual tables
  57     const String pfx = protoPrefix;
  58     clean();
  59     init();
  60     protoPrefix = pfx;
  61 }
  62
  63 void
  64 HttpReply::clean()
  65 {
  66     // we used to assert that the pipe is NULL, but now the message only
  67     // points to a pipe that is owned and initiated by another object.
  68     body_pipe = NULL;
  69
  70     body.clear();
  71     hdrCacheClean();
  72     header.clean();
  73     sline.clean();
  74     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
  75 }
  76
  77 void
  78 HttpReply::packHeadersInto(Packable * p) const
  79 {
  80     sline.packInto(p);
  81     header.packInto(p);
  82     p->append("\r\n", 2);
  83 }
  84
  85 void
  86 HttpReply::packInto(Packable * p) const
  87 {
  88     packHeadersInto(p);
  89     body.packInto(p);
  90 }
  91
  92 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
  93 MemBuf *
  94 HttpReply::pack() const
  95 {
  96     MemBuf *mb = new MemBuf;
  97     mb->init();
  98     packInto(mb);
  99     return mb;
 100 }
 101
 102 HttpReply *
 103 HttpReply::make304() const
 104 {
 105     static const Http::HdrType ImsEntries[] = {Http::HdrType::DATE, Http::HdrType::CONTENT_TYPE, Http::HdrType::EXPIRES, Http::HdrType::LAST_MODIFIED, /* eof */ Http::HdrType::OTHER};
 106
 107     HttpReply *rv = new HttpReply;
 108     int t;
 109     HttpHeaderEntry *e;
 110
 111     /* rv->content_length; */
 112     rv->date = date;
 113     rv->last_modified = last_modified;
 114     rv->expires = expires;
 115     rv->content_type = content_type;
 116     /* rv->content_range */
 117     /* rv->keep_alive */
 118     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, NULL);
 119
 120     for (t = 0; ImsEntries[t] != Http::HdrType::OTHER; ++t) {
 121         if ((e = header.findEntry(ImsEntries[t])))
 122             rv->header.addEntry(e->clone());
 123     }
 124
 125     rv->putCc(cache_control);
 126
 127     /* rv->body */
 128     return rv;
 129 }
 130
 131 MemBuf *
 132 HttpReply::packed304Reply() const
 133 {
 134     /* Not as efficient as skipping the header duplication,
 135      * but easier to maintain
 136      */
 137     HttpReply *temp = make304();
 138     MemBuf *rv = temp->pack();
 139     delete temp;
 140     return rv;
 141 }
 142
 143 void
 144 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 145                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 146 {
 147     HttpHeader *hdr;
 148     sline.set(Http::ProtocolVersion(), status, reason);
 149     hdr = &header;
 150     hdr->putStr(Http::HdrType::SERVER, visible_appname_string);
 151     hdr->putStr(Http::HdrType::MIME_VERSION, "1.0");
 152     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 153
 154     if (ctype) {
 155         hdr->putStr(Http::HdrType::CONTENT_TYPE, ctype);
 156         content_type = ctype;
 157     } else
 158         content_type = String();
 159
 160     if (clen >= 0)
 161         hdr->putInt64(Http::HdrType::CONTENT_LENGTH, clen);
 162
 163     if (expiresTime >= 0)
 164         hdr->putTime(Http::HdrType::EXPIRES, expiresTime);
 165
 166     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 167         hdr->putTime(Http::HdrType::LAST_MODIFIED, lmt);
 168
 169     date = squid_curtime;
 170
 171     content_length = clen;
 172
 173     expires = expiresTime;
 174
 175     last_modified = lmt;
 176 }
 177
 178 void
 179 HttpReply::redirect(Http::StatusCode status, const char *loc)
 180 {
 181     HttpHeader *hdr;
 182     sline.set(Http::ProtocolVersion(), status, NULL);
 183     hdr = &header;
 184     hdr->putStr(Http::HdrType::SERVER, APP_FULLNAME);
 185     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 186     hdr->putInt64(Http::HdrType::CONTENT_LENGTH, 0);
 187     hdr->putStr(Http::HdrType::LOCATION, loc);
 188     date = squid_curtime;
 189     content_length = 0;
 190 }
 191
 192 /* compare the validators of two replies.
 193  * 1 = they match
 194  * 0 = they do not match
 195  */
 196 int
 197 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 198 {
 199     String one,two;
 200     assert (otherRep);
 201     /* Numbers first - easiest to check */
 202     /* Content-Length */
 203     /* TODO: remove -1 bypass */
 204
 205     if (content_length != otherRep->content_length
 206             && content_length > -1 &&
 207             otherRep->content_length > -1)
 208         return 0;
 209
 210     /* ETag */
 211     one = header.getStrOrList(Http::HdrType::ETAG);
 212
 213     two = otherRep->header.getStrOrList(Http::HdrType::ETAG);
 214
 215     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 216         one.clean();
 217         two.clean();
 218         return 0;
 219     }
 220
 221     if (last_modified != otherRep->last_modified)
 222         return 0;
 223
 224     /* MD5 */
 225     one = header.getStrOrList(Http::HdrType::CONTENT_MD5);
 226
 227     two = otherRep->header.getStrOrList(Http::HdrType::CONTENT_MD5);
 228
 229     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 230         one.clean();
 231         two.clean();
 232         return 0;
 233     }
 234
 235     return 1;
 236 }
 237
 238 bool
 239 HttpReply::updateOnNotModified(HttpReply const * freshRep)
 240 {
 241     assert(freshRep);
 242
 243     /* update raw headers */
 244     if (!header.update(&freshRep->header))
 245         return false;
 246
 247     /* clean cache */
 248     hdrCacheClean();
 249
 250     header.compact();
 251     /* init cache */
 252     hdrCacheInit();
 253
 254     return true;
 255 }
 256
 257 /* internal routines */
 258
 259 time_t
 260 HttpReply::hdrExpirationTime()
 261 {
 262     /* The s-maxage and max-age directive takes priority over Expires */
 263
 264     if (cache_control) {
 265         if (date >= 0) {
 266             if (cache_control->hasSMaxAge())
 267                 return date + cache_control->sMaxAge();
 268
 269             if (cache_control->hasMaxAge())
 270                 return date + cache_control->maxAge();
 271         } else {
 272             /*
 273              * Conservatively handle the case when we have a max-age
 274              * header, but no Date for reference?
 275              */
 276
 277             if (cache_control->hasSMaxAge())
 278                 return squid_curtime;
 279
 280             if (cache_control->hasMaxAge())
 281                 return squid_curtime;
 282         }
 283     }
 284
 285     if (Config.onoff.vary_ignore_expire &&
 286             header.has(Http::HdrType::VARY)) {
 287         const time_t d = header.getTime(Http::HdrType::DATE);
 288         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 289
 290         if (d == e)
 291             return -1;
 292     }
 293
 294     if (header.has(Http::HdrType::EXPIRES)) {
 295         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 296         /*
 297          * HTTP/1.0 says that robust implementations should consider
 298          * bad or malformed Expires header as equivalent to "expires
 299          * immediately."
 300          */
 301         return e < 0 ? squid_curtime : e;
 302     }
 303
 304     return -1;
 305 }
 306
 307 /* sync this routine when you update HttpReply struct */
 308 void
 309 HttpReply::hdrCacheInit()
 310 {
 311     HttpMsg::hdrCacheInit();
 312
 313     http_ver = sline.version;
 314     content_length = header.getInt64(Http::HdrType::CONTENT_LENGTH);
 315     date = header.getTime(Http::HdrType::DATE);
 316     last_modified = header.getTime(Http::HdrType::LAST_MODIFIED);
 317     surrogate_control = header.getSc();
 318     content_range = header.getContRange();
 319     keep_alive = persistent() ? 1 : 0;
 320     const char *str = header.getStr(Http::HdrType::CONTENT_TYPE);
 321
 322     if (str)
 323         content_type.limitInit(str, strcspn(str, ";\t "));
 324     else
 325         content_type = String();
 326
 327     /* be sure to set expires after date and cache-control */
 328     expires = hdrExpirationTime();
 329 }
 330
 331 /* sync this routine when you update HttpReply struct */
 332 void
 333 HttpReply::hdrCacheClean()
 334 {
 335     content_type.clean();
 336
 337     if (cache_control) {
 338         delete cache_control;
 339         cache_control = NULL;
 340     }
 341
 342     if (surrogate_control) {
 343         delete surrogate_control;
 344         surrogate_control = NULL;
 345     }
 346
 347     if (content_range) {
 348         delete content_range;
 349         content_range = NULL;
 350     }
 351 }
 352
 353 /*
 354  * Returns the body size of a HTTP response
 355  */
 356 int64_t
 357 HttpReply::bodySize(const HttpRequestMethod& method) const
 358 {
 359     if (sline.version.major < 1)
 360         return -1;
 361     else if (method.id() == Http::METHOD_HEAD)
 362         return 0;
 363     else if (sline.status() == Http::scOkay)
 364         (void) 0;       /* common case, continue */
 365     else if (sline.status() == Http::scNoContent)
 366         return 0;
 367     else if (sline.status() == Http::scNotModified)
 368         return 0;
 369     else if (sline.status() < Http::scOkay)
 370         return 0;
 371
 372     return content_length;
 373 }
 374
 375 /**
 376  * Checks the first line of an HTTP Reply is valid.
 377  * currently only checks "HTTP/" exists.
 378  *
 379  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 380  */
 381 bool
 382 HttpReply::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 383 {
 384     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 385
 386     // content is long enough to possibly hold a reply
 387     // 4 being magic size of a 3-digit number plus space delimiter
 388     if (hdr_len < (size_t)(protoPrefix.psize() + 4)) {
 389         if (hdr_len > 0) {
 390             debugs(58, 3, "Too small reply header (" << hdr_len << " bytes)");
 391             *error = Http::scInvalidHeader;
 392         }
 393         return false;
 394     }
 395
 396     int pos;
 397     // catch missing or mismatched protocol identifier
 398     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 399     if (strncmp(buf, "ICY", 3) == 0) {
 400         protoPrefix = "ICY";
 401         pos = protoPrefix.psize();
 402     } else {
 403
 404         if (protoPrefix.cmp(buf, protoPrefix.size()) != 0) {
 405             debugs(58, 3, "missing protocol prefix (" << protoPrefix << ") in '" << buf << "'");
 406             *error = Http::scInvalidHeader;
 407             return false;
 408         }
 409
 410         // catch missing or negative status value (negative '-' is not a digit)
 411         pos = protoPrefix.psize();
 412
 413         // skip arbitrary number of digits and a dot in the verion portion
 414         while ((size_t)pos <= hdr_len && (*(buf+pos) == '.' || xisdigit(*(buf+pos)) ) ) ++pos;
 415
 416         // catch missing version info
 417         if (pos == protoPrefix.psize()) {
 418             debugs(58, 3, "missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf << "'");
 419             *error = Http::scInvalidHeader;
 420             return false;
 421         }
 422     }
 423
 424     // skip arbitrary number of spaces...
 425     while ((size_t)pos <= hdr_len && (char)*(buf+pos) == ' ') ++pos;
 426
 427     if ((size_t)pos < hdr_len && !xisdigit(*(buf+pos))) {
 428         debugs(58, 3, "missing or invalid status number in '" << buf << "'");
 429         *error = Http::scInvalidHeader;
 430         return false;
 431     }
 432
 433     return true;
 434 }
 435
 436 bool
 437 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 438 {
 439     return sline.parse(protoPrefix, blk_start, blk_end);
 440 }
 441
 442 /* handy: resets and returns -1 */
 443 int
 444 HttpReply::httpMsgParseError()
 445 {
 446     int result(HttpMsg::httpMsgParseError());
 447     /* indicate an error in the status line */
 448     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 449     return result;
 450 }
 451
 452 /*
 453  * Indicate whether or not we would usually expect an entity-body
 454  * along with this response
 455  */
 456 bool
 457 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 458 {
 459     bool expectBody = true;
 460
 461     if (req_method == Http::METHOD_HEAD)
 462         expectBody = false;
 463     else if (sline.status() == Http::scNoContent)
 464         expectBody = false;
 465     else if (sline.status() == Http::scNotModified)
 466         expectBody = false;
 467     else if (sline.status() < Http::scOkay)
 468         expectBody = false;
 469     else
 470         expectBody = true;
 471
 472     if (expectBody) {
 473         if (header.chunked())
 474             theSize = -1;
 475         else if (content_length >= 0)
 476             theSize = content_length;
 477         else
 478             theSize = -1;
 479     }
 480
 481     return expectBody;
 482 }
 483
 484 bool
 485 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 486 {
 487     calcMaxBodySize(request);
 488     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 489     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 490 }
 491
 492 bool
 493 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 494 {
 495     calcMaxBodySize(request);
 496     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 497
 498     if (bodySizeMax < 0) // no body size limit
 499         return false;
 500
 501     int64_t expectedSize = -1;
 502     if (!expectingBody(request.method, expectedSize))
 503         return false;
 504
 505     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 506
 507     if (expectedSize < 0) // expecting body of an unknown length
 508         return false;
 509
 510     return expectedSize > bodySizeMax;
 511 }
 512
 513 void
 514 HttpReply::calcMaxBodySize(HttpRequest& request) const
 515 {
 516     // hack: -2 is used as "we have not calculated max body size yet" state
 517     if (bodySizeMax != -2) // already tried
 518         return;
 519     bodySizeMax = -1;
 520
 521     // short-circuit ACL testing if there are none configured
 522     if (!Config.ReplyBodySize)
 523         return;
 524
 525     ACLFilledChecklist ch(NULL, &request, NULL);
 526     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 527     ch.reply = const_cast<HttpReply *>(this);
 528     HTTPMSGLOCK(ch.reply);
 529     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 530         /* if there is no ACL list or if the ACLs listed match use this size value */
 531         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 532             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 533             bodySizeMax = l->size; // may be -1
 534             break;
 535         }
 536     }
 537 }
 538
 539 // XXX: check that this is sufficient for eCAP cloning
 540 HttpReply *
 541 HttpReply::clone() const
 542 {
 543     HttpReply *rep = new HttpReply();
 544     rep->sline = sline; // used in hdrCacheInit() call below
 545     rep->header.append(&header);
 546     rep->hdrCacheInit();
 547     rep->hdr_sz = hdr_sz;
 548     rep->http_ver = http_ver;
 549     rep->pstate = pstate;
 550     rep->body_pipe = body_pipe;
 551
 552     // keep_alive is handled in hdrCacheInit()
 553     return rep;
 554 }
 555
 556 bool HttpReply::inheritProperties(const HttpMsg *aMsg)
 557 {
 558     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 559     if (!aRep)
 560         return false;
 561     keep_alive = aRep->keep_alive;
 562     sources = aRep->sources;
 563     return true;
 564 }
 565
 566 void HttpReply::removeStaleWarnings()
 567 {
 568     String warning;
 569     if (header.getList(Http::HdrType::WARNING, &warning)) {
 570         const String newWarning = removeStaleWarningValues(warning);
 571         if (warning.size() && warning.size() == newWarning.size())
 572             return; // some warnings are there and none changed
 573         header.delById(Http::HdrType::WARNING);
 574         if (newWarning.size()) { // some warnings left
 575             HttpHeaderEntry *const e =
 576                 new HttpHeaderEntry(Http::HdrType::WARNING, NULL, newWarning.termedBuf());
 577             header.addEntry(e);
 578         }
 579     }
 580 }
 581
 582 /**
 583  * Remove warning-values with warn-date different from Date value from
 584  * a single header entry. Returns a string with all valid warning-values.
 585  */
 586 String HttpReply::removeStaleWarningValues(const String &value)
 587 {
 588     String newValue;
 589     const char *item = 0;
 590     int len = 0;
 591     const char *pos = 0;
 592     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 593         bool keep = true;
 594         // Does warning-value have warn-date (which contains quoted date)?
 595         // We scan backwards, looking for two quoted strings.
 596         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 597         const char *p = item + len - 1;
 598
 599         while (p >= item && xisspace(*p)) --p; // skip whitespace
 600
 601         // warning-value MUST end with quote
 602         if (p >= item && *p == '"') {
 603             const char *const warnDateEnd = p;
 604             --p;
 605             while (p >= item && *p != '"') --p; // find the next quote
 606
 607             const char *warnDateBeg = p + 1;
 608             --p;
 609             while (p >= item && xisspace(*p)) --p; // skip whitespace
 610
 611             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 612                 // found warn-text
 613                 String warnDate;
 614                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 615                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 616                 keep = (time > 0 && time == date); // keep valid and matching date
 617             }
 618         }
 619
 620         if (keep) {
 621             if (newValue.size())
 622                 newValue.append(", ");
 623             newValue.append(item, len);
 624         }
 625     }
 626
 627     return newValue;
 628 }
 629
 630 bool
 631 HttpReply::olderThan(const HttpReply *them) const
 632 {
 633     if (!them || !them->date || !date)
 634         return false;
 635     return date < them->date;
 636 }
 637