src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "base/EnumIterator.h"
  15 #include "globals.h"
  16 #include "http/ContentLengthInterpreter.h"
  17 #include "HttpBody.h"
  18 #include "HttpHdrCc.h"
  19 #include "HttpHdrContRange.h"
  20 #include "HttpHdrSc.h"
  21 #include "HttpReply.h"
  22 #include "HttpRequest.h"
  23 #include "MemBuf.h"
  24 #include "SquidConfig.h"
  25 #include "SquidTime.h"
  26 #include "Store.h"
  27 #include "StrList.h"
  28
  29 HttpReply::HttpReply():
  30     Http::Message(hoReply),
  31     date(0),
  32     last_modified(0),
  33     expires(0),
  34     surrogate_control(nullptr),
  35     keep_alive(0),
  36     protoPrefix("HTTP/"),
  37     bodySizeMax(-2),
  38     content_range(nullptr)
  39 {
  40     init();
  41 }
  42
  43 HttpReply::~HttpReply()
  44 {
  45     if (do_clean)
  46         clean();
  47 }
  48
  49 void
  50 HttpReply::init()
  51 {
  52     hdrCacheInit();
  53     sline.init();
  54     pstate = Http::Message::psReadyToParseStartLine;
  55     do_clean = true;
  56 }
  57
  58 void HttpReply::reset()
  59 {
  60
  61     // reset should not reset the protocol; could have made protoPrefix a
  62     // virtual function instead, but it is not clear whether virtual methods
  63     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  64     // conversions are not going to kill virtual tables
  65     const String pfx = protoPrefix;
  66     clean();
  67     init();
  68     protoPrefix = pfx;
  69 }
  70
  71 void
  72 HttpReply::clean()
  73 {
  74     // we used to assert that the pipe is NULL, but now the message only
  75     // points to a pipe that is owned and initiated by another object.
  76     body_pipe = NULL;
  77
  78     body.clear();
  79     hdrCacheClean();
  80     header.clean();
  81     sline.clean();
  82     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
  83 }
  84
  85 void
  86 HttpReply::packHeadersUsingFastPacker(Packable &p) const
  87 {
  88     sline.packInto(&p);
  89     header.packInto(&p);
  90     p.append("\r\n", 2);
  91 }
  92
  93 void
  94 HttpReply::packHeadersUsingSlowPacker(Packable &p) const
  95 {
  96     MemBuf buf;
  97     buf.init();
  98     packHeadersUsingFastPacker(buf);
  99     p.append(buf.content(), buf.contentSize());
 100 }
 101
 102 void
 103 HttpReply::packInto(MemBuf &buf) const
 104 {
 105     packHeadersUsingFastPacker(buf);
 106     body.packInto(&buf);
 107 }
 108
 109 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 110 MemBuf *
 111 HttpReply::pack() const
 112 {
 113     MemBuf *mb = new MemBuf;
 114     mb->init();
 115     packInto(*mb);
 116     return mb;
 117 }
 118
 119 HttpReplyPointer
 120 HttpReply::MakeConnectionEstablished() {
 121
 122     HttpReplyPointer rep(new HttpReply);
 123     rep->sline.set(Http::ProtocolVersion(), Http::scOkay, "Connection established");
 124     return rep;
 125 }
 126
 127 HttpReplyPointer
 128 HttpReply::make304() const
 129 {
 130     static const Http::HdrType ImsEntries[] = {Http::HdrType::DATE, Http::HdrType::CONTENT_TYPE, Http::HdrType::EXPIRES, Http::HdrType::LAST_MODIFIED, /* eof */ Http::HdrType::OTHER};
 131
 132     const HttpReplyPointer rv(new HttpReply);
 133     int t;
 134     HttpHeaderEntry *e;
 135
 136     /* rv->content_length; */
 137     rv->date = date;
 138     rv->last_modified = last_modified;
 139     rv->expires = expires;
 140     rv->content_type = content_type;
 141     /* rv->content_range */
 142     /* rv->keep_alive */
 143     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, NULL);
 144
 145     for (t = 0; ImsEntries[t] != Http::HdrType::OTHER; ++t) {
 146         if ((e = header.findEntry(ImsEntries[t])))
 147             rv->header.addEntry(e->clone());
 148     }
 149
 150     rv->putCc(cache_control);
 151
 152     /* rv->body */
 153     return rv;
 154 }
 155
 156 MemBuf *
 157 HttpReply::packed304Reply() const
 158 {
 159     /* Not as efficient as skipping the header duplication,
 160      * but easier to maintain
 161      */
 162     const auto temp = make304();
 163     MemBuf *rv = temp->pack();
 164     return rv;
 165 }
 166
 167 void
 168 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 169                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 170 {
 171     HttpHeader *hdr;
 172     sline.set(Http::ProtocolVersion(), status, reason);
 173     hdr = &header;
 174     hdr->putStr(Http::HdrType::SERVER, visible_appname_string);
 175     hdr->putStr(Http::HdrType::MIME_VERSION, "1.0");
 176     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 177
 178     if (ctype) {
 179         hdr->putStr(Http::HdrType::CONTENT_TYPE, ctype);
 180         content_type = ctype;
 181     } else
 182         content_type = String();
 183
 184     if (clen >= 0)
 185         hdr->putInt64(Http::HdrType::CONTENT_LENGTH, clen);
 186
 187     if (expiresTime >= 0)
 188         hdr->putTime(Http::HdrType::EXPIRES, expiresTime);
 189
 190     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 191         hdr->putTime(Http::HdrType::LAST_MODIFIED, lmt);
 192
 193     date = squid_curtime;
 194
 195     content_length = clen;
 196
 197     expires = expiresTime;
 198
 199     last_modified = lmt;
 200 }
 201
 202 void
 203 HttpReply::redirect(Http::StatusCode status, const char *loc)
 204 {
 205     HttpHeader *hdr;
 206     sline.set(Http::ProtocolVersion(), status, NULL);
 207     hdr = &header;
 208     hdr->putStr(Http::HdrType::SERVER, APP_FULLNAME);
 209     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 210     hdr->putInt64(Http::HdrType::CONTENT_LENGTH, 0);
 211     hdr->putStr(Http::HdrType::LOCATION, loc);
 212     date = squid_curtime;
 213     content_length = 0;
 214 }
 215
 216 /* compare the validators of two replies.
 217  * 1 = they match
 218  * 0 = they do not match
 219  */
 220 int
 221 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 222 {
 223     String one,two;
 224     assert (otherRep);
 225     /* Numbers first - easiest to check */
 226     /* Content-Length */
 227     /* TODO: remove -1 bypass */
 228
 229     if (content_length != otherRep->content_length
 230             && content_length > -1 &&
 231             otherRep->content_length > -1)
 232         return 0;
 233
 234     /* ETag */
 235     one = header.getStrOrList(Http::HdrType::ETAG);
 236
 237     two = otherRep->header.getStrOrList(Http::HdrType::ETAG);
 238
 239     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 240         one.clean();
 241         two.clean();
 242         return 0;
 243     }
 244
 245     if (last_modified != otherRep->last_modified)
 246         return 0;
 247
 248     /* MD5 */
 249     one = header.getStrOrList(Http::HdrType::CONTENT_MD5);
 250
 251     two = otherRep->header.getStrOrList(Http::HdrType::CONTENT_MD5);
 252
 253     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 254         one.clean();
 255         two.clean();
 256         return 0;
 257     }
 258
 259     return 1;
 260 }
 261
 262 HttpReply::Pointer
 263 HttpReply::recreateOnNotModified(const HttpReply &reply304) const
 264 {
 265     // If enough 304s do not update, then this expensive checking is cheaper
 266     // than blindly storing reply prefix identical to the already stored one.
 267     if (!header.needUpdate(&reply304.header))
 268         return nullptr;
 269
 270     const Pointer cloned = clone();
 271     cloned->header.update(&reply304.header);
 272     cloned->hdrCacheClean();
 273     cloned->header.compact();
 274     cloned->hdrCacheInit();
 275     return cloned;
 276 }
 277
 278 /* internal routines */
 279
 280 time_t
 281 HttpReply::hdrExpirationTime()
 282 {
 283     /* The s-maxage and max-age directive takes priority over Expires */
 284
 285     if (cache_control) {
 286         int maxAge = -1;
 287         /*
 288          * Conservatively handle the case when we have a max-age
 289          * header, but no Date for reference?
 290          */
 291         if (cache_control->hasSMaxAge(&maxAge) || cache_control->hasMaxAge(&maxAge))
 292             return (date >= 0) ? date + maxAge : squid_curtime;
 293     }
 294
 295     if (Config.onoff.vary_ignore_expire &&
 296             header.has(Http::HdrType::VARY)) {
 297         const time_t d = header.getTime(Http::HdrType::DATE);
 298         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 299
 300         if (d == e)
 301             return -1;
 302     }
 303
 304     if (header.has(Http::HdrType::EXPIRES)) {
 305         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 306         /*
 307          * HTTP/1.0 says that robust implementations should consider
 308          * bad or malformed Expires header as equivalent to "expires
 309          * immediately."
 310          */
 311         return e < 0 ? squid_curtime : e;
 312     }
 313
 314     return -1;
 315 }
 316
 317 /* sync this routine when you update HttpReply struct */
 318 void
 319 HttpReply::hdrCacheInit()
 320 {
 321     Http::Message::hdrCacheInit();
 322
 323     http_ver = sline.version;
 324     content_length = header.getInt64(Http::HdrType::CONTENT_LENGTH);
 325     date = header.getTime(Http::HdrType::DATE);
 326     last_modified = header.getTime(Http::HdrType::LAST_MODIFIED);
 327     surrogate_control = header.getSc();
 328     content_range = (sline.status() == Http::scPartialContent) ?
 329                     header.getContRange() : nullptr;
 330     keep_alive = persistent() ? 1 : 0;
 331     const char *str = header.getStr(Http::HdrType::CONTENT_TYPE);
 332
 333     if (str)
 334         content_type.assign(str, strcspn(str, ";\t "));
 335     else
 336         content_type = String();
 337
 338     /* be sure to set expires after date and cache-control */
 339     expires = hdrExpirationTime();
 340 }
 341
 342 const HttpHdrContRange *
 343 HttpReply::contentRange() const
 344 {
 345     assert(!content_range || sline.status() == Http::scPartialContent);
 346     return content_range;
 347 }
 348
 349 /* sync this routine when you update HttpReply struct */
 350 void
 351 HttpReply::hdrCacheClean()
 352 {
 353     content_type.clean();
 354
 355     if (cache_control) {
 356         delete cache_control;
 357         cache_control = NULL;
 358     }
 359
 360     if (surrogate_control) {
 361         delete surrogate_control;
 362         surrogate_control = NULL;
 363     }
 364
 365     if (content_range) {
 366         delete content_range;
 367         content_range = NULL;
 368     }
 369 }
 370
 371 /*
 372  * Returns the body size of a HTTP response
 373  */
 374 int64_t
 375 HttpReply::bodySize(const HttpRequestMethod& method) const
 376 {
 377     if (sline.version.major < 1)
 378         return -1;
 379     else if (method.id() == Http::METHOD_HEAD)
 380         return 0;
 381     else if (sline.status() == Http::scOkay)
 382         (void) 0;       /* common case, continue */
 383     else if (sline.status() == Http::scNoContent)
 384         return 0;
 385     else if (sline.status() == Http::scNotModified)
 386         return 0;
 387     else if (sline.status() < Http::scOkay)
 388         return 0;
 389
 390     return content_length;
 391 }
 392
 393 /**
 394  * Checks the first line of an HTTP Reply is valid.
 395  * currently only checks "HTTP/" exists.
 396  *
 397  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 398  */
 399 bool
 400 HttpReply::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 401 {
 402     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 403
 404     // content is long enough to possibly hold a reply
 405     // 4 being magic size of a 3-digit number plus space delimiter
 406     if (hdr_len < (size_t)(protoPrefix.psize() + 4)) {
 407         if (hdr_len > 0) {
 408             debugs(58, 3, "Too small reply header (" << hdr_len << " bytes)");
 409             *error = Http::scInvalidHeader;
 410         }
 411         return false;
 412     }
 413
 414     int pos;
 415     // catch missing or mismatched protocol identifier
 416     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 417     if (strncmp(buf, "ICY", 3) == 0) {
 418         protoPrefix = "ICY";
 419         pos = protoPrefix.psize();
 420     } else {
 421
 422         if (protoPrefix.cmp(buf, protoPrefix.size()) != 0) {
 423             debugs(58, 3, "missing protocol prefix (" << protoPrefix << ") in '" << buf << "'");
 424             *error = Http::scInvalidHeader;
 425             return false;
 426         }
 427
 428         // catch missing or negative status value (negative '-' is not a digit)
 429         pos = protoPrefix.psize();
 430
 431         // skip arbitrary number of digits and a dot in the verion portion
 432         while ((size_t)pos <= hdr_len && (*(buf+pos) == '.' || xisdigit(*(buf+pos)) ) ) ++pos;
 433
 434         // catch missing version info
 435         if (pos == protoPrefix.psize()) {
 436             debugs(58, 3, "missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf << "'");
 437             *error = Http::scInvalidHeader;
 438             return false;
 439         }
 440     }
 441
 442     // skip arbitrary number of spaces...
 443     while ((size_t)pos <= hdr_len && (char)*(buf+pos) == ' ') ++pos;
 444
 445     if ((size_t)pos < hdr_len && !xisdigit(*(buf+pos))) {
 446         debugs(58, 3, "missing or invalid status number in '" << buf << "'");
 447         *error = Http::scInvalidHeader;
 448         return false;
 449     }
 450
 451     return true;
 452 }
 453
 454 bool
 455 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 456 {
 457     return sline.parse(protoPrefix, blk_start, blk_end);
 458 }
 459
 460 void
 461 HttpReply::configureContentLengthInterpreter(Http::ContentLengthInterpreter &interpreter)
 462 {
 463     interpreter.applyStatusCodeRules(sline.status());
 464 }
 465
 466 bool
 467 HttpReply::parseHeader(Http1::Parser &hp)
 468 {
 469     Http::ContentLengthInterpreter clen;
 470     return Message::parseHeader(hp, clen);
 471 }
 472
 473 /* handy: resets and returns -1 */
 474 int
 475 HttpReply::httpMsgParseError()
 476 {
 477     int result(Http::Message::httpMsgParseError());
 478     /* indicate an error in the status line */
 479     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 480     return result;
 481 }
 482
 483 /*
 484  * Indicate whether or not we would usually expect an entity-body
 485  * along with this response
 486  */
 487 bool
 488 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 489 {
 490     bool expectBody = true;
 491
 492     if (req_method == Http::METHOD_HEAD)
 493         expectBody = false;
 494     else if (sline.status() == Http::scNoContent)
 495         expectBody = false;
 496     else if (sline.status() == Http::scNotModified)
 497         expectBody = false;
 498     // TODO: Consider assuming that gray-area 0xx responses have bodies, like 9xx responses.
 499     else if (sline.status() < Http::scOkay)
 500         expectBody = false;
 501     else
 502         expectBody = true;
 503
 504     if (expectBody) {
 505         if (header.chunked())
 506             theSize = -1;
 507         else if (content_length >= 0)
 508             theSize = content_length;
 509         else
 510             theSize = -1;
 511     }
 512
 513     return expectBody;
 514 }
 515
 516 bool
 517 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 518 {
 519     calcMaxBodySize(request);
 520     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 521     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 522 }
 523
 524 bool
 525 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 526 {
 527     calcMaxBodySize(request);
 528     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 529
 530     if (bodySizeMax < 0) // no body size limit
 531         return false;
 532
 533     int64_t expectedSize = -1;
 534     if (!expectingBody(request.method, expectedSize))
 535         return false;
 536
 537     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 538
 539     if (expectedSize < 0) // expecting body of an unknown length
 540         return false;
 541
 542     return expectedSize > bodySizeMax;
 543 }
 544
 545 void
 546 HttpReply::calcMaxBodySize(HttpRequest& request) const
 547 {
 548     // hack: -2 is used as "we have not calculated max body size yet" state
 549     if (bodySizeMax != -2) // already tried
 550         return;
 551     bodySizeMax = -1;
 552
 553     // short-circuit ACL testing if there are none configured
 554     if (!Config.ReplyBodySize)
 555         return;
 556
 557     ACLFilledChecklist ch(NULL, &request, NULL);
 558     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 559     ch.reply = const_cast<HttpReply *>(this);
 560     HTTPMSGLOCK(ch.reply);
 561     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 562         /* if there is no ACL list or if the ACLs listed match use this size value */
 563         if (!l->aclList || ch.fastCheck(l->aclList).allowed()) {
 564             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 565             bodySizeMax = l->size; // may be -1
 566             break;
 567         }
 568     }
 569 }
 570
 571 // XXX: check that this is sufficient for eCAP cloning
 572 HttpReply *
 573 HttpReply::clone() const
 574 {
 575     HttpReply *rep = new HttpReply();
 576     rep->sline = sline; // used in hdrCacheInit() call below
 577     rep->header.append(&header);
 578     rep->hdrCacheInit();
 579     rep->hdr_sz = hdr_sz;
 580     rep->http_ver = http_ver;
 581     rep->pstate = pstate;
 582     rep->body_pipe = body_pipe;
 583
 584     // keep_alive is handled in hdrCacheInit()
 585     return rep;
 586 }
 587
 588 bool
 589 HttpReply::inheritProperties(const Http::Message *aMsg)
 590 {
 591     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 592     if (!aRep)
 593         return false;
 594     keep_alive = aRep->keep_alive;
 595     sources = aRep->sources;
 596     return true;
 597 }
 598
 599 void HttpReply::removeStaleWarnings()
 600 {
 601     String warning;
 602     if (header.getList(Http::HdrType::WARNING, &warning)) {
 603         const String newWarning = removeStaleWarningValues(warning);
 604         if (warning.size() && warning.size() == newWarning.size())
 605             return; // some warnings are there and none changed
 606         header.delById(Http::HdrType::WARNING);
 607         if (newWarning.size()) { // some warnings left
 608             HttpHeaderEntry *const e =
 609                 new HttpHeaderEntry(Http::HdrType::WARNING, SBuf(), newWarning.termedBuf());
 610             header.addEntry(e);
 611         }
 612     }
 613 }
 614
 615 /**
 616  * Remove warning-values with warn-date different from Date value from
 617  * a single header entry. Returns a string with all valid warning-values.
 618  */
 619 String HttpReply::removeStaleWarningValues(const String &value)
 620 {
 621     String newValue;
 622     const char *item = 0;
 623     int len = 0;
 624     const char *pos = 0;
 625     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 626         bool keep = true;
 627         // Does warning-value have warn-date (which contains quoted date)?
 628         // We scan backwards, looking for two quoted strings.
 629         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 630         const char *p = item + len - 1;
 631
 632         while (p >= item && xisspace(*p)) --p; // skip whitespace
 633
 634         // warning-value MUST end with quote
 635         if (p >= item && *p == '"') {
 636             const char *const warnDateEnd = p;
 637             --p;
 638             while (p >= item && *p != '"') --p; // find the next quote
 639
 640             const char *warnDateBeg = p + 1;
 641             --p;
 642             while (p >= item && xisspace(*p)) --p; // skip whitespace
 643
 644             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 645                 // found warn-text
 646                 String warnDate;
 647                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 648                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 649                 keep = (time > 0 && time == date); // keep valid and matching date
 650             }
 651         }
 652
 653         if (keep) {
 654             if (newValue.size())
 655                 newValue.append(", ");
 656             newValue.append(item, len);
 657         }
 658     }
 659
 660     return newValue;
 661 }
 662
 663 bool
 664 HttpReply::olderThan(const HttpReply *them) const
 665 {
 666     if (!them || !them->date || !date)
 667         return false;
 668     return date < them->date;
 669 }
 670
 671 void
 672 HttpReply::removeIrrelevantContentLength() {
 673     if (Http::ProhibitsContentLength(sline.status()))
 674         if (header.delById(Http::HdrType::CONTENT_LENGTH))
 675             debugs(58, 3, "Removing unexpected Content-Length header");
 676 }
 677