src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "base/EnumIterator.h"
  15 #include "globals.h"
  16 #include "http/ContentLengthInterpreter.h"
  17 #include "HttpBody.h"
  18 #include "HttpHdrCc.h"
  19 #include "HttpHdrContRange.h"
  20 #include "HttpHdrSc.h"
  21 #include "HttpReply.h"
  22 #include "HttpRequest.h"
  23 #include "MemBuf.h"
  24 #include "SquidConfig.h"
  25 #include "Store.h"
  26 #include "StrList.h"
  27
  28 HttpReply::HttpReply():
  29     Http::Message(hoReply),
  30     date(0),
  31     last_modified(0),
  32     expires(0),
  33     surrogate_control(nullptr),
  34     keep_alive(0),
  35     protoPrefix("HTTP/"),
  36     bodySizeMax(-2),
  37     content_range(nullptr)
  38 {
  39     init();
  40 }
  41
  42 HttpReply::~HttpReply()
  43 {
  44     if (do_clean)
  45         clean();
  46 }
  47
  48 void
  49 HttpReply::init()
  50 {
  51     hdrCacheInit();
  52     sline.init();
  53     pstate = Http::Message::psReadyToParseStartLine;
  54     do_clean = true;
  55 }
  56
  57 void HttpReply::reset()
  58 {
  59
  60     // reset should not reset the protocol; could have made protoPrefix a
  61     // virtual function instead, but it is not clear whether virtual methods
  62     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  63     // conversions are not going to kill virtual tables
  64     const String pfx = protoPrefix;
  65     clean();
  66     init();
  67     protoPrefix = pfx;
  68 }
  69
  70 void
  71 HttpReply::clean()
  72 {
  73     // we used to assert that the pipe is NULL, but now the message only
  74     // points to a pipe that is owned and initiated by another object.
  75     body_pipe = nullptr;
  76
  77     body.clear();
  78     hdrCacheClean();
  79     header.clean();
  80     sline.clean();
  81     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
  82 }
  83
  84 void
  85 HttpReply::packHeadersUsingFastPacker(Packable &p) const
  86 {
  87     sline.packInto(&p);
  88     header.packInto(&p);
  89     p.append("\r\n", 2);
  90 }
  91
  92 void
  93 HttpReply::packHeadersUsingSlowPacker(Packable &p) const
  94 {
  95     MemBuf buf;
  96     buf.init();
  97     packHeadersUsingFastPacker(buf);
  98     p.append(buf.content(), buf.contentSize());
  99 }
 100
 101 void
 102 HttpReply::packInto(MemBuf &buf) const
 103 {
 104     packHeadersUsingFastPacker(buf);
 105     body.packInto(&buf);
 106 }
 107
 108 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 109 MemBuf *
 110 HttpReply::pack() const
 111 {
 112     MemBuf *mb = new MemBuf;
 113     mb->init();
 114     packInto(*mb);
 115     return mb;
 116 }
 117
 118 HttpReplyPointer
 119 HttpReply::MakeConnectionEstablished() {
 120
 121     HttpReplyPointer rep(new HttpReply);
 122     rep->sline.set(Http::ProtocolVersion(), Http::scOkay, "Connection established");
 123     return rep;
 124 }
 125
 126 HttpReplyPointer
 127 HttpReply::make304() const
 128 {
 129     static const Http::HdrType ImsEntries[] = {Http::HdrType::DATE, Http::HdrType::CONTENT_TYPE, Http::HdrType::EXPIRES, Http::HdrType::LAST_MODIFIED, /* eof */ Http::HdrType::OTHER};
 130
 131     const HttpReplyPointer rv(new HttpReply);
 132     int t;
 133     HttpHeaderEntry *e;
 134
 135     /* rv->content_length; */
 136     rv->date = date;
 137     rv->last_modified = last_modified;
 138     rv->expires = expires;
 139     rv->content_type = content_type;
 140     /* rv->content_range */
 141     /* rv->keep_alive */
 142     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, nullptr);
 143
 144     for (t = 0; ImsEntries[t] != Http::HdrType::OTHER; ++t) {
 145         if ((e = header.findEntry(ImsEntries[t])))
 146             rv->header.addEntry(e->clone());
 147     }
 148
 149     rv->putCc(cache_control);
 150
 151     /* rv->body */
 152     return rv;
 153 }
 154
 155 MemBuf *
 156 HttpReply::packed304Reply() const
 157 {
 158     /* Not as efficient as skipping the header duplication,
 159      * but easier to maintain
 160      */
 161     const auto temp = make304();
 162     MemBuf *rv = temp->pack();
 163     return rv;
 164 }
 165
 166 void
 167 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 168                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 169 {
 170     HttpHeader *hdr;
 171     sline.set(Http::ProtocolVersion(), status, reason);
 172     hdr = &header;
 173     hdr->putStr(Http::HdrType::SERVER, visible_appname_string);
 174     hdr->putStr(Http::HdrType::MIME_VERSION, "1.0");
 175     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 176
 177     if (ctype) {
 178         hdr->putStr(Http::HdrType::CONTENT_TYPE, ctype);
 179         content_type = ctype;
 180     } else
 181         content_type = String();
 182
 183     if (clen >= 0)
 184         hdr->putInt64(Http::HdrType::CONTENT_LENGTH, clen);
 185
 186     if (expiresTime >= 0)
 187         hdr->putTime(Http::HdrType::EXPIRES, expiresTime);
 188
 189     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 190         hdr->putTime(Http::HdrType::LAST_MODIFIED, lmt);
 191
 192     date = squid_curtime;
 193
 194     content_length = clen;
 195
 196     expires = expiresTime;
 197
 198     last_modified = lmt;
 199 }
 200
 201 void
 202 HttpReply::redirect(Http::StatusCode status, const char *loc)
 203 {
 204     HttpHeader *hdr;
 205     sline.set(Http::ProtocolVersion(), status, nullptr);
 206     hdr = &header;
 207     hdr->putStr(Http::HdrType::SERVER, APP_FULLNAME);
 208     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 209     hdr->putInt64(Http::HdrType::CONTENT_LENGTH, 0);
 210     hdr->putStr(Http::HdrType::LOCATION, loc);
 211     date = squid_curtime;
 212     content_length = 0;
 213 }
 214
 215 /* compare the validators of two replies.
 216  * 1 = they match
 217  * 0 = they do not match
 218  */
 219 int
 220 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 221 {
 222     String one,two;
 223     assert (otherRep);
 224     /* Numbers first - easiest to check */
 225     /* Content-Length */
 226     /* TODO: remove -1 bypass */
 227
 228     if (content_length != otherRep->content_length
 229             && content_length > -1 &&
 230             otherRep->content_length > -1)
 231         return 0;
 232
 233     /* ETag */
 234     one = header.getStrOrList(Http::HdrType::ETAG);
 235
 236     two = otherRep->header.getStrOrList(Http::HdrType::ETAG);
 237
 238     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 239         one.clean();
 240         two.clean();
 241         return 0;
 242     }
 243
 244     if (last_modified != otherRep->last_modified)
 245         return 0;
 246
 247     /* MD5 */
 248     one = header.getStrOrList(Http::HdrType::CONTENT_MD5);
 249
 250     two = otherRep->header.getStrOrList(Http::HdrType::CONTENT_MD5);
 251
 252     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 253         one.clean();
 254         two.clean();
 255         return 0;
 256     }
 257
 258     return 1;
 259 }
 260
 261 HttpReply::Pointer
 262 HttpReply::recreateOnNotModified(const HttpReply &reply304) const
 263 {
 264     // If enough 304s do not update, then this expensive checking is cheaper
 265     // than blindly storing reply prefix identical to the already stored one.
 266     if (!header.needUpdate(&reply304.header))
 267         return nullptr;
 268
 269     const Pointer cloned = clone();
 270     cloned->header.update(&reply304.header);
 271     cloned->hdrCacheClean();
 272     cloned->header.compact();
 273     cloned->hdrCacheInit();
 274     return cloned;
 275 }
 276
 277 /* internal routines */
 278
 279 time_t
 280 HttpReply::hdrExpirationTime()
 281 {
 282     /* The s-maxage and max-age directive takes priority over Expires */
 283
 284     if (cache_control) {
 285         int maxAge = -1;
 286         /*
 287          * Conservatively handle the case when we have a max-age
 288          * header, but no Date for reference?
 289          */
 290         if (cache_control->hasSMaxAge(&maxAge) || cache_control->hasMaxAge(&maxAge))
 291             return (date >= 0) ? date + maxAge : squid_curtime;
 292     }
 293
 294     if (Config.onoff.vary_ignore_expire &&
 295             header.has(Http::HdrType::VARY)) {
 296         const time_t d = header.getTime(Http::HdrType::DATE);
 297         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 298
 299         if (d == e)
 300             return -1;
 301     }
 302
 303     if (header.has(Http::HdrType::EXPIRES)) {
 304         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 305         /*
 306          * HTTP/1.0 says that robust implementations should consider
 307          * bad or malformed Expires header as equivalent to "expires
 308          * immediately."
 309          */
 310         return e < 0 ? squid_curtime : e;
 311     }
 312
 313     return -1;
 314 }
 315
 316 /* sync this routine when you update HttpReply struct */
 317 void
 318 HttpReply::hdrCacheInit()
 319 {
 320     Http::Message::hdrCacheInit();
 321
 322     http_ver = sline.version;
 323     content_length = header.getInt64(Http::HdrType::CONTENT_LENGTH);
 324     date = header.getTime(Http::HdrType::DATE);
 325     last_modified = header.getTime(Http::HdrType::LAST_MODIFIED);
 326     surrogate_control = header.getSc();
 327     content_range = (sline.status() == Http::scPartialContent) ?
 328                     header.getContRange() : nullptr;
 329     keep_alive = persistent() ? 1 : 0;
 330     const char *str = header.getStr(Http::HdrType::CONTENT_TYPE);
 331
 332     if (str)
 333         content_type.assign(str, strcspn(str, ";\t "));
 334     else
 335         content_type = String();
 336
 337     /* be sure to set expires after date and cache-control */
 338     expires = hdrExpirationTime();
 339 }
 340
 341 const HttpHdrContRange *
 342 HttpReply::contentRange() const
 343 {
 344     assert(!content_range || sline.status() == Http::scPartialContent);
 345     return content_range;
 346 }
 347
 348 /* sync this routine when you update HttpReply struct */
 349 void
 350 HttpReply::hdrCacheClean()
 351 {
 352     content_type.clean();
 353
 354     if (cache_control) {
 355         delete cache_control;
 356         cache_control = nullptr;
 357     }
 358
 359     if (surrogate_control) {
 360         delete surrogate_control;
 361         surrogate_control = nullptr;
 362     }
 363
 364     if (content_range) {
 365         delete content_range;
 366         content_range = nullptr;
 367     }
 368 }
 369
 370 /*
 371  * Returns the body size of a HTTP response
 372  */
 373 int64_t
 374 HttpReply::bodySize(const HttpRequestMethod& method) const
 375 {
 376     if (sline.version.major < 1)
 377         return -1;
 378     else if (method.id() == Http::METHOD_HEAD)
 379         return 0;
 380     else if (sline.status() == Http::scOkay)
 381         (void) 0;       /* common case, continue */
 382     else if (sline.status() == Http::scNoContent)
 383         return 0;
 384     else if (sline.status() == Http::scNotModified)
 385         return 0;
 386     else if (sline.status() < Http::scOkay)
 387         return 0;
 388
 389     return content_length;
 390 }
 391
 392 /**
 393  * Checks the first line of an HTTP Reply is valid.
 394  * currently only checks "HTTP/" exists.
 395  *
 396  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 397  */
 398 bool
 399 HttpReply::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 400 {
 401     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 402
 403     // content is long enough to possibly hold a reply
 404     // 4 being magic size of a 3-digit number plus space delimiter
 405     if (hdr_len < (size_t)(protoPrefix.psize() + 4)) {
 406         if (hdr_len > 0) {
 407             debugs(58, 3, "Too small reply header (" << hdr_len << " bytes)");
 408             *error = Http::scInvalidHeader;
 409         }
 410         return false;
 411     }
 412
 413     int pos;
 414     // catch missing or mismatched protocol identifier
 415     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 416     if (strncmp(buf, "ICY", 3) == 0) {
 417         protoPrefix = "ICY";
 418         pos = protoPrefix.psize();
 419     } else {
 420
 421         if (protoPrefix.cmp(buf, protoPrefix.size()) != 0) {
 422             debugs(58, 3, "missing protocol prefix (" << protoPrefix << ") in '" << buf << "'");
 423             *error = Http::scInvalidHeader;
 424             return false;
 425         }
 426
 427         // catch missing or negative status value (negative '-' is not a digit)
 428         pos = protoPrefix.psize();
 429
 430         // skip arbitrary number of digits and a dot in the version portion
 431         while ((size_t)pos <= hdr_len && (*(buf+pos) == '.' || xisdigit(*(buf+pos)) ) ) ++pos;
 432
 433         // catch missing version info
 434         if (pos == protoPrefix.psize()) {
 435             debugs(58, 3, "missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf << "'");
 436             *error = Http::scInvalidHeader;
 437             return false;
 438         }
 439     }
 440
 441     // skip arbitrary number of spaces...
 442     while ((size_t)pos <= hdr_len && (char)*(buf+pos) == ' ') ++pos;
 443
 444     if ((size_t)pos < hdr_len && !xisdigit(*(buf+pos))) {
 445         debugs(58, 3, "missing or invalid status number in '" << buf << "'");
 446         *error = Http::scInvalidHeader;
 447         return false;
 448     }
 449
 450     return true;
 451 }
 452
 453 bool
 454 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 455 {
 456     return sline.parse(protoPrefix, blk_start, blk_end);
 457 }
 458
 459 void
 460 HttpReply::configureContentLengthInterpreter(Http::ContentLengthInterpreter &interpreter)
 461 {
 462     interpreter.applyStatusCodeRules(sline.status());
 463 }
 464
 465 bool
 466 HttpReply::parseHeader(Http1::Parser &hp)
 467 {
 468     Http::ContentLengthInterpreter clen;
 469     return Message::parseHeader(hp, clen);
 470 }
 471
 472 /* handy: resets and returns -1 */
 473 int
 474 HttpReply::httpMsgParseError()
 475 {
 476     int result(Http::Message::httpMsgParseError());
 477     /* indicate an error in the status line */
 478     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 479     return result;
 480 }
 481
 482 /*
 483  * Indicate whether or not we would usually expect an entity-body
 484  * along with this response
 485  */
 486 bool
 487 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 488 {
 489     bool expectBody = true;
 490
 491     if (req_method == Http::METHOD_HEAD)
 492         expectBody = false;
 493     else if (sline.status() == Http::scNoContent)
 494         expectBody = false;
 495     else if (sline.status() == Http::scNotModified)
 496         expectBody = false;
 497     // TODO: Consider assuming that gray-area 0xx responses have bodies, like 9xx responses.
 498     else if (sline.status() < Http::scOkay)
 499         expectBody = false;
 500     else
 501         expectBody = true;
 502
 503     if (expectBody) {
 504         if (header.chunked())
 505             theSize = -1;
 506         else if (content_length >= 0)
 507             theSize = content_length;
 508         else
 509             theSize = -1;
 510     }
 511
 512     return expectBody;
 513 }
 514
 515 bool
 516 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 517 {
 518     calcMaxBodySize(request);
 519     debugs(58, 3, receivedSize << " >? " << bodySizeMax);
 520     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 521 }
 522
 523 bool
 524 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 525 {
 526     calcMaxBodySize(request);
 527     debugs(58, 7, "bodySizeMax=" << bodySizeMax);
 528
 529     if (bodySizeMax < 0) // no body size limit
 530         return false;
 531
 532     int64_t expectedSize = -1;
 533     if (!expectingBody(request.method, expectedSize))
 534         return false;
 535
 536     debugs(58, 6, expectedSize << " >? " << bodySizeMax);
 537
 538     if (expectedSize < 0) // expecting body of an unknown length
 539         return false;
 540
 541     return expectedSize > bodySizeMax;
 542 }
 543
 544 void
 545 HttpReply::calcMaxBodySize(HttpRequest& request) const
 546 {
 547     // hack: -2 is used as "we have not calculated max body size yet" state
 548     if (bodySizeMax != -2) // already tried
 549         return;
 550     bodySizeMax = -1;
 551
 552     // short-circuit ACL testing if there are none configured
 553     if (!Config.ReplyBodySize)
 554         return;
 555
 556     ACLFilledChecklist ch(nullptr, &request, nullptr);
 557     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 558     ch.reply = const_cast<HttpReply *>(this);
 559     HTTPMSGLOCK(ch.reply);
 560     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 561         /* if there is no ACL list or if the ACLs listed match use this size value */
 562         if (!l->aclList || ch.fastCheck(l->aclList).allowed()) {
 563             debugs(58, 4, "bodySizeMax=" << bodySizeMax);
 564             bodySizeMax = l->size; // may be -1
 565             break;
 566         }
 567     }
 568 }
 569
 570 // XXX: check that this is sufficient for eCAP cloning
 571 HttpReply *
 572 HttpReply::clone() const
 573 {
 574     HttpReply *rep = new HttpReply();
 575     rep->sline = sline; // used in hdrCacheInit() call below
 576     rep->header.append(&header);
 577     rep->hdrCacheInit();
 578     rep->hdr_sz = hdr_sz;
 579     rep->http_ver = http_ver;
 580     rep->pstate = pstate;
 581     rep->body_pipe = body_pipe;
 582
 583     // keep_alive is handled in hdrCacheInit()
 584     return rep;
 585 }
 586
 587 bool
 588 HttpReply::inheritProperties(const Http::Message *aMsg)
 589 {
 590     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 591     if (!aRep)
 592         return false;
 593     keep_alive = aRep->keep_alive;
 594     sources = aRep->sources;
 595     return true;
 596 }
 597
 598 void HttpReply::removeStaleWarnings()
 599 {
 600     String warning;
 601     if (header.getList(Http::HdrType::WARNING, &warning)) {
 602         const String newWarning = removeStaleWarningValues(warning);
 603         if (warning.size() && warning.size() == newWarning.size())
 604             return; // some warnings are there and none changed
 605         header.delById(Http::HdrType::WARNING);
 606         if (newWarning.size()) { // some warnings left
 607             HttpHeaderEntry *const e =
 608                 new HttpHeaderEntry(Http::HdrType::WARNING, SBuf(), newWarning.termedBuf());
 609             header.addEntry(e);
 610         }
 611     }
 612 }
 613
 614 /**
 615  * Remove warning-values with warn-date different from Date value from
 616  * a single header entry. Returns a string with all valid warning-values.
 617  */
 618 String HttpReply::removeStaleWarningValues(const String &value)
 619 {
 620     String newValue;
 621     const char *item = nullptr;
 622     int len = 0;
 623     const char *pos = nullptr;
 624     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 625         bool keep = true;
 626         // Does warning-value have warn-date (which contains quoted date)?
 627         // We scan backwards, looking for two quoted strings.
 628         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 629         const char *p = item + len - 1;
 630
 631         while (p >= item && xisspace(*p)) --p; // skip whitespace
 632
 633         // warning-value MUST end with quote
 634         if (p >= item && *p == '"') {
 635             const char *const warnDateEnd = p;
 636             --p;
 637             while (p >= item && *p != '"') --p; // find the next quote
 638
 639             const char *warnDateBeg = p + 1;
 640             --p;
 641             while (p >= item && xisspace(*p)) --p; // skip whitespace
 642
 643             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 644                 // found warn-text
 645                 String warnDate;
 646                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 647                 const time_t time = Time::ParseRfc1123(warnDate.termedBuf());
 648                 keep = (time > 0 && time == date); // keep valid and matching date
 649             }
 650         }
 651
 652         if (keep) {
 653             if (newValue.size())
 654                 newValue.append(", ");
 655             newValue.append(item, len);
 656         }
 657     }
 658
 659     return newValue;
 660 }
 661
 662 bool
 663 HttpReply::olderThan(const HttpReply *them) const
 664 {
 665     if (!them || !them->date || !date)
 666         return false;
 667     return date < them->date;
 668 }
 669
 670 void
 671 HttpReply::removeIrrelevantContentLength() {
 672     if (Http::ProhibitsContentLength(sline.status()))
 673         if (header.delById(Http::HdrType::CONTENT_LENGTH))
 674             debugs(58, 3, "Removing unexpected Content-Length header");
 675 }
 676