src/HttpReply.cc

   1 /*
   2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 58    HTTP Reply (Response) */
  10
  11 #include "squid.h"
  12 #include "acl/AclSizeLimit.h"
  13 #include "acl/FilledChecklist.h"
  14 #include "base/EnumIterator.h"
  15 #include "globals.h"
  16 #include "HttpBody.h"
  17 #include "HttpHdrCc.h"
  18 #include "HttpHdrContRange.h"
  19 #include "HttpHdrSc.h"
  20 #include "HttpReply.h"
  21 #include "HttpRequest.h"
  22 #include "MemBuf.h"
  23 #include "SquidConfig.h"
  24 #include "SquidTime.h"
  25 #include "Store.h"
  26 #include "StrList.h"
  27
  28 HttpReply::HttpReply():
  29     Http::Message(hoReply),
  30     date(0),
  31     last_modified(0),
  32     expires(0),
  33     surrogate_control(nullptr),
  34     keep_alive(0),
  35     protoPrefix("HTTP/"),
  36     bodySizeMax(-2),
  37     content_range(nullptr)
  38 {
  39     init();
  40 }
  41
  42 HttpReply::~HttpReply()
  43 {
  44     if (do_clean)
  45         clean();
  46 }
  47
  48 void
  49 HttpReply::init()
  50 {
  51     hdrCacheInit();
  52     sline.init();
  53     pstate = Http::Message::psReadyToParseStartLine;
  54     do_clean = true;
  55 }
  56
  57 void HttpReply::reset()
  58 {
  59
  60     // reset should not reset the protocol; could have made protoPrefix a
  61     // virtual function instead, but it is not clear whether virtual methods
  62     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
  63     // conversions are not going to kill virtual tables
  64     const String pfx = protoPrefix;
  65     clean();
  66     init();
  67     protoPrefix = pfx;
  68 }
  69
  70 void
  71 HttpReply::clean()
  72 {
  73     // we used to assert that the pipe is NULL, but now the message only
  74     // points to a pipe that is owned and initiated by another object.
  75     body_pipe = NULL;
  76
  77     body.clear();
  78     hdrCacheClean();
  79     header.clean();
  80     sline.clean();
  81     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
  82 }
  83
  84 void
  85 HttpReply::packHeadersInto(Packable * p) const
  86 {
  87     sline.packInto(p);
  88     header.packInto(p);
  89     p->append("\r\n", 2);
  90 }
  91
  92 void
  93 HttpReply::packInto(Packable * p) const
  94 {
  95     packHeadersInto(p);
  96     body.packInto(p);
  97 }
  98
  99 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 100 MemBuf *
 101 HttpReply::pack() const
 102 {
 103     MemBuf *mb = new MemBuf;
 104     mb->init();
 105     packInto(mb);
 106     return mb;
 107 }
 108
 109 HttpReply *
 110 HttpReply::make304() const
 111 {
 112     static const Http::HdrType ImsEntries[] = {Http::HdrType::DATE, Http::HdrType::CONTENT_TYPE, Http::HdrType::EXPIRES, Http::HdrType::LAST_MODIFIED, /* eof */ Http::HdrType::OTHER};
 113
 114     HttpReply *rv = new HttpReply;
 115     int t;
 116     HttpHeaderEntry *e;
 117
 118     /* rv->content_length; */
 119     rv->date = date;
 120     rv->last_modified = last_modified;
 121     rv->expires = expires;
 122     rv->content_type = content_type;
 123     /* rv->content_range */
 124     /* rv->keep_alive */
 125     rv->sline.set(Http::ProtocolVersion(), Http::scNotModified, NULL);
 126
 127     for (t = 0; ImsEntries[t] != Http::HdrType::OTHER; ++t) {
 128         if ((e = header.findEntry(ImsEntries[t])))
 129             rv->header.addEntry(e->clone());
 130     }
 131
 132     rv->putCc(cache_control);
 133
 134     /* rv->body */
 135     return rv;
 136 }
 137
 138 MemBuf *
 139 HttpReply::packed304Reply() const
 140 {
 141     /* Not as efficient as skipping the header duplication,
 142      * but easier to maintain
 143      */
 144     HttpReply *temp = make304();
 145     MemBuf *rv = temp->pack();
 146     delete temp;
 147     return rv;
 148 }
 149
 150 void
 151 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 152                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 153 {
 154     HttpHeader *hdr;
 155     sline.set(Http::ProtocolVersion(), status, reason);
 156     hdr = &header;
 157     hdr->putStr(Http::HdrType::SERVER, visible_appname_string);
 158     hdr->putStr(Http::HdrType::MIME_VERSION, "1.0");
 159     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 160
 161     if (ctype) {
 162         hdr->putStr(Http::HdrType::CONTENT_TYPE, ctype);
 163         content_type = ctype;
 164     } else
 165         content_type = String();
 166
 167     if (clen >= 0)
 168         hdr->putInt64(Http::HdrType::CONTENT_LENGTH, clen);
 169
 170     if (expiresTime >= 0)
 171         hdr->putTime(Http::HdrType::EXPIRES, expiresTime);
 172
 173     if (lmt > 0)        /* this used to be lmt != 0 @?@ */
 174         hdr->putTime(Http::HdrType::LAST_MODIFIED, lmt);
 175
 176     date = squid_curtime;
 177
 178     content_length = clen;
 179
 180     expires = expiresTime;
 181
 182     last_modified = lmt;
 183 }
 184
 185 void
 186 HttpReply::redirect(Http::StatusCode status, const char *loc)
 187 {
 188     HttpHeader *hdr;
 189     sline.set(Http::ProtocolVersion(), status, NULL);
 190     hdr = &header;
 191     hdr->putStr(Http::HdrType::SERVER, APP_FULLNAME);
 192     hdr->putTime(Http::HdrType::DATE, squid_curtime);
 193     hdr->putInt64(Http::HdrType::CONTENT_LENGTH, 0);
 194     hdr->putStr(Http::HdrType::LOCATION, loc);
 195     date = squid_curtime;
 196     content_length = 0;
 197 }
 198
 199 /* compare the validators of two replies.
 200  * 1 = they match
 201  * 0 = they do not match
 202  */
 203 int
 204 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 205 {
 206     String one,two;
 207     assert (otherRep);
 208     /* Numbers first - easiest to check */
 209     /* Content-Length */
 210     /* TODO: remove -1 bypass */
 211
 212     if (content_length != otherRep->content_length
 213             && content_length > -1 &&
 214             otherRep->content_length > -1)
 215         return 0;
 216
 217     /* ETag */
 218     one = header.getStrOrList(Http::HdrType::ETAG);
 219
 220     two = otherRep->header.getStrOrList(Http::HdrType::ETAG);
 221
 222     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 223         one.clean();
 224         two.clean();
 225         return 0;
 226     }
 227
 228     if (last_modified != otherRep->last_modified)
 229         return 0;
 230
 231     /* MD5 */
 232     one = header.getStrOrList(Http::HdrType::CONTENT_MD5);
 233
 234     two = otherRep->header.getStrOrList(Http::HdrType::CONTENT_MD5);
 235
 236     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 237         one.clean();
 238         two.clean();
 239         return 0;
 240     }
 241
 242     return 1;
 243 }
 244
 245 bool
 246 HttpReply::updateOnNotModified(HttpReply const * freshRep)
 247 {
 248     assert(freshRep);
 249
 250     /* update raw headers */
 251     if (!header.update(&freshRep->header))
 252         return false;
 253
 254     /* clean cache */
 255     hdrCacheClean();
 256
 257     header.compact();
 258     /* init cache */
 259     hdrCacheInit();
 260
 261     return true;
 262 }
 263
 264 /* internal routines */
 265
 266 time_t
 267 HttpReply::hdrExpirationTime()
 268 {
 269     /* The s-maxage and max-age directive takes priority over Expires */
 270
 271     if (cache_control) {
 272         int maxAge = -1;
 273         /*
 274          * Conservatively handle the case when we have a max-age
 275          * header, but no Date for reference?
 276          */
 277         if (cache_control->hasSMaxAge(&maxAge) || cache_control->hasMaxAge(&maxAge))
 278             return (date >= 0) ? date + maxAge : squid_curtime;
 279     }
 280
 281     if (Config.onoff.vary_ignore_expire &&
 282             header.has(Http::HdrType::VARY)) {
 283         const time_t d = header.getTime(Http::HdrType::DATE);
 284         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 285
 286         if (d == e)
 287             return -1;
 288     }
 289
 290     if (header.has(Http::HdrType::EXPIRES)) {
 291         const time_t e = header.getTime(Http::HdrType::EXPIRES);
 292         /*
 293          * HTTP/1.0 says that robust implementations should consider
 294          * bad or malformed Expires header as equivalent to "expires
 295          * immediately."
 296          */
 297         return e < 0 ? squid_curtime : e;
 298     }
 299
 300     return -1;
 301 }
 302
 303 /* sync this routine when you update HttpReply struct */
 304 void
 305 HttpReply::hdrCacheInit()
 306 {
 307     Http::Message::hdrCacheInit();
 308
 309     http_ver = sline.version;
 310     content_length = header.getInt64(Http::HdrType::CONTENT_LENGTH);
 311     date = header.getTime(Http::HdrType::DATE);
 312     last_modified = header.getTime(Http::HdrType::LAST_MODIFIED);
 313     surrogate_control = header.getSc();
 314     content_range = (sline.status() == Http::scPartialContent) ?
 315                     header.getContRange() : nullptr;
 316     keep_alive = persistent() ? 1 : 0;
 317     const char *str = header.getStr(Http::HdrType::CONTENT_TYPE);
 318
 319     if (str)
 320         content_type.limitInit(str, strcspn(str, ";\t "));
 321     else
 322         content_type = String();
 323
 324     /* be sure to set expires after date and cache-control */
 325     expires = hdrExpirationTime();
 326 }
 327
 328 const HttpHdrContRange *
 329 HttpReply::contentRange() const
 330 {
 331     assert(!content_range || sline.status() == Http::scPartialContent);
 332     return content_range;
 333 }
 334
 335 /* sync this routine when you update HttpReply struct */
 336 void
 337 HttpReply::hdrCacheClean()
 338 {
 339     content_type.clean();
 340
 341     if (cache_control) {
 342         delete cache_control;
 343         cache_control = NULL;
 344     }
 345
 346     if (surrogate_control) {
 347         delete surrogate_control;
 348         surrogate_control = NULL;
 349     }
 350
 351     if (content_range) {
 352         delete content_range;
 353         content_range = NULL;
 354     }
 355 }
 356
 357 /*
 358  * Returns the body size of a HTTP response
 359  */
 360 int64_t
 361 HttpReply::bodySize(const HttpRequestMethod& method) const
 362 {
 363     if (sline.version.major < 1)
 364         return -1;
 365     else if (method.id() == Http::METHOD_HEAD)
 366         return 0;
 367     else if (sline.status() == Http::scOkay)
 368         (void) 0;       /* common case, continue */
 369     else if (sline.status() == Http::scNoContent)
 370         return 0;
 371     else if (sline.status() == Http::scNotModified)
 372         return 0;
 373     else if (sline.status() < Http::scOkay)
 374         return 0;
 375
 376     return content_length;
 377 }
 378
 379 /**
 380  * Checks the first line of an HTTP Reply is valid.
 381  * currently only checks "HTTP/" exists.
 382  *
 383  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 384  */
 385 bool
 386 HttpReply::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 387 {
 388     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 389
 390     // content is long enough to possibly hold a reply
 391     // 4 being magic size of a 3-digit number plus space delimiter
 392     if (hdr_len < (size_t)(protoPrefix.psize() + 4)) {
 393         if (hdr_len > 0) {
 394             debugs(58, 3, "Too small reply header (" << hdr_len << " bytes)");
 395             *error = Http::scInvalidHeader;
 396         }
 397         return false;
 398     }
 399
 400     int pos;
 401     // catch missing or mismatched protocol identifier
 402     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 403     if (strncmp(buf, "ICY", 3) == 0) {
 404         protoPrefix = "ICY";
 405         pos = protoPrefix.psize();
 406     } else {
 407
 408         if (protoPrefix.cmp(buf, protoPrefix.size()) != 0) {
 409             debugs(58, 3, "missing protocol prefix (" << protoPrefix << ") in '" << buf << "'");
 410             *error = Http::scInvalidHeader;
 411             return false;
 412         }
 413
 414         // catch missing or negative status value (negative '-' is not a digit)
 415         pos = protoPrefix.psize();
 416
 417         // skip arbitrary number of digits and a dot in the verion portion
 418         while ((size_t)pos <= hdr_len && (*(buf+pos) == '.' || xisdigit(*(buf+pos)) ) ) ++pos;
 419
 420         // catch missing version info
 421         if (pos == protoPrefix.psize()) {
 422             debugs(58, 3, "missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf << "'");
 423             *error = Http::scInvalidHeader;
 424             return false;
 425         }
 426     }
 427
 428     // skip arbitrary number of spaces...
 429     while ((size_t)pos <= hdr_len && (char)*(buf+pos) == ' ') ++pos;
 430
 431     if ((size_t)pos < hdr_len && !xisdigit(*(buf+pos))) {
 432         debugs(58, 3, "missing or invalid status number in '" << buf << "'");
 433         *error = Http::scInvalidHeader;
 434         return false;
 435     }
 436
 437     return true;
 438 }
 439
 440 bool
 441 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 442 {
 443     return sline.parse(protoPrefix, blk_start, blk_end);
 444 }
 445
 446 /* handy: resets and returns -1 */
 447 int
 448 HttpReply::httpMsgParseError()
 449 {
 450     int result(Http::Message::httpMsgParseError());
 451     /* indicate an error in the status line */
 452     sline.set(Http::ProtocolVersion(), Http::scInvalidHeader);
 453     return result;
 454 }
 455
 456 /*
 457  * Indicate whether or not we would usually expect an entity-body
 458  * along with this response
 459  */
 460 bool
 461 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 462 {
 463     bool expectBody = true;
 464
 465     if (req_method == Http::METHOD_HEAD)
 466         expectBody = false;
 467     else if (sline.status() == Http::scNoContent)
 468         expectBody = false;
 469     else if (sline.status() == Http::scNotModified)
 470         expectBody = false;
 471     else if (sline.status() < Http::scOkay)
 472         expectBody = false;
 473     else
 474         expectBody = true;
 475
 476     if (expectBody) {
 477         if (header.chunked())
 478             theSize = -1;
 479         else if (content_length >= 0)
 480             theSize = content_length;
 481         else
 482             theSize = -1;
 483     }
 484
 485     return expectBody;
 486 }
 487
 488 bool
 489 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 490 {
 491     calcMaxBodySize(request);
 492     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 493     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 494 }
 495
 496 bool
 497 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 498 {
 499     calcMaxBodySize(request);
 500     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 501
 502     if (bodySizeMax < 0) // no body size limit
 503         return false;
 504
 505     int64_t expectedSize = -1;
 506     if (!expectingBody(request.method, expectedSize))
 507         return false;
 508
 509     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 510
 511     if (expectedSize < 0) // expecting body of an unknown length
 512         return false;
 513
 514     return expectedSize > bodySizeMax;
 515 }
 516
 517 void
 518 HttpReply::calcMaxBodySize(HttpRequest& request) const
 519 {
 520     // hack: -2 is used as "we have not calculated max body size yet" state
 521     if (bodySizeMax != -2) // already tried
 522         return;
 523     bodySizeMax = -1;
 524
 525     // short-circuit ACL testing if there are none configured
 526     if (!Config.ReplyBodySize)
 527         return;
 528
 529     ACLFilledChecklist ch(NULL, &request, NULL);
 530     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 531     ch.reply = const_cast<HttpReply *>(this);
 532     HTTPMSGLOCK(ch.reply);
 533     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 534         /* if there is no ACL list or if the ACLs listed match use this size value */
 535         if (!l->aclList || ch.fastCheck(l->aclList).allowed()) {
 536             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 537             bodySizeMax = l->size; // may be -1
 538             break;
 539         }
 540     }
 541 }
 542
 543 // XXX: check that this is sufficient for eCAP cloning
 544 HttpReply *
 545 HttpReply::clone() const
 546 {
 547     HttpReply *rep = new HttpReply();
 548     rep->sline = sline; // used in hdrCacheInit() call below
 549     rep->header.append(&header);
 550     rep->hdrCacheInit();
 551     rep->hdr_sz = hdr_sz;
 552     rep->http_ver = http_ver;
 553     rep->pstate = pstate;
 554     rep->body_pipe = body_pipe;
 555
 556     // keep_alive is handled in hdrCacheInit()
 557     return rep;
 558 }
 559
 560 bool
 561 HttpReply::inheritProperties(const Http::Message *aMsg)
 562 {
 563     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 564     if (!aRep)
 565         return false;
 566     keep_alive = aRep->keep_alive;
 567     sources = aRep->sources;
 568     return true;
 569 }
 570
 571 void HttpReply::removeStaleWarnings()
 572 {
 573     String warning;
 574     if (header.getList(Http::HdrType::WARNING, &warning)) {
 575         const String newWarning = removeStaleWarningValues(warning);
 576         if (warning.size() && warning.size() == newWarning.size())
 577             return; // some warnings are there and none changed
 578         header.delById(Http::HdrType::WARNING);
 579         if (newWarning.size()) { // some warnings left
 580             HttpHeaderEntry *const e =
 581                 new HttpHeaderEntry(Http::HdrType::WARNING, SBuf(), newWarning.termedBuf());
 582             header.addEntry(e);
 583         }
 584     }
 585 }
 586
 587 /**
 588  * Remove warning-values with warn-date different from Date value from
 589  * a single header entry. Returns a string with all valid warning-values.
 590  */
 591 String HttpReply::removeStaleWarningValues(const String &value)
 592 {
 593     String newValue;
 594     const char *item = 0;
 595     int len = 0;
 596     const char *pos = 0;
 597     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 598         bool keep = true;
 599         // Does warning-value have warn-date (which contains quoted date)?
 600         // We scan backwards, looking for two quoted strings.
 601         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 602         const char *p = item + len - 1;
 603
 604         while (p >= item && xisspace(*p)) --p; // skip whitespace
 605
 606         // warning-value MUST end with quote
 607         if (p >= item && *p == '"') {
 608             const char *const warnDateEnd = p;
 609             --p;
 610             while (p >= item && *p != '"') --p; // find the next quote
 611
 612             const char *warnDateBeg = p + 1;
 613             --p;
 614             while (p >= item && xisspace(*p)) --p; // skip whitespace
 615
 616             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 617                 // found warn-text
 618                 String warnDate;
 619                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 620                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 621                 keep = (time > 0 && time == date); // keep valid and matching date
 622             }
 623         }
 624
 625         if (keep) {
 626             if (newValue.size())
 627                 newValue.append(", ");
 628             newValue.append(item, len);
 629         }
 630     }
 631
 632     return newValue;
 633 }
 634
 635 bool
 636 HttpReply::olderThan(const HttpReply *them) const
 637 {
 638     if (!them || !them->date || !date)
 639         return false;
 640     return date < them->date;
 641 }
 642