src/HttpReply.cc

   1
   2 /*
   3  * DEBUG: section 58    HTTP Reply (Response)
   4  * AUTHOR: Alex Rousskov
   5  *
   6  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   7  * ----------------------------------------------------------
   8  *
   9  *  Squid is the result of efforts by numerous individuals from
  10  *  the Internet community; see the CONTRIBUTORS file for full
  11  *  details.   Many organizations have provided support for Squid's
  12  *  development; see the SPONSORS file for full details.  Squid is
  13  *  Copyrighted (C) 2001 by the Regents of the University of
  14  *  California; see the COPYRIGHT file for full details.  Squid
  15  *  incorporates software developed and/or copyrighted by other
  16  *  sources; see the CREDITS file for full details.
  17  *
  18  *  This program is free software; you can redistribute it and/or modify
  19  *  it under the terms of the GNU General Public License as published by
  20  *  the Free Software Foundation; either version 2 of the License, or
  21  *  (at your option) any later version.
  22  *
  23  *  This program is distributed in the hope that it will be useful,
  24  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  25  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  26  *  GNU General Public License for more details.
  27  *
  28  *  You should have received a copy of the GNU General Public License
  29  *  along with this program; if not, write to the Free Software
  30  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  31  *
  32  */
  33
  34 #include "squid.h"
  35 #include "acl/AclSizeLimit.h"
  36 #include "acl/FilledChecklist.h"
  37 #include "globals.h"
  38 #include "HttpBody.h"
  39 #include "HttpHdrCc.h"
  40 #include "HttpHdrContRange.h"
  41 #include "HttpHdrSc.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequest.h"
  44 #include "MemBuf.h"
  45 #include "SquidConfig.h"
  46 #include "SquidTime.h"
  47 #include "Store.h"
  48 #include "StrList.h"
  49
  50 /* local constants */
  51
  52 /* If we receive a 304 from the origin during a cache revalidation, we must
  53  * update the headers of the existing entry. Specifically, we need to update all
  54  * end-to-end headers and not any hop-by-hop headers (rfc2616 13.5.3).
  55  *
  56  * This is not the whole story though: since it is possible for a faulty/malicious
  57  * origin server to set headers it should not in a 304, we must explicitly ignore
  58  * these too. Specifically all entity-headers except those permitted in a 304
  59  * (rfc2616 10.3.5) must be ignored.
  60  *
  61  * The list of headers we don't update is made up of:
  62  *     all hop-by-hop headers
  63  *     all entity-headers except Expires and Content-Location
  64  */
  65 static HttpHeaderMask Denied304HeadersMask;
  66 static http_hdr_type Denied304HeadersArr[] = {
  67     // hop-by-hop headers
  68     HDR_CONNECTION, HDR_KEEP_ALIVE, HDR_PROXY_AUTHENTICATE, HDR_PROXY_AUTHORIZATION,
  69     HDR_TE, HDR_TRAILER, HDR_TRANSFER_ENCODING, HDR_UPGRADE,
  70     // entity headers
  71     HDR_ALLOW, HDR_CONTENT_ENCODING, HDR_CONTENT_LANGUAGE, HDR_CONTENT_LENGTH,
  72     HDR_CONTENT_MD5, HDR_CONTENT_RANGE, HDR_CONTENT_TYPE, HDR_LAST_MODIFIED
  73 };
  74
  75 /* module initialization */
  76 void
  77 httpReplyInitModule(void)
  78 {
  79     assert(Http::scNone == 0); // HttpReply::parse() interface assumes that
  80     httpHeaderMaskInit(&Denied304HeadersMask, 0);
  81     httpHeaderCalcMask(&Denied304HeadersMask, Denied304HeadersArr, countof(Denied304HeadersArr));
  82 }
  83
  84 HttpReply::HttpReply() : HttpMsg(hoReply), date (0), last_modified (0),
  85         expires (0), surrogate_control (NULL), content_range (NULL), keep_alive (0),
  86         protoPrefix("HTTP/"), bodySizeMax(-2)
  87 {
  88     init();
  89 }
  90
  91 HttpReply::~HttpReply()
  92 {
  93     if (do_clean)
  94         clean();
  95 }
  96
  97 void
  98 HttpReply::init()
  99 {
 100     hdrCacheInit();
 101     sline.init();
 102     pstate = psReadyToParseStartLine;
 103     do_clean = true;
 104 }
 105
 106 void HttpReply::reset()
 107 {
 108
 109     // reset should not reset the protocol; could have made protoPrefix a
 110     // virtual function instead, but it is not clear whether virtual methods
 111     // are allowed with MEMPROXY_CLASS() and whether some cbdata void*
 112     // conversions are not going to kill virtual tables
 113     const String pfx = protoPrefix;
 114     clean();
 115     init();
 116     protoPrefix = pfx;
 117 }
 118
 119 void
 120 HttpReply::clean()
 121 {
 122     // we used to assert that the pipe is NULL, but now the message only
 123     // points to a pipe that is owned and initiated by another object.
 124     body_pipe = NULL;
 125
 126     body.clear();
 127     hdrCacheClean();
 128     header.clean();
 129     sline.clean();
 130     bodySizeMax = -2; // hack: make calculatedBodySizeMax() false
 131 }
 132
 133 void
 134 HttpReply::packHeadersInto(Packer * p) const
 135 {
 136     sline.packInto(p);
 137     header.packInto(p);
 138     packerAppend(p, "\r\n", 2);
 139 }
 140
 141 void
 142 HttpReply::packInto(Packer * p)
 143 {
 144     packHeadersInto(p);
 145     body.packInto(p);
 146 }
 147
 148 /* create memBuf, create mem-based packer, pack, destroy packer, return MemBuf */
 149 MemBuf *
 150 HttpReply::pack()
 151 {
 152     MemBuf *mb = new MemBuf;
 153     Packer p;
 154
 155     mb->init();
 156     packerToMemInit(&p, mb);
 157     packInto(&p);
 158     packerClean(&p);
 159     return mb;
 160 }
 161
 162 HttpReply *
 163 HttpReply::make304() const
 164 {
 165     static const http_hdr_type ImsEntries[] = {HDR_DATE, HDR_CONTENT_TYPE, HDR_EXPIRES, HDR_LAST_MODIFIED, /* eof */ HDR_OTHER};
 166
 167     HttpReply *rv = new HttpReply;
 168     int t;
 169     HttpHeaderEntry *e;
 170
 171     /* rv->content_length; */
 172     rv->date = date;
 173     rv->last_modified = last_modified;
 174     rv->expires = expires;
 175     rv->content_type = content_type;
 176     /* rv->cache_control */
 177     /* rv->content_range */
 178     /* rv->keep_alive */
 179     rv->sline.set(Http::ProtocolVersion(1,1), Http::scNotModified, NULL);
 180
 181     for (t = 0; ImsEntries[t] != HDR_OTHER; ++t)
 182         if ((e = header.findEntry(ImsEntries[t])))
 183             rv->header.addEntry(e->clone());
 184
 185     /* rv->body */
 186     return rv;
 187 }
 188
 189 MemBuf *
 190 HttpReply::packed304Reply()
 191 {
 192     /* Not as efficient as skipping the header duplication,
 193      * but easier to maintain
 194      */
 195     HttpReply *temp = make304();
 196     MemBuf *rv = temp->pack();
 197     delete temp;
 198     return rv;
 199 }
 200
 201 void
 202 HttpReply::setHeaders(Http::StatusCode status, const char *reason,
 203                       const char *ctype, int64_t clen, time_t lmt, time_t expiresTime)
 204 {
 205     HttpHeader *hdr;
 206     sline.set(Http::ProtocolVersion(1,1), status, reason);
 207     hdr = &header;
 208     hdr->putStr(HDR_SERVER, visible_appname_string);
 209     hdr->putStr(HDR_MIME_VERSION, "1.0");
 210     hdr->putTime(HDR_DATE, squid_curtime);
 211
 212     if (ctype) {
 213         hdr->putStr(HDR_CONTENT_TYPE, ctype);
 214         content_type = ctype;
 215     } else
 216         content_type = String();
 217
 218     if (clen >= 0)
 219         hdr->putInt64(HDR_CONTENT_LENGTH, clen);
 220
 221     if (expiresTime >= 0)
 222         hdr->putTime(HDR_EXPIRES, expiresTime);
 223
 224     if (lmt > 0)                /* this used to be lmt != 0 @?@ */
 225         hdr->putTime(HDR_LAST_MODIFIED, lmt);
 226
 227     date = squid_curtime;
 228
 229     content_length = clen;
 230
 231     expires = expiresTime;
 232
 233     last_modified = lmt;
 234 }
 235
 236 void
 237 HttpReply::redirect(Http::StatusCode status, const char *loc)
 238 {
 239     HttpHeader *hdr;
 240     sline.set(Http::ProtocolVersion(1,1), status, NULL);
 241     hdr = &header;
 242     hdr->putStr(HDR_SERVER, APP_FULLNAME);
 243     hdr->putTime(HDR_DATE, squid_curtime);
 244     hdr->putInt64(HDR_CONTENT_LENGTH, 0);
 245     hdr->putStr(HDR_LOCATION, loc);
 246     date = squid_curtime;
 247     content_length = 0;
 248 }
 249
 250 /* compare the validators of two replies.
 251  * 1 = they match
 252  * 0 = they do not match
 253  */
 254 int
 255 HttpReply::validatorsMatch(HttpReply const * otherRep) const
 256 {
 257     String one,two;
 258     assert (otherRep);
 259     /* Numbers first - easiest to check */
 260     /* Content-Length */
 261     /* TODO: remove -1 bypass */
 262
 263     if (content_length != otherRep->content_length
 264             && content_length > -1 &&
 265             otherRep->content_length > -1)
 266         return 0;
 267
 268     /* ETag */
 269     one = header.getStrOrList(HDR_ETAG);
 270
 271     two = otherRep->header.getStrOrList(HDR_ETAG);
 272
 273     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 274         one.clean();
 275         two.clean();
 276         return 0;
 277     }
 278
 279     if (last_modified != otherRep->last_modified)
 280         return 0;
 281
 282     /* MD5 */
 283     one = header.getStrOrList(HDR_CONTENT_MD5);
 284
 285     two = otherRep->header.getStrOrList(HDR_CONTENT_MD5);
 286
 287     if (one.size()==0 || two.size()==0 || one.caseCmp(two)!=0 ) {
 288         one.clean();
 289         two.clean();
 290         return 0;
 291     }
 292
 293     return 1;
 294 }
 295
 296 void
 297 HttpReply::updateOnNotModified(HttpReply const * freshRep)
 298 {
 299     assert(freshRep);
 300
 301     /* clean cache */
 302     hdrCacheClean();
 303     /* update raw headers */
 304     header.update(&freshRep->header,
 305                   (const HttpHeaderMask *) &Denied304HeadersMask);
 306
 307     header.compact();
 308     /* init cache */
 309     hdrCacheInit();
 310 }
 311
 312 /* internal routines */
 313
 314 time_t
 315 HttpReply::hdrExpirationTime()
 316 {
 317     /* The s-maxage and max-age directive takes priority over Expires */
 318
 319     if (cache_control) {
 320         if (date >= 0) {
 321             if (cache_control->hasSMaxAge())
 322                 return date + cache_control->sMaxAge();
 323
 324             if (cache_control->hasMaxAge())
 325                 return date + cache_control->maxAge();
 326         } else {
 327             /*
 328              * Conservatively handle the case when we have a max-age
 329              * header, but no Date for reference?
 330              */
 331
 332             if (cache_control->hasSMaxAge())
 333                 return squid_curtime;
 334
 335             if (cache_control->hasMaxAge())
 336                 return squid_curtime;
 337         }
 338     }
 339
 340     if (Config.onoff.vary_ignore_expire &&
 341             header.has(HDR_VARY)) {
 342         const time_t d = header.getTime(HDR_DATE);
 343         const time_t e = header.getTime(HDR_EXPIRES);
 344
 345         if (d == e)
 346             return -1;
 347     }
 348
 349     if (header.has(HDR_EXPIRES)) {
 350         const time_t e = header.getTime(HDR_EXPIRES);
 351         /*
 352          * HTTP/1.0 says that robust implementations should consider
 353          * bad or malformed Expires header as equivalent to "expires
 354          * immediately."
 355          */
 356         return e < 0 ? squid_curtime : e;
 357     }
 358
 359     return -1;
 360 }
 361
 362 /* sync this routine when you update HttpReply struct */
 363 void
 364 HttpReply::hdrCacheInit()
 365 {
 366     HttpMsg::hdrCacheInit();
 367
 368     http_ver = sline.version;
 369     content_length = header.getInt64(HDR_CONTENT_LENGTH);
 370     date = header.getTime(HDR_DATE);
 371     last_modified = header.getTime(HDR_LAST_MODIFIED);
 372     surrogate_control = header.getSc();
 373     content_range = header.getContRange();
 374     keep_alive = persistent() ? 1 : 0;
 375     const char *str = header.getStr(HDR_CONTENT_TYPE);
 376
 377     if (str)
 378         content_type.limitInit(str, strcspn(str, ";\t "));
 379     else
 380         content_type = String();
 381
 382     /* be sure to set expires after date and cache-control */
 383     expires = hdrExpirationTime();
 384 }
 385
 386 /* sync this routine when you update HttpReply struct */
 387 void
 388 HttpReply::hdrCacheClean()
 389 {
 390     content_type.clean();
 391
 392     if (cache_control) {
 393         delete cache_control;
 394         cache_control = NULL;
 395     }
 396
 397     if (surrogate_control) {
 398         delete surrogate_control;
 399         surrogate_control = NULL;
 400     }
 401
 402     if (content_range) {
 403         httpHdrContRangeDestroy(content_range);
 404         content_range = NULL;
 405     }
 406 }
 407
 408 /*
 409  * Returns the body size of a HTTP response
 410  */
 411 int64_t
 412 HttpReply::bodySize(const HttpRequestMethod& method) const
 413 {
 414     if (sline.version.major < 1)
 415         return -1;
 416     else if (method.id() == Http::METHOD_HEAD)
 417         return 0;
 418     else if (sline.status() == Http::scOkay)
 419         (void) 0;               /* common case, continue */
 420     else if (sline.status() == Http::scNoContent)
 421         return 0;
 422     else if (sline.status() == Http::scNotModified)
 423         return 0;
 424     else if (sline.status() < Http::scOkay)
 425         return 0;
 426
 427     return content_length;
 428 }
 429
 430 /**
 431  * Checks the first line of an HTTP Reply is valid.
 432  * currently only checks "HTTP/" exists.
 433  *
 434  * NP: not all error cases are detected yet. Some are left for detection later in parse.
 435  */
 436 bool
 437 HttpReply::sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, Http::StatusCode *error)
 438 {
 439     // hack warning: using psize instead of size here due to type mismatches with MemBuf.
 440
 441     // content is long enough to possibly hold a reply
 442     // 4 being magic size of a 3-digit number plus space delimiter
 443     if ( buf->contentSize() < (protoPrefix.psize() + 4) ) {
 444         if (hdr_len > 0) {
 445             debugs(58, 3, HERE << "Too small reply header (" << hdr_len << " bytes)");
 446             *error = Http::scInvalidHeader;
 447         }
 448         return false;
 449     }
 450
 451     int pos;
 452     // catch missing or mismatched protocol identifier
 453     // allow special-case for ICY protocol (non-HTTP identifier) in response to faked HTTP request.
 454     if (strncmp(buf->content(), "ICY", 3) == 0) {
 455         protoPrefix = "ICY";
 456         pos = protoPrefix.psize();
 457     } else {
 458
 459         if (protoPrefix.cmp(buf->content(), protoPrefix.size()) != 0) {
 460             debugs(58, 3, "HttpReply::sanityCheckStartLine: missing protocol prefix (" << protoPrefix << ") in '" << buf->content() << "'");
 461             *error = Http::scInvalidHeader;
 462             return false;
 463         }
 464
 465         // catch missing or negative status value (negative '-' is not a digit)
 466         pos = protoPrefix.psize();
 467
 468         // skip arbitrary number of digits and a dot in the verion portion
 469         while ( pos <= buf->contentSize() && (*(buf->content()+pos) == '.' || xisdigit(*(buf->content()+pos)) ) ) ++pos;
 470
 471         // catch missing version info
 472         if (pos == protoPrefix.psize()) {
 473             debugs(58, 3, "HttpReply::sanityCheckStartLine: missing protocol version numbers (ie. " << protoPrefix << "/1.0) in '" << buf->content() << "'");
 474             *error = Http::scInvalidHeader;
 475             return false;
 476         }
 477     }
 478
 479     // skip arbitrary number of spaces...
 480     while (pos <= buf->contentSize() && (char)*(buf->content()+pos) == ' ') ++pos;
 481
 482     if (pos < buf->contentSize() && !xisdigit(*(buf->content()+pos))) {
 483         debugs(58, 3, "HttpReply::sanityCheckStartLine: missing or invalid status number in '" << buf->content() << "'");
 484         *error = Http::scInvalidHeader;
 485         return false;
 486     }
 487
 488     return true;
 489 }
 490
 491 bool
 492 HttpReply::parseFirstLine(const char *blk_start, const char *blk_end)
 493 {
 494     return sline.parse(protoPrefix, blk_start, blk_end);
 495 }
 496
 497 /* handy: resets and returns -1 */
 498 int
 499 HttpReply::httpMsgParseError()
 500 {
 501     int result(HttpMsg::httpMsgParseError());
 502     /* indicate an error in the status line */
 503     sline.set(Http::ProtocolVersion(1,1), Http::scInvalidHeader);
 504     return result;
 505 }
 506
 507 /*
 508  * Indicate whether or not we would usually expect an entity-body
 509  * along with this response
 510  */
 511 bool
 512 HttpReply::expectingBody(const HttpRequestMethod& req_method, int64_t& theSize) const
 513 {
 514     bool expectBody = true;
 515
 516     if (req_method == Http::METHOD_HEAD)
 517         expectBody = false;
 518     else if (sline.status() == Http::scNoContent)
 519         expectBody = false;
 520     else if (sline.status() == Http::scNotModified)
 521         expectBody = false;
 522     else if (sline.status() < Http::scOkay)
 523         expectBody = false;
 524     else
 525         expectBody = true;
 526
 527     if (expectBody) {
 528         if (header.chunked())
 529             theSize = -1;
 530         else if (content_length >= 0)
 531             theSize = content_length;
 532         else
 533             theSize = -1;
 534     }
 535
 536     return expectBody;
 537 }
 538
 539 bool
 540 HttpReply::receivedBodyTooLarge(HttpRequest& request, int64_t receivedSize)
 541 {
 542     calcMaxBodySize(request);
 543     debugs(58, 3, HERE << receivedSize << " >? " << bodySizeMax);
 544     return bodySizeMax >= 0 && receivedSize > bodySizeMax;
 545 }
 546
 547 bool
 548 HttpReply::expectedBodyTooLarge(HttpRequest& request)
 549 {
 550     calcMaxBodySize(request);
 551     debugs(58, 7, HERE << "bodySizeMax=" << bodySizeMax);
 552
 553     if (bodySizeMax < 0) // no body size limit
 554         return false;
 555
 556     int64_t expectedSize = -1;
 557     if (!expectingBody(request.method, expectedSize))
 558         return false;
 559
 560     debugs(58, 6, HERE << expectedSize << " >? " << bodySizeMax);
 561
 562     if (expectedSize < 0) // expecting body of an unknown length
 563         return false;
 564
 565     return expectedSize > bodySizeMax;
 566 }
 567
 568 void
 569 HttpReply::calcMaxBodySize(HttpRequest& request) const
 570 {
 571     // hack: -2 is used as "we have not calculated max body size yet" state
 572     if (bodySizeMax != -2) // already tried
 573         return;
 574     bodySizeMax = -1;
 575
 576     // short-circuit ACL testing if there are none configured
 577     if (!Config.ReplyBodySize)
 578         return;
 579
 580     ACLFilledChecklist ch(NULL, &request, NULL);
 581     // XXX: cont-cast becomes irrelevant when checklist is HttpReply::Pointer
 582     ch.reply = const_cast<HttpReply *>(this);
 583     HTTPMSGLOCK(ch.reply);
 584     for (AclSizeLimit *l = Config.ReplyBodySize; l; l = l -> next) {
 585         /* if there is no ACL list or if the ACLs listed match use this size value */
 586         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 587             debugs(58, 4, HERE << "bodySizeMax=" << bodySizeMax);
 588             bodySizeMax = l->size; // may be -1
 589             break;
 590         }
 591     }
 592 }
 593
 594 // XXX: check that this is sufficient for eCAP cloning
 595 HttpReply *
 596 HttpReply::clone() const
 597 {
 598     HttpReply *rep = new HttpReply();
 599     rep->sline = sline; // used in hdrCacheInit() call below
 600     rep->header.append(&header);
 601     rep->hdrCacheInit();
 602     rep->hdr_sz = hdr_sz;
 603     rep->http_ver = http_ver;
 604     rep->pstate = pstate;
 605     rep->body_pipe = body_pipe;
 606
 607     // keep_alive is handled in hdrCacheInit()
 608     return rep;
 609 }
 610
 611 bool HttpReply::inheritProperties(const HttpMsg *aMsg)
 612 {
 613     const HttpReply *aRep = dynamic_cast<const HttpReply*>(aMsg);
 614     if (!aRep)
 615         return false;
 616     keep_alive = aRep->keep_alive;
 617     return true;
 618 }
 619
 620 void HttpReply::removeStaleWarnings()
 621 {
 622     String warning;
 623     if (header.getList(HDR_WARNING, &warning)) {
 624         const String newWarning = removeStaleWarningValues(warning);
 625         if (warning.size() && warning.size() == newWarning.size())
 626             return; // some warnings are there and none changed
 627         header.delById(HDR_WARNING);
 628         if (newWarning.size()) { // some warnings left
 629             HttpHeaderEntry *const e =
 630                 new HttpHeaderEntry(HDR_WARNING, NULL, newWarning.termedBuf());
 631             header.addEntry(e);
 632         }
 633     }
 634 }
 635
 636 /**
 637  * Remove warning-values with warn-date different from Date value from
 638  * a single header entry. Returns a string with all valid warning-values.
 639  */
 640 String HttpReply::removeStaleWarningValues(const String &value)
 641 {
 642     String newValue;
 643     const char *item = 0;
 644     int len = 0;
 645     const char *pos = 0;
 646     while (strListGetItem(&value, ',', &item, &len, &pos)) {
 647         bool keep = true;
 648         // Does warning-value have warn-date (which contains quoted date)?
 649         // We scan backwards, looking for two quoted strings.
 650         // warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
 651         const char *p = item + len - 1;
 652
 653         while (p >= item && xisspace(*p)) --p; // skip whitespace
 654
 655         // warning-value MUST end with quote
 656         if (p >= item && *p == '"') {
 657             const char *const warnDateEnd = p;
 658             --p;
 659             while (p >= item && *p != '"') --p; // find the next quote
 660
 661             const char *warnDateBeg = p + 1;
 662             --p;
 663             while (p >= item && xisspace(*p)) --p; // skip whitespace
 664
 665             if (p >= item && *p == '"' && warnDateBeg - p > 2) {
 666                 // found warn-text
 667                 String warnDate;
 668                 warnDate.append(warnDateBeg, warnDateEnd - warnDateBeg);
 669                 const time_t time = parse_rfc1123(warnDate.termedBuf());
 670                 keep = (time > 0 && time == date); // keep valid and matching date
 671             }
 672         }
 673
 674         if (keep) {
 675             if (newValue.size())
 676                 newValue.append(", ");
 677             newValue.append(item, len);
 678         }
 679     }
 680
 681     return newValue;
 682 }