src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "dns/LookupDetails.h"
  17 #include "err_detail_type.h"
  18 #include "globals.h"
  19 #include "gopher.h"
  20 #include "http.h"
  21 #include "http/one/RequestParser.h"
  22 #include "HttpHdrCc.h"
  23 #include "HttpHeaderRange.h"
  24 #include "HttpRequest.h"
  25 #include "log/Config.h"
  26 #include "MemBuf.h"
  27 #include "SquidConfig.h"
  28 #include "Store.h"
  29 #include "URL.h"
  30
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if ICAP_CLIENT
  35 #include "adaptation/icap/icap_log.h"
  36 #endif
  37
  38 HttpRequest::HttpRequest() :
  39     HttpMsg(hoRequest)
  40 {
  41     init();
  42 }
  43
  44 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  45     HttpMsg(hoRequest)
  46 {
  47     static unsigned int id = 1;
  48     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  49     init();
  50     initHTTP(aMethod, aProtocol, aUrlpath);
  51 }
  52
  53 HttpRequest::~HttpRequest()
  54 {
  55     clean();
  56     debugs(93,7, HERE << "destructed, this=" << this);
  57 }
  58
  59 void
  60 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  61 {
  62     method = aMethod;
  63     url.setScheme(aProtocol);
  64     url.path(aUrlpath);
  65 }
  66
  67 void
  68 HttpRequest::init()
  69 {
  70     method = Http::METHOD_NONE;
  71     url.clear();
  72 #if USE_AUTH
  73     auth_user_request = NULL;
  74 #endif
  75     memset(&flags, '\0', sizeof(flags));
  76     range = NULL;
  77     ims = -1;
  78     imslen = 0;
  79     lastmod = -1;
  80     client_addr.setEmpty();
  81     my_addr.setEmpty();
  82     body_pipe = NULL;
  83     // hier
  84     dnsWait = -1;
  85     errType = ERR_NONE;
  86     errDetail = ERR_DETAIL_NONE;
  87     peer_login = NULL;      // not allocated/deallocated by this class
  88     peer_domain = NULL;     // not allocated/deallocated by this class
  89     peer_host = NULL;
  90     vary_headers = NULL;
  91     myportname = null_string;
  92     tag = null_string;
  93 #if USE_AUTH
  94     extacl_user = null_string;
  95     extacl_passwd = null_string;
  96 #endif
  97     extacl_log = null_string;
  98     extacl_message = null_string;
  99     pstate = psReadyToParseStartLine;
 100 #if FOLLOW_X_FORWARDED_FOR
 101     indirect_client_addr.setEmpty();
 102 #endif /* FOLLOW_X_FORWARDED_FOR */
 103 #if USE_ADAPTATION
 104     adaptHistory_ = NULL;
 105 #endif
 106 #if ICAP_CLIENT
 107     icapHistory_ = NULL;
 108 #endif
 109     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 110     forcedBodyContinuation = false;
 111 }
 112
 113 void
 114 HttpRequest::clean()
 115 {
 116     // we used to assert that the pipe is NULL, but now the request only
 117     // points to a pipe that is owned and initiated by another object.
 118     body_pipe = NULL;
 119 #if USE_AUTH
 120     auth_user_request = NULL;
 121 #endif
 122     safe_free(vary_headers);
 123
 124     url.clear();
 125
 126     header.clean();
 127
 128     if (cache_control) {
 129         delete cache_control;
 130         cache_control = NULL;
 131     }
 132
 133     if (range) {
 134         delete range;
 135         range = NULL;
 136     }
 137
 138     myportname.clean();
 139
 140     notes = NULL;
 141
 142     tag.clean();
 143 #if USE_AUTH
 144     extacl_user.clean();
 145     extacl_passwd.clean();
 146 #endif
 147     extacl_log.clean();
 148
 149     extacl_message.clean();
 150
 151     etag.clean();
 152
 153 #if USE_ADAPTATION
 154     adaptHistory_ = NULL;
 155 #endif
 156 #if ICAP_CLIENT
 157     icapHistory_ = NULL;
 158 #endif
 159 }
 160
 161 void
 162 HttpRequest::reset()
 163 {
 164     clean();
 165     init();
 166 }
 167
 168 HttpRequest *
 169 HttpRequest::clone() const
 170 {
 171     HttpRequest *copy = new HttpRequest();
 172     copy->method = method;
 173     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 174     copy->header.append(&header);
 175     copy->hdrCacheInit();
 176     copy->hdr_sz = hdr_sz;
 177     copy->http_ver = http_ver;
 178     copy->pstate = pstate; // TODO: should we assert a specific state here?
 179     copy->body_pipe = body_pipe;
 180
 181     copy->url.setScheme(url.getScheme());
 182     copy->url.userInfo(url.userInfo());
 183     copy->url.host(url.host());
 184     copy->url.port(url.port());
 185     copy->url.path(url.path());
 186
 187     // range handled in hdrCacheInit()
 188     copy->ims = ims;
 189     copy->imslen = imslen;
 190     copy->hier = hier; // Is it safe to copy? Should we?
 191
 192     copy->errType = errType;
 193
 194     // XXX: what to do with copy->peer_login?
 195
 196     copy->lastmod = lastmod;
 197     copy->etag = etag;
 198     copy->vary_headers = vary_headers ? xstrdup(vary_headers) : NULL;
 199     // XXX: what to do with copy->peer_domain?
 200
 201     copy->tag = tag;
 202     copy->extacl_log = extacl_log;
 203     copy->extacl_message = extacl_message;
 204
 205     const bool inheritWorked = copy->inheritProperties(this);
 206     assert(inheritWorked);
 207
 208     return copy;
 209 }
 210
 211 bool
 212 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 213 {
 214     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 215     if (!aReq)
 216         return false;
 217
 218     client_addr = aReq->client_addr;
 219 #if FOLLOW_X_FORWARDED_FOR
 220     indirect_client_addr = aReq->indirect_client_addr;
 221 #endif
 222     my_addr = aReq->my_addr;
 223
 224     dnsWait = aReq->dnsWait;
 225
 226 #if USE_ADAPTATION
 227     adaptHistory_ = aReq->adaptHistory();
 228 #endif
 229 #if ICAP_CLIENT
 230     icapHistory_ = aReq->icapHistory();
 231 #endif
 232
 233     // This may be too conservative for the 204 No Content case
 234     // may eventually need cloneNullAdaptationImmune() for that.
 235     flags = aReq->flags.cloneAdaptationImmune();
 236
 237     errType = aReq->errType;
 238     errDetail = aReq->errDetail;
 239 #if USE_AUTH
 240     auth_user_request = aReq->auth_user_request;
 241     extacl_user = aReq->extacl_user;
 242     extacl_passwd = aReq->extacl_passwd;
 243 #endif
 244
 245     myportname = aReq->myportname;
 246
 247     forcedBodyContinuation = aReq->forcedBodyContinuation;
 248
 249     // main property is which connection the request was received on (if any)
 250     clientConnectionManager = aReq->clientConnectionManager;
 251
 252     notes = aReq->notes;
 253     return true;
 254 }
 255
 256 /**
 257  * Checks the first line of an HTTP request is valid
 258  * currently just checks the request method is present.
 259  *
 260  * NP: Other errors are left for detection later in the parse.
 261  */
 262 bool
 263 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 264 {
 265     // content is long enough to possibly hold a reply
 266     // 2 being magic size of a 1-byte request method plus space delimiter
 267     if (hdr_len < 2) {
 268         // this is ony a real error if the headers apparently complete.
 269         if (hdr_len > 0) {
 270             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 271             *error = Http::scInvalidHeader;
 272         }
 273         return false;
 274     }
 275
 276     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 277     HttpRequestMethod m;
 278     m.HttpRequestMethodXXX(buf);
 279     if (m == Http::METHOD_NONE) {
 280         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 281         *error = Http::scInvalidHeader;
 282         return false;
 283     }
 284
 285     return true;
 286 }
 287
 288 bool
 289 HttpRequest::parseFirstLine(const char *start, const char *end)
 290 {
 291     method.HttpRequestMethodXXX(start);
 292
 293     if (method == Http::METHOD_NONE)
 294         return false;
 295
 296     // XXX: performance regression, strcspn() over the method bytes a second time.
 297     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 298     const char *t = start + strcspn(start, w_space);
 299
 300     start = t + strspn(t, w_space); // skip w_space after method
 301
 302     const char *ver = findTrailingHTTPVersion(start, end);
 303
 304     if (ver) {
 305         end = ver - 1;
 306
 307         while (xisspace(*end)) // find prev non-space
 308             --end;
 309
 310         ++end;                 // back to space
 311
 312         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 313             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 314             return false;
 315         }
 316     } else {
 317         http_ver.major = 0;
 318         http_ver.minor = 9;
 319     }
 320
 321     if (end < start)   // missing URI
 322         return false;
 323
 324     char save = *end;
 325
 326     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 327
 328     HttpRequest *tmp = urlParse(method, (char *) start, this);
 329
 330     * (char *) end = save;
 331
 332     if (NULL == tmp)
 333         return false;
 334
 335     return true;
 336 }
 337
 338 bool
 339 HttpRequest::parseHeader(Http1::RequestParser &hp)
 340 {
 341     // HTTP/1 message contains "zero or more header fields"
 342     // zero does not need parsing
 343     if (!hp.headerBlockSize())
 344         return true;
 345
 346     // XXX: c_str() reallocates. performance regression.
 347     const bool result = header.parse(hp.mimeHeader().c_str(), hp.headerBlockSize());
 348
 349     if (result)
 350         hdrCacheInit();
 351
 352     return result;
 353 }
 354
 355 /* swaps out request using httpRequestPack */
 356 void
 357 HttpRequest::swapOut(StoreEntry * e)
 358 {
 359     assert(e);
 360     e->buffer();
 361     pack(e);
 362     e->flush();
 363 }
 364
 365 /* packs request-line and headers, appends <crlf> terminator */
 366 void
 367 HttpRequest::pack(Packable * p)
 368 {
 369     assert(p);
 370     /* pack request-line */
 371     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 372                SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
 373                http_ver.major, http_ver.minor);
 374     /* headers */
 375     header.packInto(p);
 376     /* trailer */
 377     p->append("\r\n", 2);
 378 }
 379
 380 /*
 381  * A wrapper for debugObj()
 382  */
 383 void
 384 httpRequestPack(void *obj, Packable *p)
 385 {
 386     HttpRequest *request = static_cast<HttpRequest*>(obj);
 387     request->pack(p);
 388 }
 389
 390 /* returns the length of request line + headers + crlf */
 391 int
 392 HttpRequest::prefixLen() const
 393 {
 394     return method.image().length() + 1 +
 395            url.path().length() + 1 +
 396            4 + 1 + 3 + 2 +
 397            header.len + 2;
 398 }
 399
 400 /* sync this routine when you update HttpRequest struct */
 401 void
 402 HttpRequest::hdrCacheInit()
 403 {
 404     HttpMsg::hdrCacheInit();
 405
 406     assert(!range);
 407     range = header.getRange();
 408 }
 409
 410 #if ICAP_CLIENT
 411 Adaptation::Icap::History::Pointer
 412 HttpRequest::icapHistory() const
 413 {
 414     if (!icapHistory_) {
 415         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 416             icapHistory_ = new Adaptation::Icap::History();
 417             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 418         }
 419     }
 420
 421     return icapHistory_;
 422 }
 423 #endif
 424
 425 #if USE_ADAPTATION
 426 Adaptation::History::Pointer
 427 HttpRequest::adaptHistory(bool createIfNone) const
 428 {
 429     if (!adaptHistory_ && createIfNone) {
 430         adaptHistory_ = new Adaptation::History();
 431         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 432     }
 433
 434     return adaptHistory_;
 435 }
 436
 437 Adaptation::History::Pointer
 438 HttpRequest::adaptLogHistory() const
 439 {
 440     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 441 }
 442
 443 void
 444 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 445 {
 446     if (!adaptHistory_) {
 447         adaptHistory_ = them.adaptHistory_; // may be nil
 448     } else {
 449         // check that histories did not diverge
 450         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 451     }
 452 }
 453
 454 #endif
 455
 456 bool
 457 HttpRequest::multipartRangeRequest() const
 458 {
 459     return (range && range->specs.size() > 1);
 460 }
 461
 462 bool
 463 HttpRequest::bodyNibbled() const
 464 {
 465     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 466 }
 467
 468 void
 469 HttpRequest::detailError(err_type aType, int aDetail)
 470 {
 471     if (errType || errDetail)
 472         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 473     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 474     // checking type and detail separately may cause inconsistency, but
 475     // may result in more details available if they only become available later
 476     if (!errType)
 477         errType = aType;
 478     if (!errDetail)
 479         errDetail = aDetail;
 480 }
 481
 482 void
 483 HttpRequest::clearError()
 484 {
 485     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 486     errType = ERR_NONE;
 487     errDetail = ERR_DETAIL_NONE;
 488 }
 489
 490 void
 491 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
 492 {
 493     const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
 494
 495     // form HTTP request-line
 496     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 497                SQUIDSBUFPRINT(method.image()),
 498                SQUIDSBUFPRINT(tmp),
 499                http_ver.major, http_ver.minor);
 500 }
 501
 502 /*
 503  * Indicate whether or not we would expect an entity-body
 504  * along with this request
 505  */
 506 bool
 507 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
 508 {
 509     bool expectBody = false;
 510
 511     /*
 512      * Note: Checks for message validity is in clientIsContentLengthValid().
 513      * this just checks if a entity-body is expected based on HTTP message syntax
 514      */
 515     if (header.chunked()) {
 516         expectBody = true;
 517         theSize = -1;
 518     } else if (content_length >= 0) {
 519         expectBody = true;
 520         theSize = content_length;
 521     } else {
 522         expectBody = false;
 523         // theSize undefined
 524     }
 525
 526     return expectBody;
 527 }
 528
 529 /*
 530  * Create a Request from a URL and METHOD.
 531  *
 532  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 533  * If the request cannot be created cleanly, NULL is returned
 534  */
 535 HttpRequest *
 536 HttpRequest::CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method)
 537 {
 538     return urlParse(method, url, NULL);
 539 }
 540
 541 /*
 542  * Create a Request from a URL.
 543  *
 544  * If the request cannot be created cleanly, NULL is returned
 545  */
 546 HttpRequest *
 547 HttpRequest::CreateFromUrl(char * url)
 548 {
 549     return urlParse(Http::METHOD_GET, url, NULL);
 550 }
 551
 552 /**
 553  * Are responses to this request possible cacheable ?
 554  * If false then no matter what the response must not be cached.
 555  */
 556 bool
 557 HttpRequest::maybeCacheable()
 558 {
 559     // Intercepted request with Host: header which cannot be trusted.
 560     // Because it failed verification, or someone bypassed the security tests
 561     // we cannot cache the reponse for sharing between clients.
 562     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 563     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 564         return false;
 565
 566     switch (url.getScheme()) {
 567     case AnyP::PROTO_HTTP:
 568     case AnyP::PROTO_HTTPS:
 569         if (!method.respMaybeCacheable())
 570             return false;
 571
 572         // XXX: this would seem the correct place to detect request cache-controls
 573         //      no-store, private and related which block cacheability
 574         break;
 575
 576     case AnyP::PROTO_GOPHER:
 577         if (!gopherCachable(this))
 578             return false;
 579         break;
 580
 581     case AnyP::PROTO_CACHE_OBJECT:
 582         return false;
 583
 584     //case AnyP::PROTO_FTP:
 585     default:
 586         break;
 587     }
 588
 589     return true;
 590 }
 591
 592 bool
 593 HttpRequest::conditional() const
 594 {
 595     return flags.ims ||
 596            header.has(Http::HdrType::IF_MATCH) ||
 597            header.has(Http::HdrType::IF_NONE_MATCH);
 598 }
 599
 600 void
 601 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
 602 {
 603     if (dns.wait >= 0) { // known delay
 604         if (dnsWait >= 0) // have recorded DNS wait before
 605             dnsWait += dns.wait;
 606         else
 607             dnsWait = dns.wait;
 608     }
 609 }
 610
 611 int64_t
 612 HttpRequest::getRangeOffsetLimit()
 613 {
 614     /* -2 is the starting value of rangeOffsetLimit.
 615      * If it is -2, that means we haven't checked it yet.
 616      *  Otherwise, return the current value */
 617     if (rangeOffsetLimit != -2)
 618         return rangeOffsetLimit;
 619
 620     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 621
 622     ACLFilledChecklist ch(NULL, this, NULL);
 623     ch.src_addr = client_addr;
 624     ch.my_addr =  my_addr;
 625
 626     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 627         /* if there is no ACL list or if the ACLs listed match use this limit value */
 628         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 629             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 630             rangeOffsetLimit = l->size; // may be -1
 631             break;
 632         }
 633     }
 634
 635     return rangeOffsetLimit;
 636 }
 637
 638 void
 639 HttpRequest::ignoreRange(const char *reason)
 640 {
 641     if (range) {
 642         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 643         delete range;
 644         range = NULL;
 645     }
 646     // Some callers also reset isRanged but it may not be safe for all callers:
 647     // isRanged is used to determine whether a weak ETag comparison is allowed,
 648     // and that check should not ignore the Range header if it was present.
 649     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 650 }
 651
 652 bool
 653 HttpRequest::canHandle1xx() const
 654 {
 655     // old clients do not support 1xx unless they sent Expect: 100-continue
 656     // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
 657     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
 658         return false;
 659
 660     // others must support 1xx control messages
 661     return true;
 662 }
 663
 664 ConnStateData *
 665 HttpRequest::pinnedConnection()
 666 {
 667     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 668         return clientConnectionManager.get();
 669     return NULL;
 670 }
 671
 672 const SBuf
 673 HttpRequest::storeId()
 674 {
 675     if (store_id.size() != 0) {
 676         debugs(73, 3, "sent back store_id: " << store_id);
 677         return SBuf(store_id);
 678     }
 679     debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
 680     return effectiveRequestUri();
 681 }
 682
 683 const SBuf &
 684 HttpRequest::effectiveRequestUri() const
 685 {
 686     if (method.id() == Http::METHOD_CONNECT)
 687         return url.authority(true); // host:port
 688     return url.absolute();
 689 }
 690