src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "DnsLookupDetails.h"
  17 #include "err_detail_type.h"
  18 #include "globals.h"
  19 #include "gopher.h"
  20 #include "http.h"
  21 #include "http/one/RequestParser.h"
  22 #include "HttpHdrCc.h"
  23 #include "HttpHeaderRange.h"
  24 #include "HttpRequest.h"
  25 #include "log/Config.h"
  26 #include "MemBuf.h"
  27 #include "SquidConfig.h"
  28 #include "Store.h"
  29 #include "URL.h"
  30
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if ICAP_CLIENT
  35 #include "adaptation/icap/icap_log.h"
  36 #endif
  37
  38 HttpRequest::HttpRequest() :
  39         HttpMsg(hoRequest)
  40 {
  41     init();
  42 }
  43
  44 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  45         HttpMsg(hoRequest)
  46 {
  47     static unsigned int id = 1;
  48     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  49     init();
  50     initHTTP(aMethod, aProtocol, aUrlpath);
  51 }
  52
  53 HttpRequest::~HttpRequest()
  54 {
  55     clean();
  56     debugs(93,7, HERE << "destructed, this=" << this);
  57 }
  58
  59 void
  60 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  61 {
  62     method = aMethod;
  63     url.setScheme(aProtocol);
  64     urlpath = aUrlpath;
  65 }
  66
  67 void
  68 HttpRequest::init()
  69 {
  70     method = Http::METHOD_NONE;
  71     url.clear();
  72     urlpath = NULL;
  73     host[0] = '\0';
  74     host_is_numeric = -1;
  75 #if USE_AUTH
  76     auth_user_request = NULL;
  77 #endif
  78     port = 0;
  79     canonical = NULL;
  80     memset(&flags, '\0', sizeof(flags));
  81     range = NULL;
  82     ims = -1;
  83     imslen = 0;
  84     lastmod = -1;
  85     client_addr.setEmpty();
  86     my_addr.setEmpty();
  87     body_pipe = NULL;
  88     // hier
  89     dnsWait = -1;
  90     errType = ERR_NONE;
  91     errDetail = ERR_DETAIL_NONE;
  92     peer_login = NULL;          // not allocated/deallocated by this class
  93     peer_domain = NULL;         // not allocated/deallocated by this class
  94     peer_host = NULL;
  95     vary_headers = NULL;
  96     myportname = null_string;
  97     tag = null_string;
  98 #if USE_AUTH
  99     extacl_user = null_string;
 100     extacl_passwd = null_string;
 101 #endif
 102     extacl_log = null_string;
 103     extacl_message = null_string;
 104     pstate = psReadyToParseStartLine;
 105 #if FOLLOW_X_FORWARDED_FOR
 106     indirect_client_addr.setEmpty();
 107 #endif /* FOLLOW_X_FORWARDED_FOR */
 108 #if USE_ADAPTATION
 109     adaptHistory_ = NULL;
 110 #endif
 111 #if ICAP_CLIENT
 112     icapHistory_ = NULL;
 113 #endif
 114     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 115     forcedBodyContinuation = false;
 116 }
 117
 118 void
 119 HttpRequest::clean()
 120 {
 121     // we used to assert that the pipe is NULL, but now the request only
 122     // points to a pipe that is owned and initiated by another object.
 123     body_pipe = NULL;
 124 #if USE_AUTH
 125     auth_user_request = NULL;
 126 #endif
 127     safe_free(canonical);
 128
 129     safe_free(vary_headers);
 130
 131     url.clear();
 132     urlpath.clean();
 133
 134     header.clean();
 135
 136     if (cache_control) {
 137         delete cache_control;
 138         cache_control = NULL;
 139     }
 140
 141     if (range) {
 142         delete range;
 143         range = NULL;
 144     }
 145
 146     myportname.clean();
 147
 148     notes = NULL;
 149
 150     tag.clean();
 151 #if USE_AUTH
 152     extacl_user.clean();
 153     extacl_passwd.clean();
 154 #endif
 155     extacl_log.clean();
 156
 157     extacl_message.clean();
 158
 159     etag.clean();
 160
 161 #if USE_ADAPTATION
 162     adaptHistory_ = NULL;
 163 #endif
 164 #if ICAP_CLIENT
 165     icapHistory_ = NULL;
 166 #endif
 167 }
 168
 169 void
 170 HttpRequest::reset()
 171 {
 172     clean();
 173     init();
 174 }
 175
 176 HttpRequest *
 177 HttpRequest::clone() const
 178 {
 179     HttpRequest *copy = new HttpRequest(method, url.getScheme(), urlpath.termedBuf());
 180     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 181     copy->header.append(&header);
 182     copy->hdrCacheInit();
 183     copy->hdr_sz = hdr_sz;
 184     copy->http_ver = http_ver;
 185     copy->pstate = pstate; // TODO: should we assert a specific state here?
 186     copy->body_pipe = body_pipe;
 187
 188     copy->url.userInfo(url.userInfo());
 189     strncpy(copy->host, host, sizeof(host)); // SQUIDHOSTNAMELEN
 190     copy->host_addr = host_addr;
 191
 192     copy->port = port;
 193     // urlPath handled in ctor
 194     copy->canonical = canonical ? xstrdup(canonical) : NULL;
 195
 196     // range handled in hdrCacheInit()
 197     copy->ims = ims;
 198     copy->imslen = imslen;
 199     copy->hier = hier; // Is it safe to copy? Should we?
 200
 201     copy->errType = errType;
 202
 203     // XXX: what to do with copy->peer_login?
 204
 205     copy->lastmod = lastmod;
 206     copy->etag = etag;
 207     copy->vary_headers = vary_headers ? xstrdup(vary_headers) : NULL;
 208     // XXX: what to do with copy->peer_domain?
 209
 210     copy->tag = tag;
 211     copy->extacl_log = extacl_log;
 212     copy->extacl_message = extacl_message;
 213
 214     const bool inheritWorked = copy->inheritProperties(this);
 215     assert(inheritWorked);
 216
 217     return copy;
 218 }
 219
 220 bool
 221 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 222 {
 223     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 224     if (!aReq)
 225         return false;
 226
 227     client_addr = aReq->client_addr;
 228 #if FOLLOW_X_FORWARDED_FOR
 229     indirect_client_addr = aReq->indirect_client_addr;
 230 #endif
 231     my_addr = aReq->my_addr;
 232
 233     dnsWait = aReq->dnsWait;
 234
 235 #if USE_ADAPTATION
 236     adaptHistory_ = aReq->adaptHistory();
 237 #endif
 238 #if ICAP_CLIENT
 239     icapHistory_ = aReq->icapHistory();
 240 #endif
 241
 242     // This may be too conservative for the 204 No Content case
 243     // may eventually need cloneNullAdaptationImmune() for that.
 244     flags = aReq->flags.cloneAdaptationImmune();
 245
 246     errType = aReq->errType;
 247     errDetail = aReq->errDetail;
 248 #if USE_AUTH
 249     auth_user_request = aReq->auth_user_request;
 250     extacl_user = aReq->extacl_user;
 251     extacl_passwd = aReq->extacl_passwd;
 252 #endif
 253
 254     myportname = aReq->myportname;
 255
 256     forcedBodyContinuation = aReq->forcedBodyContinuation;
 257
 258     // main property is which connection the request was received on (if any)
 259     clientConnectionManager = aReq->clientConnectionManager;
 260
 261     notes = aReq->notes;
 262     return true;
 263 }
 264
 265 /**
 266  * Checks the first line of an HTTP request is valid
 267  * currently just checks the request method is present.
 268  *
 269  * NP: Other errors are left for detection later in the parse.
 270  */
 271 bool
 272 HttpRequest::sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, Http::StatusCode *error)
 273 {
 274     // content is long enough to possibly hold a reply
 275     // 2 being magic size of a 1-byte request method plus space delimiter
 276     if ( buf->contentSize() < 2 ) {
 277         // this is ony a real error if the headers apparently complete.
 278         if (hdr_len > 0) {
 279             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 280             *error = Http::scInvalidHeader;
 281         }
 282         return false;
 283     }
 284
 285     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 286     HttpRequestMethod m;
 287     m.HttpRequestMethodXXX(buf->content());
 288     if (m == Http::METHOD_NONE) {
 289         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 290         *error = Http::scInvalidHeader;
 291         return false;
 292     }
 293
 294     return true;
 295 }
 296
 297 bool
 298 HttpRequest::parseFirstLine(const char *start, const char *end)
 299 {
 300     method.HttpRequestMethodXXX(start);
 301
 302     if (method == Http::METHOD_NONE)
 303         return false;
 304
 305     // XXX: performance regression, strcspn() over the method bytes a second time.
 306     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 307     const char *t = start + strcspn(start, w_space);
 308
 309     start = t + strspn(t, w_space); // skip w_space after method
 310
 311     const char *ver = findTrailingHTTPVersion(start, end);
 312
 313     if (ver) {
 314         end = ver - 1;
 315
 316         while (xisspace(*end)) // find prev non-space
 317             --end;
 318
 319         ++end;                 // back to space
 320
 321         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 322             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 323             return false;
 324         }
 325     } else {
 326         http_ver.major = 0;
 327         http_ver.minor = 9;
 328     }
 329
 330     if (end < start)   // missing URI
 331         return false;
 332
 333     char save = *end;
 334
 335     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 336
 337     HttpRequest *tmp = urlParse(method, (char *) start, this);
 338
 339     * (char *) end = save;
 340
 341     if (NULL == tmp)
 342         return false;
 343
 344     return true;
 345 }
 346
 347 bool
 348 HttpRequest::parseHeader(Http1::RequestParser &hp)
 349 {
 350     // HTTP/1 message contains "zero or more header fields"
 351     // zero does not need parsing
 352     if (!hp.headerBlockSize())
 353         return true;
 354
 355     // XXX: c_str() reallocates. performance regression.
 356     const bool result = header.parse(hp.mimeHeader().c_str(), hp.headerBlockSize());
 357
 358     if (result)
 359         hdrCacheInit();
 360
 361     return result;
 362 }
 363
 364 /* swaps out request using httpRequestPack */
 365 void
 366 HttpRequest::swapOut(StoreEntry * e)
 367 {
 368     Packer p;
 369     assert(e);
 370     packerToStoreInit(&p, e);
 371     pack(&p);
 372     packerClean(&p);
 373 }
 374
 375 /* packs request-line and headers, appends <crlf> terminator */
 376 void
 377 HttpRequest::pack(Packer * p)
 378 {
 379     assert(p);
 380     /* pack request-line */
 381     packerPrintf(p, SQUIDSBUFPH " " SQUIDSTRINGPH " HTTP/%d.%d\r\n",
 382                  SQUIDSBUFPRINT(method.image()), SQUIDSTRINGPRINT(urlpath),
 383                  http_ver.major, http_ver.minor);
 384     /* headers */
 385     header.packInto(p);
 386     /* trailer */
 387     packerAppend(p, "\r\n", 2);
 388 }
 389
 390 /*
 391  * A wrapper for debugObj()
 392  */
 393 void
 394 httpRequestPack(void *obj, Packer *p)
 395 {
 396     HttpRequest *request = static_cast<HttpRequest*>(obj);
 397     request->pack(p);
 398 }
 399
 400 /* returns the length of request line + headers + crlf */
 401 int
 402 HttpRequest::prefixLen()
 403 {
 404     return method.image().length() + 1 +
 405            urlpath.size() + 1 +
 406            4 + 1 + 3 + 2 +
 407            header.len + 2;
 408 }
 409
 410 /* sync this routine when you update HttpRequest struct */
 411 void
 412 HttpRequest::hdrCacheInit()
 413 {
 414     HttpMsg::hdrCacheInit();
 415
 416     assert(!range);
 417     range = header.getRange();
 418 }
 419
 420 #if ICAP_CLIENT
 421 Adaptation::Icap::History::Pointer
 422 HttpRequest::icapHistory() const
 423 {
 424     if (!icapHistory_) {
 425         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 426             icapHistory_ = new Adaptation::Icap::History();
 427             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 428         }
 429     }
 430
 431     return icapHistory_;
 432 }
 433 #endif
 434
 435 #if USE_ADAPTATION
 436 Adaptation::History::Pointer
 437 HttpRequest::adaptHistory(bool createIfNone) const
 438 {
 439     if (!adaptHistory_ && createIfNone) {
 440         adaptHistory_ = new Adaptation::History();
 441         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 442     }
 443
 444     return adaptHistory_;
 445 }
 446
 447 Adaptation::History::Pointer
 448 HttpRequest::adaptLogHistory() const
 449 {
 450     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 451 }
 452
 453 void
 454 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 455 {
 456     if (!adaptHistory_) {
 457         adaptHistory_ = them.adaptHistory_; // may be nil
 458     } else {
 459         // check that histories did not diverge
 460         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 461     }
 462 }
 463
 464 #endif
 465
 466 bool
 467 HttpRequest::multipartRangeRequest() const
 468 {
 469     return (range && range->specs.size() > 1);
 470 }
 471
 472 bool
 473 HttpRequest::bodyNibbled() const
 474 {
 475     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 476 }
 477
 478 void
 479 HttpRequest::detailError(err_type aType, int aDetail)
 480 {
 481     if (errType || errDetail)
 482         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 483     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 484     // checking type and detail separately may cause inconsistency, but
 485     // may result in more details available if they only become available later
 486     if (!errType)
 487         errType = aType;
 488     if (!errDetail)
 489         errDetail = aDetail;
 490 }
 491
 492 void
 493 HttpRequest::clearError()
 494 {
 495     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 496     errType = ERR_NONE;
 497     errDetail = ERR_DETAIL_NONE;
 498 }
 499
 500 const char *HttpRequest::packableURI(bool full_uri) const
 501 {
 502     if (full_uri)
 503         return urlCanonical((HttpRequest*)this);
 504
 505     if (urlpath.size())
 506         return urlpath.termedBuf();
 507
 508     return "/";
 509 }
 510
 511 void HttpRequest::packFirstLineInto(Packer * p, bool full_uri) const
 512 {
 513     // form HTTP request-line
 514     packerPrintf(p, SQUIDSBUFPH " %s HTTP/%d.%d\r\n",
 515                  SQUIDSBUFPRINT(method.image()),
 516                  packableURI(full_uri),
 517                  http_ver.major, http_ver.minor);
 518 }
 519
 520 /*
 521  * Indicate whether or not we would expect an entity-body
 522  * along with this request
 523  */
 524 bool
 525 HttpRequest::expectingBody(const HttpRequestMethod& unused, int64_t& theSize) const
 526 {
 527     bool expectBody = false;
 528
 529     /*
 530      * Note: Checks for message validity is in clientIsContentLengthValid().
 531      * this just checks if a entity-body is expected based on HTTP message syntax
 532      */
 533     if (header.chunked()) {
 534         expectBody = true;
 535         theSize = -1;
 536     } else if (content_length >= 0) {
 537         expectBody = true;
 538         theSize = content_length;
 539     } else {
 540         expectBody = false;
 541         // theSize undefined
 542     }
 543
 544     return expectBody;
 545 }
 546
 547 /*
 548  * Create a Request from a URL and METHOD.
 549  *
 550  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 551  * If the request cannot be created cleanly, NULL is returned
 552  */
 553 HttpRequest *
 554 HttpRequest::CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method)
 555 {
 556     return urlParse(method, url, NULL);
 557 }
 558
 559 /*
 560  * Create a Request from a URL.
 561  *
 562  * If the request cannot be created cleanly, NULL is returned
 563  */
 564 HttpRequest *
 565 HttpRequest::CreateFromUrl(char * url)
 566 {
 567     return urlParse(Http::METHOD_GET, url, NULL);
 568 }
 569
 570 /**
 571  * Are responses to this request possible cacheable ?
 572  * If false then no matter what the response must not be cached.
 573  */
 574 bool
 575 HttpRequest::maybeCacheable()
 576 {
 577     // Intercepted request with Host: header which cannot be trusted.
 578     // Because it failed verification, or someone bypassed the security tests
 579     // we cannot cache the reponse for sharing between clients.
 580     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 581     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 582         return false;
 583
 584     switch (url.getScheme()) {
 585     case AnyP::PROTO_HTTP:
 586     case AnyP::PROTO_HTTPS:
 587         if (!method.respMaybeCacheable())
 588             return false;
 589
 590         // XXX: this would seem the correct place to detect request cache-controls
 591         //      no-store, private and related which block cacheability
 592         break;
 593
 594     case AnyP::PROTO_GOPHER:
 595         if (!gopherCachable(this))
 596             return false;
 597         break;
 598
 599     case AnyP::PROTO_CACHE_OBJECT:
 600         return false;
 601
 602         //case AnyP::PROTO_FTP:
 603     default:
 604         break;
 605     }
 606
 607     return true;
 608 }
 609
 610 bool
 611 HttpRequest::conditional() const
 612 {
 613     return flags.ims ||
 614            header.has(HDR_IF_MATCH) ||
 615            header.has(HDR_IF_NONE_MATCH);
 616 }
 617
 618 void
 619 HttpRequest::recordLookup(const DnsLookupDetails &dns)
 620 {
 621     if (dns.wait >= 0) { // known delay
 622         if (dnsWait >= 0) // have recorded DNS wait before
 623             dnsWait += dns.wait;
 624         else
 625             dnsWait = dns.wait;
 626     }
 627 }
 628
 629 int64_t
 630 HttpRequest::getRangeOffsetLimit()
 631 {
 632     /* -2 is the starting value of rangeOffsetLimit.
 633      * If it is -2, that means we haven't checked it yet.
 634      *  Otherwise, return the current value */
 635     if (rangeOffsetLimit != -2)
 636         return rangeOffsetLimit;
 637
 638     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 639
 640     ACLFilledChecklist ch(NULL, this, NULL);
 641     ch.src_addr = client_addr;
 642     ch.my_addr =  my_addr;
 643
 644     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 645         /* if there is no ACL list or if the ACLs listed match use this limit value */
 646         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 647             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 648             rangeOffsetLimit = l->size; // may be -1
 649             break;
 650         }
 651     }
 652
 653     return rangeOffsetLimit;
 654 }
 655
 656 void
 657 HttpRequest::ignoreRange(const char *reason)
 658 {
 659     if (range) {
 660         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 661         delete range;
 662         range = NULL;
 663     }
 664     // Some callers also reset isRanged but it may not be safe for all callers:
 665     // isRanged is used to determine whether a weak ETag comparison is allowed,
 666     // and that check should not ignore the Range header if it was present.
 667     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 668 }
 669
 670 bool
 671 HttpRequest::canHandle1xx() const
 672 {
 673     // old clients do not support 1xx unless they sent Expect: 100-continue
 674     // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT)
 675     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(HDR_EXPECT))
 676         return false;
 677
 678     // others must support 1xx control messages
 679     return true;
 680 }
 681
 682 ConnStateData *
 683 HttpRequest::pinnedConnection()
 684 {
 685     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 686         return clientConnectionManager.get();
 687     return NULL;
 688 }
 689
 690 const char *
 691 HttpRequest::storeId()
 692 {
 693     if (store_id.size() != 0) {
 694         debugs(73, 3, "sent back store_id:" << store_id);
 695
 696         return store_id.termedBuf();
 697     }
 698     debugs(73, 3, "sent back canonicalUrl:" << urlCanonical(this) );
 699
 700     return urlCanonical(this);
 701 }