src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "dns/LookupDetails.h"
  17 #include "err_detail_type.h"
  18 #include "globals.h"
  19 #include "gopher.h"
  20 #include "http.h"
  21 #include "http/one/RequestParser.h"
  22 #include "HttpHdrCc.h"
  23 #include "HttpHeaderRange.h"
  24 #include "HttpRequest.h"
  25 #include "log/Config.h"
  26 #include "MemBuf.h"
  27 #include "SquidConfig.h"
  28 #include "Store.h"
  29 #include "URL.h"
  30
  31 #if USE_AUTH
  32 #include "auth/UserRequest.h"
  33 #endif
  34 #if ICAP_CLIENT
  35 #include "adaptation/icap/icap_log.h"
  36 #endif
  37
  38 HttpRequest::HttpRequest() :
  39     HttpMsg(hoRequest)
  40 {
  41     init();
  42 }
  43
  44 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  45     HttpMsg(hoRequest)
  46 {
  47     static unsigned int id = 1;
  48     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  49     init();
  50     initHTTP(aMethod, aProtocol, aUrlpath);
  51 }
  52
  53 HttpRequest::~HttpRequest()
  54 {
  55     clean();
  56     debugs(93,7, HERE << "destructed, this=" << this);
  57 }
  58
  59 void
  60 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  61 {
  62     method = aMethod;
  63     url.setScheme(aProtocol);
  64     url.path(aUrlpath);
  65 }
  66
  67 void
  68 HttpRequest::init()
  69 {
  70     method = Http::METHOD_NONE;
  71     url.clear();
  72 #if USE_AUTH
  73     auth_user_request = NULL;
  74 #endif
  75     canonical = NULL;
  76     memset(&flags, '\0', sizeof(flags));
  77     range = NULL;
  78     ims = -1;
  79     imslen = 0;
  80     lastmod = -1;
  81     client_addr.setEmpty();
  82     my_addr.setEmpty();
  83     body_pipe = NULL;
  84     // hier
  85     dnsWait = -1;
  86     errType = ERR_NONE;
  87     errDetail = ERR_DETAIL_NONE;
  88     peer_login = NULL;      // not allocated/deallocated by this class
  89     peer_domain = NULL;     // not allocated/deallocated by this class
  90     peer_host = NULL;
  91     vary_headers = NULL;
  92     myportname = null_string;
  93     tag = null_string;
  94 #if USE_AUTH
  95     extacl_user = null_string;
  96     extacl_passwd = null_string;
  97 #endif
  98     extacl_log = null_string;
  99     extacl_message = null_string;
 100     pstate = psReadyToParseStartLine;
 101 #if FOLLOW_X_FORWARDED_FOR
 102     indirect_client_addr.setEmpty();
 103 #endif /* FOLLOW_X_FORWARDED_FOR */
 104 #if USE_ADAPTATION
 105     adaptHistory_ = NULL;
 106 #endif
 107 #if ICAP_CLIENT
 108     icapHistory_ = NULL;
 109 #endif
 110     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 111     forcedBodyContinuation = false;
 112 }
 113
 114 void
 115 HttpRequest::clean()
 116 {
 117     // we used to assert that the pipe is NULL, but now the request only
 118     // points to a pipe that is owned and initiated by another object.
 119     body_pipe = NULL;
 120 #if USE_AUTH
 121     auth_user_request = NULL;
 122 #endif
 123     safe_free(canonical);
 124
 125     safe_free(vary_headers);
 126
 127     url.clear();
 128
 129     header.clean();
 130
 131     if (cache_control) {
 132         delete cache_control;
 133         cache_control = NULL;
 134     }
 135
 136     if (range) {
 137         delete range;
 138         range = NULL;
 139     }
 140
 141     myportname.clean();
 142
 143     notes = NULL;
 144
 145     tag.clean();
 146 #if USE_AUTH
 147     extacl_user.clean();
 148     extacl_passwd.clean();
 149 #endif
 150     extacl_log.clean();
 151
 152     extacl_message.clean();
 153
 154     etag.clean();
 155
 156 #if USE_ADAPTATION
 157     adaptHistory_ = NULL;
 158 #endif
 159 #if ICAP_CLIENT
 160     icapHistory_ = NULL;
 161 #endif
 162 }
 163
 164 void
 165 HttpRequest::reset()
 166 {
 167     clean();
 168     init();
 169 }
 170
 171 HttpRequest *
 172 HttpRequest::clone() const
 173 {
 174     HttpRequest *copy = new HttpRequest();
 175     copy->method = method;
 176     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 177     copy->header.append(&header);
 178     copy->hdrCacheInit();
 179     copy->hdr_sz = hdr_sz;
 180     copy->http_ver = http_ver;
 181     copy->pstate = pstate; // TODO: should we assert a specific state here?
 182     copy->body_pipe = body_pipe;
 183
 184     copy->url.setScheme(url.getScheme());
 185     copy->url.userInfo(url.userInfo());
 186     copy->url.host(url.host());
 187     copy->url.port(url.port());
 188     copy->url.path(url.path());
 189
 190     // urlPath handled in ctor
 191     copy->canonical = canonical ? xstrdup(canonical) : NULL;
 192
 193     // range handled in hdrCacheInit()
 194     copy->ims = ims;
 195     copy->imslen = imslen;
 196     copy->hier = hier; // Is it safe to copy? Should we?
 197
 198     copy->errType = errType;
 199
 200     // XXX: what to do with copy->peer_login?
 201
 202     copy->lastmod = lastmod;
 203     copy->etag = etag;
 204     copy->vary_headers = vary_headers ? xstrdup(vary_headers) : NULL;
 205     // XXX: what to do with copy->peer_domain?
 206
 207     copy->tag = tag;
 208     copy->extacl_log = extacl_log;
 209     copy->extacl_message = extacl_message;
 210
 211     const bool inheritWorked = copy->inheritProperties(this);
 212     assert(inheritWorked);
 213
 214     return copy;
 215 }
 216
 217 bool
 218 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 219 {
 220     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 221     if (!aReq)
 222         return false;
 223
 224     client_addr = aReq->client_addr;
 225 #if FOLLOW_X_FORWARDED_FOR
 226     indirect_client_addr = aReq->indirect_client_addr;
 227 #endif
 228     my_addr = aReq->my_addr;
 229
 230     dnsWait = aReq->dnsWait;
 231
 232 #if USE_ADAPTATION
 233     adaptHistory_ = aReq->adaptHistory();
 234 #endif
 235 #if ICAP_CLIENT
 236     icapHistory_ = aReq->icapHistory();
 237 #endif
 238
 239     // This may be too conservative for the 204 No Content case
 240     // may eventually need cloneNullAdaptationImmune() for that.
 241     flags = aReq->flags.cloneAdaptationImmune();
 242
 243     errType = aReq->errType;
 244     errDetail = aReq->errDetail;
 245 #if USE_AUTH
 246     auth_user_request = aReq->auth_user_request;
 247     extacl_user = aReq->extacl_user;
 248     extacl_passwd = aReq->extacl_passwd;
 249 #endif
 250
 251     myportname = aReq->myportname;
 252
 253     forcedBodyContinuation = aReq->forcedBodyContinuation;
 254
 255     // main property is which connection the request was received on (if any)
 256     clientConnectionManager = aReq->clientConnectionManager;
 257
 258     notes = aReq->notes;
 259     return true;
 260 }
 261
 262 /**
 263  * Checks the first line of an HTTP request is valid
 264  * currently just checks the request method is present.
 265  *
 266  * NP: Other errors are left for detection later in the parse.
 267  */
 268 bool
 269 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 270 {
 271     // content is long enough to possibly hold a reply
 272     // 2 being magic size of a 1-byte request method plus space delimiter
 273     if (hdr_len < 2) {
 274         // this is ony a real error if the headers apparently complete.
 275         if (hdr_len > 0) {
 276             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 277             *error = Http::scInvalidHeader;
 278         }
 279         return false;
 280     }
 281
 282     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 283     HttpRequestMethod m;
 284     m.HttpRequestMethodXXX(buf);
 285     if (m == Http::METHOD_NONE) {
 286         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 287         *error = Http::scInvalidHeader;
 288         return false;
 289     }
 290
 291     return true;
 292 }
 293
 294 bool
 295 HttpRequest::parseFirstLine(const char *start, const char *end)
 296 {
 297     method.HttpRequestMethodXXX(start);
 298
 299     if (method == Http::METHOD_NONE)
 300         return false;
 301
 302     // XXX: performance regression, strcspn() over the method bytes a second time.
 303     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 304     const char *t = start + strcspn(start, w_space);
 305
 306     start = t + strspn(t, w_space); // skip w_space after method
 307
 308     const char *ver = findTrailingHTTPVersion(start, end);
 309
 310     if (ver) {
 311         end = ver - 1;
 312
 313         while (xisspace(*end)) // find prev non-space
 314             --end;
 315
 316         ++end;                 // back to space
 317
 318         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 319             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 320             return false;
 321         }
 322     } else {
 323         http_ver.major = 0;
 324         http_ver.minor = 9;
 325     }
 326
 327     if (end < start)   // missing URI
 328         return false;
 329
 330     char save = *end;
 331
 332     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 333
 334     HttpRequest *tmp = urlParse(method, (char *) start, this);
 335
 336     * (char *) end = save;
 337
 338     if (NULL == tmp)
 339         return false;
 340
 341     return true;
 342 }
 343
 344 bool
 345 HttpRequest::parseHeader(Http1::RequestParser &hp)
 346 {
 347     // HTTP/1 message contains "zero or more header fields"
 348     // zero does not need parsing
 349     if (!hp.headerBlockSize())
 350         return true;
 351
 352     // XXX: c_str() reallocates. performance regression.
 353     const bool result = header.parse(hp.mimeHeader().c_str(), hp.headerBlockSize());
 354
 355     if (result)
 356         hdrCacheInit();
 357
 358     return result;
 359 }
 360
 361 /* swaps out request using httpRequestPack */
 362 void
 363 HttpRequest::swapOut(StoreEntry * e)
 364 {
 365     assert(e);
 366     e->buffer();
 367     pack(e);
 368 }
 369
 370 /* packs request-line and headers, appends <crlf> terminator */
 371 void
 372 HttpRequest::pack(Packable * p)
 373 {
 374     assert(p);
 375     /* pack request-line */
 376     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 377                SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
 378                http_ver.major, http_ver.minor);
 379     /* headers */
 380     header.packInto(p);
 381     /* trailer */
 382     p->append("\r\n", 2);
 383 }
 384
 385 /*
 386  * A wrapper for debugObj()
 387  */
 388 void
 389 httpRequestPack(void *obj, Packable *p)
 390 {
 391     HttpRequest *request = static_cast<HttpRequest*>(obj);
 392     request->pack(p);
 393 }
 394
 395 /* returns the length of request line + headers + crlf */
 396 int
 397 HttpRequest::prefixLen() const
 398 {
 399     return method.image().length() + 1 +
 400            url.path().length() + 1 +
 401            4 + 1 + 3 + 2 +
 402            header.len + 2;
 403 }
 404
 405 /* sync this routine when you update HttpRequest struct */
 406 void
 407 HttpRequest::hdrCacheInit()
 408 {
 409     HttpMsg::hdrCacheInit();
 410
 411     assert(!range);
 412     range = header.getRange();
 413 }
 414
 415 #if ICAP_CLIENT
 416 Adaptation::Icap::History::Pointer
 417 HttpRequest::icapHistory() const
 418 {
 419     if (!icapHistory_) {
 420         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 421             icapHistory_ = new Adaptation::Icap::History();
 422             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 423         }
 424     }
 425
 426     return icapHistory_;
 427 }
 428 #endif
 429
 430 #if USE_ADAPTATION
 431 Adaptation::History::Pointer
 432 HttpRequest::adaptHistory(bool createIfNone) const
 433 {
 434     if (!adaptHistory_ && createIfNone) {
 435         adaptHistory_ = new Adaptation::History();
 436         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 437     }
 438
 439     return adaptHistory_;
 440 }
 441
 442 Adaptation::History::Pointer
 443 HttpRequest::adaptLogHistory() const
 444 {
 445     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 446 }
 447
 448 void
 449 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 450 {
 451     if (!adaptHistory_) {
 452         adaptHistory_ = them.adaptHistory_; // may be nil
 453     } else {
 454         // check that histories did not diverge
 455         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 456     }
 457 }
 458
 459 #endif
 460
 461 bool
 462 HttpRequest::multipartRangeRequest() const
 463 {
 464     return (range && range->specs.size() > 1);
 465 }
 466
 467 bool
 468 HttpRequest::bodyNibbled() const
 469 {
 470     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 471 }
 472
 473 void
 474 HttpRequest::detailError(err_type aType, int aDetail)
 475 {
 476     if (errType || errDetail)
 477         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 478     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 479     // checking type and detail separately may cause inconsistency, but
 480     // may result in more details available if they only become available later
 481     if (!errType)
 482         errType = aType;
 483     if (!errDetail)
 484         errDetail = aDetail;
 485 }
 486
 487 void
 488 HttpRequest::clearError()
 489 {
 490     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 491     errType = ERR_NONE;
 492     errDetail = ERR_DETAIL_NONE;
 493 }
 494
 495 void
 496 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
 497 {
 498     SBuf tmp;
 499     if (full_uri)
 500         tmp = urlCanonical((HttpRequest*)this);
 501     else
 502         tmp = url.path();
 503
 504     // form HTTP request-line
 505     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 506                SQUIDSBUFPRINT(method.image()),
 507                SQUIDSBUFPRINT(tmp),
 508                http_ver.major, http_ver.minor);
 509 }
 510
 511 /*
 512  * Indicate whether or not we would expect an entity-body
 513  * along with this request
 514  */
 515 bool
 516 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
 517 {
 518     bool expectBody = false;
 519
 520     /*
 521      * Note: Checks for message validity is in clientIsContentLengthValid().
 522      * this just checks if a entity-body is expected based on HTTP message syntax
 523      */
 524     if (header.chunked()) {
 525         expectBody = true;
 526         theSize = -1;
 527     } else if (content_length >= 0) {
 528         expectBody = true;
 529         theSize = content_length;
 530     } else {
 531         expectBody = false;
 532         // theSize undefined
 533     }
 534
 535     return expectBody;
 536 }
 537
 538 /*
 539  * Create a Request from a URL and METHOD.
 540  *
 541  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 542  * If the request cannot be created cleanly, NULL is returned
 543  */
 544 HttpRequest *
 545 HttpRequest::CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method)
 546 {
 547     return urlParse(method, url, NULL);
 548 }
 549
 550 /*
 551  * Create a Request from a URL.
 552  *
 553  * If the request cannot be created cleanly, NULL is returned
 554  */
 555 HttpRequest *
 556 HttpRequest::CreateFromUrl(char * url)
 557 {
 558     return urlParse(Http::METHOD_GET, url, NULL);
 559 }
 560
 561 /**
 562  * Are responses to this request possible cacheable ?
 563  * If false then no matter what the response must not be cached.
 564  */
 565 bool
 566 HttpRequest::maybeCacheable()
 567 {
 568     // Intercepted request with Host: header which cannot be trusted.
 569     // Because it failed verification, or someone bypassed the security tests
 570     // we cannot cache the reponse for sharing between clients.
 571     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 572     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 573         return false;
 574
 575     switch (url.getScheme()) {
 576     case AnyP::PROTO_HTTP:
 577     case AnyP::PROTO_HTTPS:
 578         if (!method.respMaybeCacheable())
 579             return false;
 580
 581         // XXX: this would seem the correct place to detect request cache-controls
 582         //      no-store, private and related which block cacheability
 583         break;
 584
 585     case AnyP::PROTO_GOPHER:
 586         if (!gopherCachable(this))
 587             return false;
 588         break;
 589
 590     case AnyP::PROTO_CACHE_OBJECT:
 591         return false;
 592
 593     //case AnyP::PROTO_FTP:
 594     default:
 595         break;
 596     }
 597
 598     return true;
 599 }
 600
 601 bool
 602 HttpRequest::conditional() const
 603 {
 604     return flags.ims ||
 605            header.has(HDR_IF_MATCH) ||
 606            header.has(HDR_IF_NONE_MATCH);
 607 }
 608
 609 void
 610 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
 611 {
 612     if (dns.wait >= 0) { // known delay
 613         if (dnsWait >= 0) // have recorded DNS wait before
 614             dnsWait += dns.wait;
 615         else
 616             dnsWait = dns.wait;
 617     }
 618 }
 619
 620 int64_t
 621 HttpRequest::getRangeOffsetLimit()
 622 {
 623     /* -2 is the starting value of rangeOffsetLimit.
 624      * If it is -2, that means we haven't checked it yet.
 625      *  Otherwise, return the current value */
 626     if (rangeOffsetLimit != -2)
 627         return rangeOffsetLimit;
 628
 629     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 630
 631     ACLFilledChecklist ch(NULL, this, NULL);
 632     ch.src_addr = client_addr;
 633     ch.my_addr =  my_addr;
 634
 635     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 636         /* if there is no ACL list or if the ACLs listed match use this limit value */
 637         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 638             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 639             rangeOffsetLimit = l->size; // may be -1
 640             break;
 641         }
 642     }
 643
 644     return rangeOffsetLimit;
 645 }
 646
 647 void
 648 HttpRequest::ignoreRange(const char *reason)
 649 {
 650     if (range) {
 651         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 652         delete range;
 653         range = NULL;
 654     }
 655     // Some callers also reset isRanged but it may not be safe for all callers:
 656     // isRanged is used to determine whether a weak ETag comparison is allowed,
 657     // and that check should not ignore the Range header if it was present.
 658     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 659 }
 660
 661 bool
 662 HttpRequest::canHandle1xx() const
 663 {
 664     // old clients do not support 1xx unless they sent Expect: 100-continue
 665     // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT)
 666     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(HDR_EXPECT))
 667         return false;
 668
 669     // others must support 1xx control messages
 670     return true;
 671 }
 672
 673 ConnStateData *
 674 HttpRequest::pinnedConnection()
 675 {
 676     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 677         return clientConnectionManager.get();
 678     return NULL;
 679 }
 680
 681 const char *
 682 HttpRequest::storeId()
 683 {
 684     if (store_id.size() != 0) {
 685         debugs(73, 3, "sent back store_id:" << store_id);
 686
 687         return store_id.termedBuf();
 688     }
 689     debugs(73, 3, "sent back canonicalUrl:" << urlCanonical(this) );
 690
 691     return urlCanonical(this);
 692 }
 693