src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "dns/LookupDetails.h"
  17 #include "Downloader.h"
  18 #include "err_detail_type.h"
  19 #include "globals.h"
  20 #include "gopher.h"
  21 #include "http.h"
  22 #include "http/one/RequestParser.h"
  23 #include "http/Stream.h"
  24 #include "HttpHdrCc.h"
  25 #include "HttpHeaderRange.h"
  26 #include "HttpRequest.h"
  27 #include "log/Config.h"
  28 #include "MemBuf.h"
  29 #include "sbuf/StringConvert.h"
  30 #include "SquidConfig.h"
  31 #include "Store.h"
  32 #include "URL.h"
  33
  34 #if USE_AUTH
  35 #include "auth/UserRequest.h"
  36 #endif
  37 #if ICAP_CLIENT
  38 #include "adaptation/icap/icap_log.h"
  39 #endif
  40
  41 HttpRequest::HttpRequest() :
  42     HttpMsg(hoRequest)
  43 {
  44     init();
  45 }
  46
  47 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  48     HttpMsg(hoRequest)
  49 {
  50     static unsigned int id = 1;
  51     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  52     init();
  53     initHTTP(aMethod, aProtocol, aUrlpath);
  54 }
  55
  56 HttpRequest::~HttpRequest()
  57 {
  58     clean();
  59     debugs(93,7, HERE << "destructed, this=" << this);
  60 }
  61
  62 void
  63 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  64 {
  65     method = aMethod;
  66     url.setScheme(aProtocol);
  67     url.path(aUrlpath);
  68 }
  69
  70 void
  71 HttpRequest::init()
  72 {
  73     method = Http::METHOD_NONE;
  74     url.clear();
  75 #if USE_AUTH
  76     auth_user_request = NULL;
  77 #endif
  78     memset(&flags, '\0', sizeof(flags));
  79     range = NULL;
  80     ims = -1;
  81     imslen = 0;
  82     lastmod = -1;
  83     client_addr.setEmpty();
  84     my_addr.setEmpty();
  85     body_pipe = NULL;
  86     // hier
  87     dnsWait = -1;
  88     errType = ERR_NONE;
  89     errDetail = ERR_DETAIL_NONE;
  90     peer_login = NULL;      // not allocated/deallocated by this class
  91     peer_domain = NULL;     // not allocated/deallocated by this class
  92     peer_host = NULL;
  93     vary_headers = SBuf();
  94     myportname = null_string;
  95     tag = null_string;
  96 #if USE_AUTH
  97     extacl_user = null_string;
  98     extacl_passwd = null_string;
  99 #endif
 100     extacl_log = null_string;
 101     extacl_message = null_string;
 102     pstate = psReadyToParseStartLine;
 103 #if FOLLOW_X_FORWARDED_FOR
 104     indirect_client_addr.setEmpty();
 105 #endif /* FOLLOW_X_FORWARDED_FOR */
 106 #if USE_ADAPTATION
 107     adaptHistory_ = NULL;
 108 #endif
 109 #if ICAP_CLIENT
 110     icapHistory_ = NULL;
 111 #endif
 112     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 113     forcedBodyContinuation = false;
 114 }
 115
 116 void
 117 HttpRequest::clean()
 118 {
 119     // we used to assert that the pipe is NULL, but now the request only
 120     // points to a pipe that is owned and initiated by another object.
 121     body_pipe = NULL;
 122 #if USE_AUTH
 123     auth_user_request = NULL;
 124 #endif
 125     vary_headers.clear();
 126     url.clear();
 127
 128     header.clean();
 129
 130     if (cache_control) {
 131         delete cache_control;
 132         cache_control = NULL;
 133     }
 134
 135     if (range) {
 136         delete range;
 137         range = NULL;
 138     }
 139
 140     myportname.clean();
 141
 142     notes = NULL;
 143
 144     tag.clean();
 145 #if USE_AUTH
 146     extacl_user.clean();
 147     extacl_passwd.clean();
 148 #endif
 149     extacl_log.clean();
 150
 151     extacl_message.clean();
 152
 153     etag.clean();
 154
 155 #if USE_ADAPTATION
 156     adaptHistory_ = NULL;
 157 #endif
 158 #if ICAP_CLIENT
 159     icapHistory_ = NULL;
 160 #endif
 161 }
 162
 163 void
 164 HttpRequest::reset()
 165 {
 166     clean();
 167     init();
 168 }
 169
 170 HttpRequest *
 171 HttpRequest::clone() const
 172 {
 173     HttpRequest *copy = new HttpRequest();
 174     copy->method = method;
 175     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 176     copy->header.append(&header);
 177     copy->hdrCacheInit();
 178     copy->hdr_sz = hdr_sz;
 179     copy->http_ver = http_ver;
 180     copy->pstate = pstate; // TODO: should we assert a specific state here?
 181     copy->body_pipe = body_pipe;
 182
 183     copy->url.setScheme(url.getScheme());
 184     copy->url.userInfo(url.userInfo());
 185     copy->url.host(url.host());
 186     copy->url.port(url.port());
 187     copy->url.path(url.path());
 188
 189     // range handled in hdrCacheInit()
 190     copy->ims = ims;
 191     copy->imslen = imslen;
 192     copy->hier = hier; // Is it safe to copy? Should we?
 193
 194     copy->errType = errType;
 195
 196     // XXX: what to do with copy->peer_login?
 197
 198     copy->lastmod = lastmod;
 199     copy->etag = etag;
 200     copy->vary_headers = vary_headers;
 201     // XXX: what to do with copy->peer_domain?
 202
 203     copy->tag = tag;
 204     copy->extacl_log = extacl_log;
 205     copy->extacl_message = extacl_message;
 206
 207     const bool inheritWorked = copy->inheritProperties(this);
 208     assert(inheritWorked);
 209
 210     return copy;
 211 }
 212
 213 bool
 214 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 215 {
 216     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 217     if (!aReq)
 218         return false;
 219
 220     client_addr = aReq->client_addr;
 221 #if FOLLOW_X_FORWARDED_FOR
 222     indirect_client_addr = aReq->indirect_client_addr;
 223 #endif
 224     my_addr = aReq->my_addr;
 225
 226     dnsWait = aReq->dnsWait;
 227
 228 #if USE_ADAPTATION
 229     adaptHistory_ = aReq->adaptHistory();
 230 #endif
 231 #if ICAP_CLIENT
 232     icapHistory_ = aReq->icapHistory();
 233 #endif
 234
 235     // This may be too conservative for the 204 No Content case
 236     // may eventually need cloneNullAdaptationImmune() for that.
 237     flags = aReq->flags.cloneAdaptationImmune();
 238
 239     errType = aReq->errType;
 240     errDetail = aReq->errDetail;
 241 #if USE_AUTH
 242     auth_user_request = aReq->auth_user_request;
 243     extacl_user = aReq->extacl_user;
 244     extacl_passwd = aReq->extacl_passwd;
 245 #endif
 246
 247     myportname = aReq->myportname;
 248
 249     forcedBodyContinuation = aReq->forcedBodyContinuation;
 250
 251     // main property is which connection the request was received on (if any)
 252     clientConnectionManager = aReq->clientConnectionManager;
 253
 254     downloader = aReq->downloader;
 255
 256     notes = aReq->notes;
 257
 258     sources = aReq->sources;
 259     return true;
 260 }
 261
 262 /**
 263  * Checks the first line of an HTTP request is valid
 264  * currently just checks the request method is present.
 265  *
 266  * NP: Other errors are left for detection later in the parse.
 267  */
 268 bool
 269 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 270 {
 271     // content is long enough to possibly hold a reply
 272     // 2 being magic size of a 1-byte request method plus space delimiter
 273     if (hdr_len < 2) {
 274         // this is ony a real error if the headers apparently complete.
 275         if (hdr_len > 0) {
 276             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 277             *error = Http::scInvalidHeader;
 278         }
 279         return false;
 280     }
 281
 282     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 283     HttpRequestMethod m;
 284     m.HttpRequestMethodXXX(buf);
 285     if (m == Http::METHOD_NONE) {
 286         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 287         *error = Http::scInvalidHeader;
 288         return false;
 289     }
 290
 291     return true;
 292 }
 293
 294 bool
 295 HttpRequest::parseFirstLine(const char *start, const char *end)
 296 {
 297     method.HttpRequestMethodXXX(start);
 298
 299     if (method == Http::METHOD_NONE)
 300         return false;
 301
 302     // XXX: performance regression, strcspn() over the method bytes a second time.
 303     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 304     const char *t = start + strcspn(start, w_space);
 305
 306     start = t + strspn(t, w_space); // skip w_space after method
 307
 308     const char *ver = findTrailingHTTPVersion(start, end);
 309
 310     if (ver) {
 311         end = ver - 1;
 312
 313         while (xisspace(*end)) // find prev non-space
 314             --end;
 315
 316         ++end;                 // back to space
 317
 318         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 319             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 320             return false;
 321         }
 322     } else {
 323         http_ver.major = 0;
 324         http_ver.minor = 9;
 325     }
 326
 327     if (end < start)   // missing URI
 328         return false;
 329
 330     char save = *end;
 331
 332     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 333
 334     HttpRequest *tmp = urlParse(method, (char *) start, this);
 335
 336     * (char *) end = save;
 337
 338     if (NULL == tmp)
 339         return false;
 340
 341     return true;
 342 }
 343
 344 /* swaps out request using httpRequestPack */
 345 void
 346 HttpRequest::swapOut(StoreEntry * e)
 347 {
 348     assert(e);
 349     e->buffer();
 350     pack(e);
 351     e->flush();
 352 }
 353
 354 /* packs request-line and headers, appends <crlf> terminator */
 355 void
 356 HttpRequest::pack(Packable * p) const
 357 {
 358     assert(p);
 359     /* pack request-line */
 360     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 361                SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
 362                http_ver.major, http_ver.minor);
 363     /* headers */
 364     header.packInto(p);
 365     /* trailer */
 366     p->append("\r\n", 2);
 367 }
 368
 369 /*
 370  * A wrapper for debugObj()
 371  */
 372 void
 373 httpRequestPack(void *obj, Packable *p)
 374 {
 375     HttpRequest *request = static_cast<HttpRequest*>(obj);
 376     request->pack(p);
 377 }
 378
 379 /* returns the length of request line + headers + crlf */
 380 int
 381 HttpRequest::prefixLen() const
 382 {
 383     return method.image().length() + 1 +
 384            url.path().length() + 1 +
 385            4 + 1 + 3 + 2 +
 386            header.len + 2;
 387 }
 388
 389 /* sync this routine when you update HttpRequest struct */
 390 void
 391 HttpRequest::hdrCacheInit()
 392 {
 393     HttpMsg::hdrCacheInit();
 394
 395     assert(!range);
 396     range = header.getRange();
 397 }
 398
 399 #if ICAP_CLIENT
 400 Adaptation::Icap::History::Pointer
 401 HttpRequest::icapHistory() const
 402 {
 403     if (!icapHistory_) {
 404         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 405             icapHistory_ = new Adaptation::Icap::History();
 406             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 407         }
 408     }
 409
 410     return icapHistory_;
 411 }
 412 #endif
 413
 414 #if USE_ADAPTATION
 415 Adaptation::History::Pointer
 416 HttpRequest::adaptHistory(bool createIfNone) const
 417 {
 418     if (!adaptHistory_ && createIfNone) {
 419         adaptHistory_ = new Adaptation::History();
 420         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 421     }
 422
 423     return adaptHistory_;
 424 }
 425
 426 Adaptation::History::Pointer
 427 HttpRequest::adaptLogHistory() const
 428 {
 429     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 430 }
 431
 432 void
 433 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 434 {
 435     if (!adaptHistory_) {
 436         adaptHistory_ = them.adaptHistory_; // may be nil
 437     } else {
 438         // check that histories did not diverge
 439         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 440     }
 441 }
 442
 443 #endif
 444
 445 bool
 446 HttpRequest::multipartRangeRequest() const
 447 {
 448     return (range && range->specs.size() > 1);
 449 }
 450
 451 bool
 452 HttpRequest::bodyNibbled() const
 453 {
 454     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 455 }
 456
 457 void
 458 HttpRequest::detailError(err_type aType, int aDetail)
 459 {
 460     if (errType || errDetail)
 461         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 462     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 463     // checking type and detail separately may cause inconsistency, but
 464     // may result in more details available if they only become available later
 465     if (!errType)
 466         errType = aType;
 467     if (!errDetail)
 468         errDetail = aDetail;
 469 }
 470
 471 void
 472 HttpRequest::clearError()
 473 {
 474     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 475     errType = ERR_NONE;
 476     errDetail = ERR_DETAIL_NONE;
 477 }
 478
 479 void
 480 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
 481 {
 482     const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
 483
 484     // form HTTP request-line
 485     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 486                SQUIDSBUFPRINT(method.image()),
 487                SQUIDSBUFPRINT(tmp),
 488                http_ver.major, http_ver.minor);
 489 }
 490
 491 /*
 492  * Indicate whether or not we would expect an entity-body
 493  * along with this request
 494  */
 495 bool
 496 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
 497 {
 498     bool expectBody = false;
 499
 500     /*
 501      * Note: Checks for message validity is in clientIsContentLengthValid().
 502      * this just checks if a entity-body is expected based on HTTP message syntax
 503      */
 504     if (header.chunked()) {
 505         expectBody = true;
 506         theSize = -1;
 507     } else if (content_length >= 0) {
 508         expectBody = true;
 509         theSize = content_length;
 510     } else {
 511         expectBody = false;
 512         // theSize undefined
 513     }
 514
 515     return expectBody;
 516 }
 517
 518 /*
 519  * Create a Request from a URL and METHOD.
 520  *
 521  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 522  * If the request cannot be created cleanly, NULL is returned
 523  */
 524 HttpRequest *
 525 HttpRequest::CreateFromUrl(char * url, const HttpRequestMethod& method)
 526 {
 527     return urlParse(method, url, NULL);
 528 }
 529
 530 /**
 531  * Are responses to this request possible cacheable ?
 532  * If false then no matter what the response must not be cached.
 533  */
 534 bool
 535 HttpRequest::maybeCacheable()
 536 {
 537     // Intercepted request with Host: header which cannot be trusted.
 538     // Because it failed verification, or someone bypassed the security tests
 539     // we cannot cache the reponse for sharing between clients.
 540     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 541     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 542         return false;
 543
 544     switch (url.getScheme()) {
 545     case AnyP::PROTO_HTTP:
 546     case AnyP::PROTO_HTTPS:
 547         if (!method.respMaybeCacheable())
 548             return false;
 549
 550         // XXX: this would seem the correct place to detect request cache-controls
 551         //      no-store, private and related which block cacheability
 552         break;
 553
 554     case AnyP::PROTO_GOPHER:
 555         if (!gopherCachable(this))
 556             return false;
 557         break;
 558
 559     case AnyP::PROTO_CACHE_OBJECT:
 560         return false;
 561
 562     //case AnyP::PROTO_FTP:
 563     default:
 564         break;
 565     }
 566
 567     return true;
 568 }
 569
 570 bool
 571 HttpRequest::conditional() const
 572 {
 573     return flags.ims ||
 574            header.has(Http::HdrType::IF_MATCH) ||
 575            header.has(Http::HdrType::IF_NONE_MATCH);
 576 }
 577
 578 void
 579 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
 580 {
 581     if (dns.wait >= 0) { // known delay
 582         if (dnsWait >= 0) // have recorded DNS wait before
 583             dnsWait += dns.wait;
 584         else
 585             dnsWait = dns.wait;
 586     }
 587 }
 588
 589 int64_t
 590 HttpRequest::getRangeOffsetLimit()
 591 {
 592     /* -2 is the starting value of rangeOffsetLimit.
 593      * If it is -2, that means we haven't checked it yet.
 594      *  Otherwise, return the current value */
 595     if (rangeOffsetLimit != -2)
 596         return rangeOffsetLimit;
 597
 598     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 599
 600     ACLFilledChecklist ch(NULL, this, NULL);
 601     ch.src_addr = client_addr;
 602     ch.my_addr =  my_addr;
 603
 604     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 605         /* if there is no ACL list or if the ACLs listed match use this limit value */
 606         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 607             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 608             rangeOffsetLimit = l->size; // may be -1
 609             break;
 610         }
 611     }
 612
 613     return rangeOffsetLimit;
 614 }
 615
 616 void
 617 HttpRequest::ignoreRange(const char *reason)
 618 {
 619     if (range) {
 620         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 621         delete range;
 622         range = NULL;
 623     }
 624     // Some callers also reset isRanged but it may not be safe for all callers:
 625     // isRanged is used to determine whether a weak ETag comparison is allowed,
 626     // and that check should not ignore the Range header if it was present.
 627     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 628 }
 629
 630 bool
 631 HttpRequest::canHandle1xx() const
 632 {
 633     // old clients do not support 1xx unless they sent Expect: 100-continue
 634     // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
 635     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
 636         return false;
 637
 638     // others must support 1xx control messages
 639     return true;
 640 }
 641
 642 ConnStateData *
 643 HttpRequest::pinnedConnection()
 644 {
 645     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 646         return clientConnectionManager.get();
 647     return NULL;
 648 }
 649
 650 const SBuf
 651 HttpRequest::storeId()
 652 {
 653     if (store_id.size() != 0) {
 654         debugs(73, 3, "sent back store_id: " << store_id);
 655         return StringToSBuf(store_id);
 656     }
 657     debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
 658     return effectiveRequestUri();
 659 }
 660
 661 const SBuf &
 662 HttpRequest::effectiveRequestUri() const
 663 {
 664     if (method.id() == Http::METHOD_CONNECT)
 665         return url.authority(true); // host:port
 666     return url.absolute();
 667 }
 668