src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "dns/LookupDetails.h"
  17 #include "Downloader.h"
  18 #include "err_detail_type.h"
  19 #include "globals.h"
  20 #include "gopher.h"
  21 #include "http.h"
  22 #include "http/one/RequestParser.h"
  23 #include "http/Stream.h"
  24 #include "HttpHdrCc.h"
  25 #include "HttpHeaderRange.h"
  26 #include "HttpRequest.h"
  27 #include "log/Config.h"
  28 #include "MemBuf.h"
  29 #include "sbuf/StringConvert.h"
  30 #include "SquidConfig.h"
  31 #include "Store.h"
  32 #include "URL.h"
  33
  34 #if USE_AUTH
  35 #include "auth/UserRequest.h"
  36 #endif
  37 #if ICAP_CLIENT
  38 #include "adaptation/icap/icap_log.h"
  39 #endif
  40
  41 HttpRequest::HttpRequest() :
  42     Http::Message(hoRequest)
  43 {
  44     init();
  45 }
  46
  47 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath) :
  48     Http::Message(hoRequest)
  49 {
  50     static unsigned int id = 1;
  51     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  52     init();
  53     initHTTP(aMethod, aProtocol, aSchemeImg, aUrlpath);
  54 }
  55
  56 HttpRequest::~HttpRequest()
  57 {
  58     clean();
  59     debugs(93,7, HERE << "destructed, this=" << this);
  60 }
  61
  62 void
  63 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aSchemeImg, const char *aUrlpath)
  64 {
  65     method = aMethod;
  66     url.setScheme(aProtocol, aSchemeImg);
  67     url.path(aUrlpath);
  68 }
  69
  70 void
  71 HttpRequest::init()
  72 {
  73     method = Http::METHOD_NONE;
  74     url.clear();
  75 #if USE_AUTH
  76     auth_user_request = NULL;
  77 #endif
  78     memset(&flags, '\0', sizeof(flags));
  79     range = NULL;
  80     ims = -1;
  81     imslen = 0;
  82     lastmod = -1;
  83     client_addr.setEmpty();
  84     my_addr.setEmpty();
  85     body_pipe = NULL;
  86     // hier
  87     dnsWait = -1;
  88     errType = ERR_NONE;
  89     errDetail = ERR_DETAIL_NONE;
  90     peer_login = NULL;      // not allocated/deallocated by this class
  91     peer_domain = NULL;     // not allocated/deallocated by this class
  92     peer_host = NULL;
  93     vary_headers = SBuf();
  94     myportname = null_string;
  95     tag = null_string;
  96 #if USE_AUTH
  97     extacl_user = null_string;
  98     extacl_passwd = null_string;
  99 #endif
 100     extacl_log = null_string;
 101     extacl_message = null_string;
 102     pstate = psReadyToParseStartLine;
 103 #if FOLLOW_X_FORWARDED_FOR
 104     indirect_client_addr.setEmpty();
 105 #endif /* FOLLOW_X_FORWARDED_FOR */
 106 #if USE_ADAPTATION
 107     adaptHistory_ = NULL;
 108 #endif
 109 #if ICAP_CLIENT
 110     icapHistory_ = NULL;
 111 #endif
 112     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 113     forcedBodyContinuation = false;
 114 }
 115
 116 void
 117 HttpRequest::clean()
 118 {
 119     // we used to assert that the pipe is NULL, but now the request only
 120     // points to a pipe that is owned and initiated by another object.
 121     body_pipe = NULL;
 122 #if USE_AUTH
 123     auth_user_request = NULL;
 124 #endif
 125     vary_headers.clear();
 126     url.clear();
 127
 128     header.clean();
 129
 130     if (cache_control) {
 131         delete cache_control;
 132         cache_control = NULL;
 133     }
 134
 135     if (range) {
 136         delete range;
 137         range = NULL;
 138     }
 139
 140     myportname.clean();
 141
 142     theNotes = nullptr;
 143
 144     tag.clean();
 145 #if USE_AUTH
 146     extacl_user.clean();
 147     extacl_passwd.clean();
 148 #endif
 149     extacl_log.clean();
 150
 151     extacl_message.clean();
 152
 153     etag.clean();
 154
 155 #if USE_ADAPTATION
 156     adaptHistory_ = NULL;
 157 #endif
 158 #if ICAP_CLIENT
 159     icapHistory_ = NULL;
 160 #endif
 161 }
 162
 163 void
 164 HttpRequest::reset()
 165 {
 166     clean();
 167     init();
 168 }
 169
 170 HttpRequest *
 171 HttpRequest::clone() const
 172 {
 173     HttpRequest *copy = new HttpRequest();
 174     copy->method = method;
 175     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 176     copy->header.append(&header);
 177     copy->hdrCacheInit();
 178     copy->hdr_sz = hdr_sz;
 179     copy->http_ver = http_ver;
 180     copy->pstate = pstate; // TODO: should we assert a specific state here?
 181     copy->body_pipe = body_pipe;
 182
 183     copy->url = url;
 184
 185     // range handled in hdrCacheInit()
 186     copy->ims = ims;
 187     copy->imslen = imslen;
 188     copy->hier = hier; // Is it safe to copy? Should we?
 189
 190     copy->errType = errType;
 191
 192     // XXX: what to do with copy->peer_login?
 193
 194     copy->lastmod = lastmod;
 195     copy->etag = etag;
 196     copy->vary_headers = vary_headers;
 197     // XXX: what to do with copy->peer_domain?
 198
 199     copy->tag = tag;
 200     copy->extacl_log = extacl_log;
 201     copy->extacl_message = extacl_message;
 202
 203     const bool inheritWorked = copy->inheritProperties(this);
 204     assert(inheritWorked);
 205
 206     return copy;
 207 }
 208
 209 bool
 210 HttpRequest::inheritProperties(const Http::Message *aMsg)
 211 {
 212     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 213     if (!aReq)
 214         return false;
 215
 216     client_addr = aReq->client_addr;
 217 #if FOLLOW_X_FORWARDED_FOR
 218     indirect_client_addr = aReq->indirect_client_addr;
 219 #endif
 220     my_addr = aReq->my_addr;
 221
 222     dnsWait = aReq->dnsWait;
 223
 224 #if USE_ADAPTATION
 225     adaptHistory_ = aReq->adaptHistory();
 226 #endif
 227 #if ICAP_CLIENT
 228     icapHistory_ = aReq->icapHistory();
 229 #endif
 230
 231     // This may be too conservative for the 204 No Content case
 232     // may eventually need cloneNullAdaptationImmune() for that.
 233     flags = aReq->flags.cloneAdaptationImmune();
 234
 235     errType = aReq->errType;
 236     errDetail = aReq->errDetail;
 237 #if USE_AUTH
 238     auth_user_request = aReq->auth_user_request;
 239     extacl_user = aReq->extacl_user;
 240     extacl_passwd = aReq->extacl_passwd;
 241 #endif
 242
 243     myportname = aReq->myportname;
 244
 245     forcedBodyContinuation = aReq->forcedBodyContinuation;
 246
 247     // main property is which connection the request was received on (if any)
 248     clientConnectionManager = aReq->clientConnectionManager;
 249
 250     downloader = aReq->downloader;
 251
 252     theNotes = aReq->theNotes;
 253
 254     sources = aReq->sources;
 255     return true;
 256 }
 257
 258 /**
 259  * Checks the first line of an HTTP request is valid
 260  * currently just checks the request method is present.
 261  *
 262  * NP: Other errors are left for detection later in the parse.
 263  */
 264 bool
 265 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 266 {
 267     // content is long enough to possibly hold a reply
 268     // 2 being magic size of a 1-byte request method plus space delimiter
 269     if (hdr_len < 2) {
 270         // this is ony a real error if the headers apparently complete.
 271         if (hdr_len > 0) {
 272             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 273             *error = Http::scInvalidHeader;
 274         }
 275         return false;
 276     }
 277
 278     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 279     HttpRequestMethod m;
 280     m.HttpRequestMethodXXX(buf);
 281     if (m == Http::METHOD_NONE) {
 282         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 283         *error = Http::scInvalidHeader;
 284         return false;
 285     }
 286
 287     return true;
 288 }
 289
 290 bool
 291 HttpRequest::parseFirstLine(const char *start, const char *end)
 292 {
 293     method.HttpRequestMethodXXX(start);
 294
 295     if (method == Http::METHOD_NONE)
 296         return false;
 297
 298     // XXX: performance regression, strcspn() over the method bytes a second time.
 299     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 300     const char *t = start + strcspn(start, w_space);
 301
 302     start = t + strspn(t, w_space); // skip w_space after method
 303
 304     const char *ver = findTrailingHTTPVersion(start, end);
 305
 306     if (ver) {
 307         end = ver - 1;
 308
 309         while (xisspace(*end)) // find prev non-space
 310             --end;
 311
 312         ++end;                 // back to space
 313
 314         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 315             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 316             return false;
 317         }
 318     } else {
 319         http_ver.major = 0;
 320         http_ver.minor = 9;
 321     }
 322
 323     if (end < start)   // missing URI
 324         return false;
 325
 326     char save = *end;
 327
 328     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 329
 330     HttpRequest *tmp = urlParse(method, (char *) start, this);
 331
 332     * (char *) end = save;
 333
 334     if (NULL == tmp)
 335         return false;
 336
 337     return true;
 338 }
 339
 340 /* swaps out request using httpRequestPack */
 341 void
 342 HttpRequest::swapOut(StoreEntry * e)
 343 {
 344     assert(e);
 345     e->buffer();
 346     pack(e);
 347     e->flush();
 348 }
 349
 350 /* packs request-line and headers, appends <crlf> terminator */
 351 void
 352 HttpRequest::pack(Packable * p) const
 353 {
 354     assert(p);
 355     /* pack request-line */
 356     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 357                SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
 358                http_ver.major, http_ver.minor);
 359     /* headers */
 360     header.packInto(p);
 361     /* trailer */
 362     p->append("\r\n", 2);
 363 }
 364
 365 /*
 366  * A wrapper for debugObj()
 367  */
 368 void
 369 httpRequestPack(void *obj, Packable *p)
 370 {
 371     HttpRequest *request = static_cast<HttpRequest*>(obj);
 372     request->pack(p);
 373 }
 374
 375 /* returns the length of request line + headers + crlf */
 376 int
 377 HttpRequest::prefixLen() const
 378 {
 379     return method.image().length() + 1 +
 380            url.path().length() + 1 +
 381            4 + 1 + 3 + 2 +
 382            header.len + 2;
 383 }
 384
 385 /* sync this routine when you update HttpRequest struct */
 386 void
 387 HttpRequest::hdrCacheInit()
 388 {
 389     Http::Message::hdrCacheInit();
 390
 391     assert(!range);
 392     range = header.getRange();
 393 }
 394
 395 #if ICAP_CLIENT
 396 Adaptation::Icap::History::Pointer
 397 HttpRequest::icapHistory() const
 398 {
 399     if (!icapHistory_) {
 400         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 401             icapHistory_ = new Adaptation::Icap::History();
 402             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 403         }
 404     }
 405
 406     return icapHistory_;
 407 }
 408 #endif
 409
 410 #if USE_ADAPTATION
 411 Adaptation::History::Pointer
 412 HttpRequest::adaptHistory(bool createIfNone) const
 413 {
 414     if (!adaptHistory_ && createIfNone) {
 415         adaptHistory_ = new Adaptation::History();
 416         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 417     }
 418
 419     return adaptHistory_;
 420 }
 421
 422 Adaptation::History::Pointer
 423 HttpRequest::adaptLogHistory() const
 424 {
 425     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 426 }
 427
 428 void
 429 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 430 {
 431     if (!adaptHistory_) {
 432         adaptHistory_ = them.adaptHistory_; // may be nil
 433     } else {
 434         // check that histories did not diverge
 435         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 436     }
 437 }
 438
 439 #endif
 440
 441 bool
 442 HttpRequest::multipartRangeRequest() const
 443 {
 444     return (range && range->specs.size() > 1);
 445 }
 446
 447 bool
 448 HttpRequest::bodyNibbled() const
 449 {
 450     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 451 }
 452
 453 void
 454 HttpRequest::detailError(err_type aType, int aDetail)
 455 {
 456     if (errType || errDetail)
 457         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 458     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 459     // checking type and detail separately may cause inconsistency, but
 460     // may result in more details available if they only become available later
 461     if (!errType)
 462         errType = aType;
 463     if (!errDetail)
 464         errDetail = aDetail;
 465 }
 466
 467 void
 468 HttpRequest::clearError()
 469 {
 470     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 471     errType = ERR_NONE;
 472     errDetail = ERR_DETAIL_NONE;
 473 }
 474
 475 void
 476 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
 477 {
 478     const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
 479
 480     // form HTTP request-line
 481     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 482                SQUIDSBUFPRINT(method.image()),
 483                SQUIDSBUFPRINT(tmp),
 484                http_ver.major, http_ver.minor);
 485 }
 486
 487 /*
 488  * Indicate whether or not we would expect an entity-body
 489  * along with this request
 490  */
 491 bool
 492 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
 493 {
 494     bool expectBody = false;
 495
 496     /*
 497      * Note: Checks for message validity is in clientIsContentLengthValid().
 498      * this just checks if a entity-body is expected based on HTTP message syntax
 499      */
 500     if (header.chunked()) {
 501         expectBody = true;
 502         theSize = -1;
 503     } else if (content_length >= 0) {
 504         expectBody = true;
 505         theSize = content_length;
 506     } else {
 507         expectBody = false;
 508         // theSize undefined
 509     }
 510
 511     return expectBody;
 512 }
 513
 514 /*
 515  * Create a Request from a URL and METHOD.
 516  *
 517  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 518  * If the request cannot be created cleanly, NULL is returned
 519  */
 520 HttpRequest *
 521 HttpRequest::CreateFromUrl(char * url, const HttpRequestMethod& method)
 522 {
 523     return urlParse(method, url, NULL);
 524 }
 525
 526 /**
 527  * Are responses to this request possible cacheable ?
 528  * If false then no matter what the response must not be cached.
 529  */
 530 bool
 531 HttpRequest::maybeCacheable()
 532 {
 533     // Intercepted request with Host: header which cannot be trusted.
 534     // Because it failed verification, or someone bypassed the security tests
 535     // we cannot cache the reponse for sharing between clients.
 536     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 537     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 538         return false;
 539
 540     switch (url.getScheme()) {
 541     case AnyP::PROTO_HTTP:
 542     case AnyP::PROTO_HTTPS:
 543         if (!method.respMaybeCacheable())
 544             return false;
 545
 546         // RFC 7234 section 5.2.1.5:
 547         // "cache MUST NOT store any part of either this request or any response to it"
 548         //
 549         // NP: refresh_pattern ignore-no-store only applies to response messages
 550         //     this test is handling request message CC header.
 551         if (!flags.ignoreCc && cache_control && cache_control->noStore())
 552             return false;
 553         break;
 554
 555     case AnyP::PROTO_GOPHER:
 556         if (!gopherCachable(this))
 557             return false;
 558         break;
 559
 560     case AnyP::PROTO_CACHE_OBJECT:
 561         return false;
 562
 563     //case AnyP::PROTO_FTP:
 564     default:
 565         break;
 566     }
 567
 568     return true;
 569 }
 570
 571 bool
 572 HttpRequest::conditional() const
 573 {
 574     return flags.ims ||
 575            header.has(Http::HdrType::IF_MATCH) ||
 576            header.has(Http::HdrType::IF_NONE_MATCH);
 577 }
 578
 579 void
 580 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
 581 {
 582     if (dns.wait >= 0) { // known delay
 583         if (dnsWait >= 0) // have recorded DNS wait before
 584             dnsWait += dns.wait;
 585         else
 586             dnsWait = dns.wait;
 587     }
 588 }
 589
 590 int64_t
 591 HttpRequest::getRangeOffsetLimit()
 592 {
 593     /* -2 is the starting value of rangeOffsetLimit.
 594      * If it is -2, that means we haven't checked it yet.
 595      *  Otherwise, return the current value */
 596     if (rangeOffsetLimit != -2)
 597         return rangeOffsetLimit;
 598
 599     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 600
 601     ACLFilledChecklist ch(NULL, this, NULL);
 602     ch.src_addr = client_addr;
 603     ch.my_addr =  my_addr;
 604
 605     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 606         /* if there is no ACL list or if the ACLs listed match use this limit value */
 607         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 608             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 609             rangeOffsetLimit = l->size; // may be -1
 610             break;
 611         }
 612     }
 613
 614     return rangeOffsetLimit;
 615 }
 616
 617 void
 618 HttpRequest::ignoreRange(const char *reason)
 619 {
 620     if (range) {
 621         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 622         delete range;
 623         range = NULL;
 624     }
 625     // Some callers also reset isRanged but it may not be safe for all callers:
 626     // isRanged is used to determine whether a weak ETag comparison is allowed,
 627     // and that check should not ignore the Range header if it was present.
 628     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 629 }
 630
 631 bool
 632 HttpRequest::canHandle1xx() const
 633 {
 634     // old clients do not support 1xx unless they sent Expect: 100-continue
 635     // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
 636     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
 637         return false;
 638
 639     // others must support 1xx control messages
 640     return true;
 641 }
 642
 643 ConnStateData *
 644 HttpRequest::pinnedConnection()
 645 {
 646     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 647         return clientConnectionManager.get();
 648     return NULL;
 649 }
 650
 651 const SBuf
 652 HttpRequest::storeId()
 653 {
 654     if (store_id.size() != 0) {
 655         debugs(73, 3, "sent back store_id: " << store_id);
 656         return StringToSBuf(store_id);
 657     }
 658     debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
 659     return effectiveRequestUri();
 660 }
 661
 662 const SBuf &
 663 HttpRequest::effectiveRequestUri() const
 664 {
 665     if (method.id() == Http::METHOD_CONNECT || url.getScheme() == AnyP::PROTO_AUTHORITY_FORM)
 666         return url.authority(true); // host:port
 667     return url.absolute();
 668 }
 669
 670 NotePairs::Pointer
 671 HttpRequest::notes()
 672 {
 673     if (!theNotes)
 674         theNotes = new NotePairs;
 675     return theNotes;
 676 }
 677
 678 void
 679 UpdateRequestNotes(ConnStateData *csd, HttpRequest &request, NotePairs const &helperNotes)
 680 {
 681     // Tag client connection if the helper responded with clt_conn_tag=tag.
 682     const char *cltTag = "clt_conn_tag";
 683     if (const char *connTag = helperNotes.findFirst(cltTag)) {
 684         if (csd) {
 685             csd->notes()->remove(cltTag);
 686             csd->notes()->add(cltTag, connTag);
 687         }
 688     }
 689     request.notes()->replaceOrAdd(&helperNotes);
 690 }
 691