src/HttpRequest.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 73    HTTP Request */
  10
  11 #include "squid.h"
  12 #include "AccessLogEntry.h"
  13 #include "acl/AclSizeLimit.h"
  14 #include "acl/FilledChecklist.h"
  15 #include "client_side.h"
  16 #include "dns/LookupDetails.h"
  17 #include "err_detail_type.h"
  18 #include "globals.h"
  19 #include "gopher.h"
  20 #include "http.h"
  21 #include "http/one/RequestParser.h"
  22 #include "http/Stream.h"
  23 #include "HttpHdrCc.h"
  24 #include "HttpHeaderRange.h"
  25 #include "HttpRequest.h"
  26 #include "log/Config.h"
  27 #include "MemBuf.h"
  28 #include "sbuf/StringConvert.h"
  29 #include "SquidConfig.h"
  30 #include "Store.h"
  31 #include "URL.h"
  32
  33 #if USE_AUTH
  34 #include "auth/UserRequest.h"
  35 #endif
  36 #if ICAP_CLIENT
  37 #include "adaptation/icap/icap_log.h"
  38 #endif
  39
  40 HttpRequest::HttpRequest() :
  41     HttpMsg(hoRequest)
  42 {
  43     init();
  44 }
  45
  46 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  47     HttpMsg(hoRequest)
  48 {
  49     static unsigned int id = 1;
  50     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  51     init();
  52     initHTTP(aMethod, aProtocol, aUrlpath);
  53 }
  54
  55 HttpRequest::~HttpRequest()
  56 {
  57     clean();
  58     debugs(93,7, HERE << "destructed, this=" << this);
  59 }
  60
  61 void
  62 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  63 {
  64     method = aMethod;
  65     url.setScheme(aProtocol);
  66     url.path(aUrlpath);
  67 }
  68
  69 void
  70 HttpRequest::init()
  71 {
  72     method = Http::METHOD_NONE;
  73     url.clear();
  74 #if USE_AUTH
  75     auth_user_request = NULL;
  76 #endif
  77     memset(&flags, '\0', sizeof(flags));
  78     range = NULL;
  79     ims = -1;
  80     imslen = 0;
  81     lastmod = -1;
  82     client_addr.setEmpty();
  83     my_addr.setEmpty();
  84     body_pipe = NULL;
  85     // hier
  86     dnsWait = -1;
  87     errType = ERR_NONE;
  88     errDetail = ERR_DETAIL_NONE;
  89     peer_login = NULL;      // not allocated/deallocated by this class
  90     peer_domain = NULL;     // not allocated/deallocated by this class
  91     peer_host = NULL;
  92     vary_headers = SBuf();
  93     myportname = null_string;
  94     tag = null_string;
  95 #if USE_AUTH
  96     extacl_user = null_string;
  97     extacl_passwd = null_string;
  98 #endif
  99     extacl_log = null_string;
 100     extacl_message = null_string;
 101     pstate = psReadyToParseStartLine;
 102 #if FOLLOW_X_FORWARDED_FOR
 103     indirect_client_addr.setEmpty();
 104 #endif /* FOLLOW_X_FORWARDED_FOR */
 105 #if USE_ADAPTATION
 106     adaptHistory_ = NULL;
 107 #endif
 108 #if ICAP_CLIENT
 109     icapHistory_ = NULL;
 110 #endif
 111     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 112     forcedBodyContinuation = false;
 113 }
 114
 115 void
 116 HttpRequest::clean()
 117 {
 118     // we used to assert that the pipe is NULL, but now the request only
 119     // points to a pipe that is owned and initiated by another object.
 120     body_pipe = NULL;
 121 #if USE_AUTH
 122     auth_user_request = NULL;
 123 #endif
 124     vary_headers.clear();
 125     url.clear();
 126
 127     header.clean();
 128
 129     if (cache_control) {
 130         delete cache_control;
 131         cache_control = NULL;
 132     }
 133
 134     if (range) {
 135         delete range;
 136         range = NULL;
 137     }
 138
 139     myportname.clean();
 140
 141     notes = NULL;
 142
 143     tag.clean();
 144 #if USE_AUTH
 145     extacl_user.clean();
 146     extacl_passwd.clean();
 147 #endif
 148     extacl_log.clean();
 149
 150     extacl_message.clean();
 151
 152     etag.clean();
 153
 154 #if USE_ADAPTATION
 155     adaptHistory_ = NULL;
 156 #endif
 157 #if ICAP_CLIENT
 158     icapHistory_ = NULL;
 159 #endif
 160 }
 161
 162 void
 163 HttpRequest::reset()
 164 {
 165     clean();
 166     init();
 167 }
 168
 169 HttpRequest *
 170 HttpRequest::clone() const
 171 {
 172     HttpRequest *copy = new HttpRequest();
 173     copy->method = method;
 174     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 175     copy->header.append(&header);
 176     copy->hdrCacheInit();
 177     copy->hdr_sz = hdr_sz;
 178     copy->http_ver = http_ver;
 179     copy->pstate = pstate; // TODO: should we assert a specific state here?
 180     copy->body_pipe = body_pipe;
 181
 182     copy->url.setScheme(url.getScheme());
 183     copy->url.userInfo(url.userInfo());
 184     copy->url.host(url.host());
 185     copy->url.port(url.port());
 186     copy->url.path(url.path());
 187
 188     // range handled in hdrCacheInit()
 189     copy->ims = ims;
 190     copy->imslen = imslen;
 191     copy->hier = hier; // Is it safe to copy? Should we?
 192
 193     copy->errType = errType;
 194
 195     // XXX: what to do with copy->peer_login?
 196
 197     copy->lastmod = lastmod;
 198     copy->etag = etag;
 199     copy->vary_headers = vary_headers;
 200     // XXX: what to do with copy->peer_domain?
 201
 202     copy->tag = tag;
 203     copy->extacl_log = extacl_log;
 204     copy->extacl_message = extacl_message;
 205
 206     const bool inheritWorked = copy->inheritProperties(this);
 207     assert(inheritWorked);
 208
 209     return copy;
 210 }
 211
 212 bool
 213 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 214 {
 215     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 216     if (!aReq)
 217         return false;
 218
 219     client_addr = aReq->client_addr;
 220 #if FOLLOW_X_FORWARDED_FOR
 221     indirect_client_addr = aReq->indirect_client_addr;
 222 #endif
 223     my_addr = aReq->my_addr;
 224
 225     dnsWait = aReq->dnsWait;
 226
 227 #if USE_ADAPTATION
 228     adaptHistory_ = aReq->adaptHistory();
 229 #endif
 230 #if ICAP_CLIENT
 231     icapHistory_ = aReq->icapHistory();
 232 #endif
 233
 234     // This may be too conservative for the 204 No Content case
 235     // may eventually need cloneNullAdaptationImmune() for that.
 236     flags = aReq->flags.cloneAdaptationImmune();
 237
 238     errType = aReq->errType;
 239     errDetail = aReq->errDetail;
 240 #if USE_AUTH
 241     auth_user_request = aReq->auth_user_request;
 242     extacl_user = aReq->extacl_user;
 243     extacl_passwd = aReq->extacl_passwd;
 244 #endif
 245
 246     myportname = aReq->myportname;
 247
 248     forcedBodyContinuation = aReq->forcedBodyContinuation;
 249
 250     // main property is which connection the request was received on (if any)
 251     clientConnectionManager = aReq->clientConnectionManager;
 252
 253     notes = aReq->notes;
 254
 255     sources = aReq->sources;
 256     return true;
 257 }
 258
 259 /**
 260  * Checks the first line of an HTTP request is valid
 261  * currently just checks the request method is present.
 262  *
 263  * NP: Other errors are left for detection later in the parse.
 264  */
 265 bool
 266 HttpRequest::sanityCheckStartLine(const char *buf, const size_t hdr_len, Http::StatusCode *error)
 267 {
 268     // content is long enough to possibly hold a reply
 269     // 2 being magic size of a 1-byte request method plus space delimiter
 270     if (hdr_len < 2) {
 271         // this is ony a real error if the headers apparently complete.
 272         if (hdr_len > 0) {
 273             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 274             *error = Http::scInvalidHeader;
 275         }
 276         return false;
 277     }
 278
 279     /* See if the request buffer starts with a non-whitespace HTTP request 'method'. */
 280     HttpRequestMethod m;
 281     m.HttpRequestMethodXXX(buf);
 282     if (m == Http::METHOD_NONE) {
 283         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 284         *error = Http::scInvalidHeader;
 285         return false;
 286     }
 287
 288     return true;
 289 }
 290
 291 bool
 292 HttpRequest::parseFirstLine(const char *start, const char *end)
 293 {
 294     method.HttpRequestMethodXXX(start);
 295
 296     if (method == Http::METHOD_NONE)
 297         return false;
 298
 299     // XXX: performance regression, strcspn() over the method bytes a second time.
 300     // cheaper than allocate+copy+deallocate cycle to SBuf convert a piece of start.
 301     const char *t = start + strcspn(start, w_space);
 302
 303     start = t + strspn(t, w_space); // skip w_space after method
 304
 305     const char *ver = findTrailingHTTPVersion(start, end);
 306
 307     if (ver) {
 308         end = ver - 1;
 309
 310         while (xisspace(*end)) // find prev non-space
 311             --end;
 312
 313         ++end;                 // back to space
 314
 315         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 316             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 317             return false;
 318         }
 319     } else {
 320         http_ver.major = 0;
 321         http_ver.minor = 9;
 322     }
 323
 324     if (end < start)   // missing URI
 325         return false;
 326
 327     char save = *end;
 328
 329     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 330
 331     HttpRequest *tmp = urlParse(method, (char *) start, this);
 332
 333     * (char *) end = save;
 334
 335     if (NULL == tmp)
 336         return false;
 337
 338     return true;
 339 }
 340
 341 /* swaps out request using httpRequestPack */
 342 void
 343 HttpRequest::swapOut(StoreEntry * e)
 344 {
 345     assert(e);
 346     e->buffer();
 347     pack(e);
 348     e->flush();
 349 }
 350
 351 /* packs request-line and headers, appends <crlf> terminator */
 352 void
 353 HttpRequest::pack(Packable * p) const
 354 {
 355     assert(p);
 356     /* pack request-line */
 357     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 358                SQUIDSBUFPRINT(method.image()), SQUIDSBUFPRINT(url.path()),
 359                http_ver.major, http_ver.minor);
 360     /* headers */
 361     header.packInto(p);
 362     /* trailer */
 363     p->append("\r\n", 2);
 364 }
 365
 366 /*
 367  * A wrapper for debugObj()
 368  */
 369 void
 370 httpRequestPack(void *obj, Packable *p)
 371 {
 372     HttpRequest *request = static_cast<HttpRequest*>(obj);
 373     request->pack(p);
 374 }
 375
 376 /* returns the length of request line + headers + crlf */
 377 int
 378 HttpRequest::prefixLen() const
 379 {
 380     return method.image().length() + 1 +
 381            url.path().length() + 1 +
 382            4 + 1 + 3 + 2 +
 383            header.len + 2;
 384 }
 385
 386 /* sync this routine when you update HttpRequest struct */
 387 void
 388 HttpRequest::hdrCacheInit()
 389 {
 390     HttpMsg::hdrCacheInit();
 391
 392     assert(!range);
 393     range = header.getRange();
 394 }
 395
 396 #if ICAP_CLIENT
 397 Adaptation::Icap::History::Pointer
 398 HttpRequest::icapHistory() const
 399 {
 400     if (!icapHistory_) {
 401         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 402             icapHistory_ = new Adaptation::Icap::History();
 403             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 404         }
 405     }
 406
 407     return icapHistory_;
 408 }
 409 #endif
 410
 411 #if USE_ADAPTATION
 412 Adaptation::History::Pointer
 413 HttpRequest::adaptHistory(bool createIfNone) const
 414 {
 415     if (!adaptHistory_ && createIfNone) {
 416         adaptHistory_ = new Adaptation::History();
 417         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 418     }
 419
 420     return adaptHistory_;
 421 }
 422
 423 Adaptation::History::Pointer
 424 HttpRequest::adaptLogHistory() const
 425 {
 426     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 427 }
 428
 429 void
 430 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 431 {
 432     if (!adaptHistory_) {
 433         adaptHistory_ = them.adaptHistory_; // may be nil
 434     } else {
 435         // check that histories did not diverge
 436         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 437     }
 438 }
 439
 440 #endif
 441
 442 bool
 443 HttpRequest::multipartRangeRequest() const
 444 {
 445     return (range && range->specs.size() > 1);
 446 }
 447
 448 bool
 449 HttpRequest::bodyNibbled() const
 450 {
 451     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 452 }
 453
 454 void
 455 HttpRequest::detailError(err_type aType, int aDetail)
 456 {
 457     if (errType || errDetail)
 458         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 459     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 460     // checking type and detail separately may cause inconsistency, but
 461     // may result in more details available if they only become available later
 462     if (!errType)
 463         errType = aType;
 464     if (!errDetail)
 465         errDetail = aDetail;
 466 }
 467
 468 void
 469 HttpRequest::clearError()
 470 {
 471     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 472     errType = ERR_NONE;
 473     errDetail = ERR_DETAIL_NONE;
 474 }
 475
 476 void
 477 HttpRequest::packFirstLineInto(Packable * p, bool full_uri) const
 478 {
 479     const SBuf tmp(full_uri ? effectiveRequestUri() : url.path());
 480
 481     // form HTTP request-line
 482     p->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " HTTP/%d.%d\r\n",
 483                SQUIDSBUFPRINT(method.image()),
 484                SQUIDSBUFPRINT(tmp),
 485                http_ver.major, http_ver.minor);
 486 }
 487
 488 /*
 489  * Indicate whether or not we would expect an entity-body
 490  * along with this request
 491  */
 492 bool
 493 HttpRequest::expectingBody(const HttpRequestMethod &, int64_t &theSize) const
 494 {
 495     bool expectBody = false;
 496
 497     /*
 498      * Note: Checks for message validity is in clientIsContentLengthValid().
 499      * this just checks if a entity-body is expected based on HTTP message syntax
 500      */
 501     if (header.chunked()) {
 502         expectBody = true;
 503         theSize = -1;
 504     } else if (content_length >= 0) {
 505         expectBody = true;
 506         theSize = content_length;
 507     } else {
 508         expectBody = false;
 509         // theSize undefined
 510     }
 511
 512     return expectBody;
 513 }
 514
 515 /*
 516  * Create a Request from a URL and METHOD.
 517  *
 518  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 519  * If the request cannot be created cleanly, NULL is returned
 520  */
 521 HttpRequest *
 522 HttpRequest::CreateFromUrl(char * url, const HttpRequestMethod& method)
 523 {
 524     return urlParse(method, url, NULL);
 525 }
 526
 527 /**
 528  * Are responses to this request possible cacheable ?
 529  * If false then no matter what the response must not be cached.
 530  */
 531 bool
 532 HttpRequest::maybeCacheable()
 533 {
 534     // Intercepted request with Host: header which cannot be trusted.
 535     // Because it failed verification, or someone bypassed the security tests
 536     // we cannot cache the reponse for sharing between clients.
 537     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 538     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 539         return false;
 540
 541     switch (url.getScheme()) {
 542     case AnyP::PROTO_HTTP:
 543     case AnyP::PROTO_HTTPS:
 544         if (!method.respMaybeCacheable())
 545             return false;
 546
 547         // XXX: this would seem the correct place to detect request cache-controls
 548         //      no-store, private and related which block cacheability
 549         break;
 550
 551     case AnyP::PROTO_GOPHER:
 552         if (!gopherCachable(this))
 553             return false;
 554         break;
 555
 556     case AnyP::PROTO_CACHE_OBJECT:
 557         return false;
 558
 559     //case AnyP::PROTO_FTP:
 560     default:
 561         break;
 562     }
 563
 564     return true;
 565 }
 566
 567 bool
 568 HttpRequest::conditional() const
 569 {
 570     return flags.ims ||
 571            header.has(Http::HdrType::IF_MATCH) ||
 572            header.has(Http::HdrType::IF_NONE_MATCH);
 573 }
 574
 575 void
 576 HttpRequest::recordLookup(const Dns::LookupDetails &dns)
 577 {
 578     if (dns.wait >= 0) { // known delay
 579         if (dnsWait >= 0) // have recorded DNS wait before
 580             dnsWait += dns.wait;
 581         else
 582             dnsWait = dns.wait;
 583     }
 584 }
 585
 586 int64_t
 587 HttpRequest::getRangeOffsetLimit()
 588 {
 589     /* -2 is the starting value of rangeOffsetLimit.
 590      * If it is -2, that means we haven't checked it yet.
 591      *  Otherwise, return the current value */
 592     if (rangeOffsetLimit != -2)
 593         return rangeOffsetLimit;
 594
 595     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 596
 597     ACLFilledChecklist ch(NULL, this, NULL);
 598     ch.src_addr = client_addr;
 599     ch.my_addr =  my_addr;
 600
 601     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 602         /* if there is no ACL list or if the ACLs listed match use this limit value */
 603         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 604             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 605             rangeOffsetLimit = l->size; // may be -1
 606             break;
 607         }
 608     }
 609
 610     return rangeOffsetLimit;
 611 }
 612
 613 void
 614 HttpRequest::ignoreRange(const char *reason)
 615 {
 616     if (range) {
 617         debugs(73, 3, static_cast<void*>(range) << " for " << reason);
 618         delete range;
 619         range = NULL;
 620     }
 621     // Some callers also reset isRanged but it may not be safe for all callers:
 622     // isRanged is used to determine whether a weak ETag comparison is allowed,
 623     // and that check should not ignore the Range header if it was present.
 624     // TODO: Some callers also delete HDR_RANGE, HDR_REQUEST_RANGE. Should we?
 625 }
 626
 627 bool
 628 HttpRequest::canHandle1xx() const
 629 {
 630     // old clients do not support 1xx unless they sent Expect: 100-continue
 631     // (we reject all other Http::HdrType::EXPECT values so just check for Http::HdrType::EXPECT)
 632     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(Http::HdrType::EXPECT))
 633         return false;
 634
 635     // others must support 1xx control messages
 636     return true;
 637 }
 638
 639 ConnStateData *
 640 HttpRequest::pinnedConnection()
 641 {
 642     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 643         return clientConnectionManager.get();
 644     return NULL;
 645 }
 646
 647 const SBuf
 648 HttpRequest::storeId()
 649 {
 650     if (store_id.size() != 0) {
 651         debugs(73, 3, "sent back store_id: " << store_id);
 652         return StringToSBuf(store_id);
 653     }
 654     debugs(73, 3, "sent back effectiveRequestUrl: " << effectiveRequestUri());
 655     return effectiveRequestUri();
 656 }
 657
 658 const SBuf &
 659 HttpRequest::effectiveRequestUri() const
 660 {
 661     if (method.id() == Http::METHOD_CONNECT)
 662         return url.authority(true); // host:port
 663     return url.absolute();
 664 }
 665