src/HttpRequest.cc

   1 /*
   2  * DEBUG: section 73    HTTP Request
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  * Copyright (c) 2003, Robert Collins <robertc@squid-cache.org>
  32  */
  33
  34 #include "squid.h"
  35 #include "AccessLogEntry.h"
  36 #include "acl/AclSizeLimit.h"
  37 #include "acl/FilledChecklist.h"
  38 #include "client_side.h"
  39 #include "DnsLookupDetails.h"
  40 #include "err_detail_type.h"
  41 #include "globals.h"
  42 #include "gopher.h"
  43 #include "http.h"
  44 #include "http/Http1Parser.h"
  45 #include "HttpHdrCc.h"
  46 #include "HttpHeaderRange.h"
  47 #include "HttpRequest.h"
  48 #include "log/Config.h"
  49 #include "MemBuf.h"
  50 #include "SquidConfig.h"
  51 #include "Store.h"
  52 #include "URL.h"
  53
  54 #if USE_AUTH
  55 #include "auth/UserRequest.h"
  56 #endif
  57 #if ICAP_CLIENT
  58 #include "adaptation/icap/icap_log.h"
  59 #endif
  60
  61 HttpRequest::HttpRequest() :
  62         HttpMsg(hoRequest)
  63 {
  64     init();
  65 }
  66
  67 HttpRequest::HttpRequest(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath) :
  68         HttpMsg(hoRequest)
  69 {
  70     static unsigned int id = 1;
  71     debugs(93,7, HERE << "constructed, this=" << this << " id=" << ++id);
  72     init();
  73     initHTTP(aMethod, aProtocol, aUrlpath);
  74 }
  75
  76 HttpRequest::~HttpRequest()
  77 {
  78     clean();
  79     debugs(93,7, HERE << "destructed, this=" << this);
  80 }
  81
  82 void
  83 HttpRequest::initHTTP(const HttpRequestMethod& aMethod, AnyP::ProtocolType aProtocol, const char *aUrlpath)
  84 {
  85     method = aMethod;
  86     protocol = aProtocol;
  87     urlpath = aUrlpath;
  88 }
  89
  90 void
  91 HttpRequest::init()
  92 {
  93     method = Http::METHOD_NONE;
  94     protocol = AnyP::PROTO_NONE;
  95     urlpath = NULL;
  96     login[0] = '\0';
  97     host[0] = '\0';
  98     host_is_numeric = -1;
  99 #if USE_AUTH
 100     auth_user_request = NULL;
 101 #endif
 102     port = 0;
 103     canonical = NULL;
 104     memset(&flags, '\0', sizeof(flags));
 105     range = NULL;
 106     ims = -1;
 107     imslen = 0;
 108     lastmod = -1;
 109     client_addr.setEmpty();
 110     my_addr.setEmpty();
 111     body_pipe = NULL;
 112     // hier
 113     dnsWait = -1;
 114     errType = ERR_NONE;
 115     errDetail = ERR_DETAIL_NONE;
 116     peer_login = NULL;          // not allocated/deallocated by this class
 117     peer_domain = NULL;         // not allocated/deallocated by this class
 118     peer_host = NULL;
 119     vary_headers = NULL;
 120     myportname = null_string;
 121     tag = null_string;
 122 #if USE_AUTH
 123     extacl_user = null_string;
 124     extacl_passwd = null_string;
 125 #endif
 126     extacl_log = null_string;
 127     extacl_message = null_string;
 128     pstate = psReadyToParseStartLine;
 129 #if FOLLOW_X_FORWARDED_FOR
 130     indirect_client_addr.setEmpty();
 131 #endif /* FOLLOW_X_FORWARDED_FOR */
 132 #if USE_ADAPTATION
 133     adaptHistory_ = NULL;
 134 #endif
 135 #if ICAP_CLIENT
 136     icapHistory_ = NULL;
 137 #endif
 138     rangeOffsetLimit = -2; //a value of -2 means not checked yet
 139 }
 140
 141 void
 142 HttpRequest::clean()
 143 {
 144     // we used to assert that the pipe is NULL, but now the request only
 145     // points to a pipe that is owned and initiated by another object.
 146     body_pipe = NULL;
 147 #if USE_AUTH
 148     auth_user_request = NULL;
 149 #endif
 150     safe_free(canonical);
 151
 152     safe_free(vary_headers);
 153
 154     urlpath.clean();
 155
 156     header.clean();
 157
 158     if (cache_control) {
 159         delete cache_control;
 160         cache_control = NULL;
 161     }
 162
 163     if (range) {
 164         delete range;
 165         range = NULL;
 166     }
 167
 168     myportname.clean();
 169
 170     notes = NULL;
 171
 172     tag.clean();
 173 #if USE_AUTH
 174     extacl_user.clean();
 175     extacl_passwd.clean();
 176 #endif
 177     extacl_log.clean();
 178
 179     extacl_message.clean();
 180
 181     etag.clean();
 182
 183 #if USE_ADAPTATION
 184     adaptHistory_ = NULL;
 185 #endif
 186 #if ICAP_CLIENT
 187     icapHistory_ = NULL;
 188 #endif
 189 }
 190
 191 void
 192 HttpRequest::reset()
 193 {
 194     clean();
 195     init();
 196 }
 197
 198 HttpRequest *
 199 HttpRequest::clone() const
 200 {
 201     HttpRequest *copy = new HttpRequest(method, protocol, urlpath.termedBuf());
 202     // TODO: move common cloning clone to Msg::copyTo() or copy ctor
 203     copy->header.append(&header);
 204     copy->hdrCacheInit();
 205     copy->hdr_sz = hdr_sz;
 206     copy->http_ver = http_ver;
 207     copy->pstate = pstate; // TODO: should we assert a specific state here?
 208     copy->body_pipe = body_pipe;
 209
 210     strncpy(copy->login, login, sizeof(login)); // MAX_LOGIN_SZ
 211     strncpy(copy->host, host, sizeof(host)); // SQUIDHOSTNAMELEN
 212     copy->host_addr = host_addr;
 213
 214     copy->port = port;
 215     // urlPath handled in ctor
 216     copy->canonical = canonical ? xstrdup(canonical) : NULL;
 217
 218     // range handled in hdrCacheInit()
 219     copy->ims = ims;
 220     copy->imslen = imslen;
 221     copy->hier = hier; // Is it safe to copy? Should we?
 222
 223     copy->errType = errType;
 224
 225     // XXX: what to do with copy->peer_login?
 226
 227     copy->lastmod = lastmod;
 228     copy->etag = etag;
 229     copy->vary_headers = vary_headers ? xstrdup(vary_headers) : NULL;
 230     // XXX: what to do with copy->peer_domain?
 231
 232     copy->tag = tag;
 233     copy->extacl_log = extacl_log;
 234     copy->extacl_message = extacl_message;
 235
 236     const bool inheritWorked = copy->inheritProperties(this);
 237     assert(inheritWorked);
 238
 239     return copy;
 240 }
 241
 242 bool
 243 HttpRequest::inheritProperties(const HttpMsg *aMsg)
 244 {
 245     const HttpRequest* aReq = dynamic_cast<const HttpRequest*>(aMsg);
 246     if (!aReq)
 247         return false;
 248
 249     client_addr = aReq->client_addr;
 250 #if FOLLOW_X_FORWARDED_FOR
 251     indirect_client_addr = aReq->indirect_client_addr;
 252 #endif
 253     my_addr = aReq->my_addr;
 254
 255     dnsWait = aReq->dnsWait;
 256
 257 #if USE_ADAPTATION
 258     adaptHistory_ = aReq->adaptHistory();
 259 #endif
 260 #if ICAP_CLIENT
 261     icapHistory_ = aReq->icapHistory();
 262 #endif
 263
 264     // This may be too conservative for the 204 No Content case
 265     // may eventually need cloneNullAdaptationImmune() for that.
 266     flags = aReq->flags.cloneAdaptationImmune();
 267
 268     errType = aReq->errType;
 269     errDetail = aReq->errDetail;
 270 #if USE_AUTH
 271     auth_user_request = aReq->auth_user_request;
 272     extacl_user = aReq->extacl_user;
 273     extacl_passwd = aReq->extacl_passwd;
 274 #endif
 275
 276     myportname = aReq->myportname;
 277
 278     // main property is which connection the request was received on (if any)
 279     clientConnectionManager = aReq->clientConnectionManager;
 280
 281     notes = aReq->notes;
 282     return true;
 283 }
 284
 285 /**
 286  * Checks the first line of an HTTP request is valid
 287  * currently just checks the request method is present.
 288  *
 289  * NP: Other errors are left for detection later in the parse.
 290  */
 291 bool
 292 HttpRequest::sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, Http::StatusCode *error)
 293 {
 294     // content is long enough to possibly hold a reply
 295     // 2 being magic size of a 1-byte request method plus space delimiter
 296     if ( buf->contentSize() < 2 ) {
 297         // this is ony a real error if the headers apparently complete.
 298         if (hdr_len > 0) {
 299             debugs(58, 3, HERE << "Too large request header (" << hdr_len << " bytes)");
 300             *error = Http::scInvalidHeader;
 301         }
 302         return false;
 303     }
 304
 305     /* See if the request buffer starts with a known HTTP request method. */
 306     if (HttpRequestMethod(buf->content(),NULL) == Http::METHOD_NONE) {
 307         debugs(73, 3, "HttpRequest::sanityCheckStartLine: did not find HTTP request method");
 308         *error = Http::scInvalidHeader;
 309         return false;
 310     }
 311
 312     return true;
 313 }
 314
 315 bool
 316 HttpRequest::parseFirstLine(const char *start, const char *end)
 317 {
 318     const char *t = start + strcspn(start, w_space);
 319     method = HttpRequestMethod(start, t);
 320
 321     if (method == Http::METHOD_NONE)
 322         return false;
 323
 324     start = t + strspn(t, w_space);
 325
 326     const char *ver = findTrailingHTTPVersion(start, end);
 327
 328     if (ver) {
 329         end = ver - 1;
 330
 331         while (xisspace(*end)) // find prev non-space
 332             --end;
 333
 334         ++end;                 // back to space
 335
 336         if (2 != sscanf(ver + 5, "%d.%d", &http_ver.major, &http_ver.minor)) {
 337             debugs(73, DBG_IMPORTANT, "parseRequestLine: Invalid HTTP identifier.");
 338             return false;
 339         }
 340     } else {
 341         http_ver.major = 0;
 342         http_ver.minor = 9;
 343     }
 344
 345     if (end < start)   // missing URI
 346         return false;
 347
 348     char save = *end;
 349
 350     * (char *) end = '\0';     // temp terminate URI, XXX dangerous?
 351
 352     HttpRequest *tmp = urlParse(method, (char *) start, this);
 353
 354     * (char *) end = save;
 355
 356     if (NULL == tmp)
 357         return false;
 358
 359     return true;
 360 }
 361
 362 bool
 363 HttpRequest::parseHeader(Http1::RequestParser &hp)
 364 {
 365     if (!hp.headerBlockSize())
 366         return false;
 367
 368     bool result = header.parse(hp.rawHeaderBuf(), hp.headerBlockSize());
 369
 370     if (result)
 371         hdrCacheInit();
 372
 373     return result;
 374 }
 375
 376 /* swaps out request using httpRequestPack */
 377 void
 378 HttpRequest::swapOut(StoreEntry * e)
 379 {
 380     Packer p;
 381     assert(e);
 382     packerToStoreInit(&p, e);
 383     pack(&p);
 384     packerClean(&p);
 385 }
 386
 387 /* packs request-line and headers, appends <crlf> terminator */
 388 void
 389 HttpRequest::pack(Packer * p)
 390 {
 391     assert(p);
 392     /* pack request-line */
 393     packerPrintf(p, "%s " SQUIDSTRINGPH " HTTP/%d.%d\r\n",
 394                  RequestMethodStr(method), SQUIDSTRINGPRINT(urlpath),
 395                  http_ver.major, http_ver.minor);
 396     /* headers */
 397     header.packInto(p);
 398     /* trailer */
 399     packerAppend(p, "\r\n", 2);
 400 }
 401
 402 /*
 403  * A wrapper for debugObj()
 404  */
 405 void
 406 httpRequestPack(void *obj, Packer *p)
 407 {
 408     HttpRequest *request = static_cast<HttpRequest*>(obj);
 409     request->pack(p);
 410 }
 411
 412 /* returns the length of request line + headers + crlf */
 413 int
 414 HttpRequest::prefixLen()
 415 {
 416     return strlen(RequestMethodStr(method)) + 1 +
 417            urlpath.size() + 1 +
 418            4 + 1 + 3 + 2 +
 419            header.len + 2;
 420 }
 421
 422 /* sync this routine when you update HttpRequest struct */
 423 void
 424 HttpRequest::hdrCacheInit()
 425 {
 426     HttpMsg::hdrCacheInit();
 427
 428     assert(!range);
 429     range = header.getRange();
 430 }
 431
 432 #if ICAP_CLIENT
 433 Adaptation::Icap::History::Pointer
 434 HttpRequest::icapHistory() const
 435 {
 436     if (!icapHistory_) {
 437         if (Log::TheConfig.hasIcapToken || IcapLogfileStatus == LOG_ENABLE) {
 438             icapHistory_ = new Adaptation::Icap::History();
 439             debugs(93,4, HERE << "made " << icapHistory_ << " for " << this);
 440         }
 441     }
 442
 443     return icapHistory_;
 444 }
 445 #endif
 446
 447 #if USE_ADAPTATION
 448 Adaptation::History::Pointer
 449 HttpRequest::adaptHistory(bool createIfNone) const
 450 {
 451     if (!adaptHistory_ && createIfNone) {
 452         adaptHistory_ = new Adaptation::History();
 453         debugs(93,4, HERE << "made " << adaptHistory_ << " for " << this);
 454     }
 455
 456     return adaptHistory_;
 457 }
 458
 459 Adaptation::History::Pointer
 460 HttpRequest::adaptLogHistory() const
 461 {
 462     return HttpRequest::adaptHistory(Log::TheConfig.hasAdaptToken);
 463 }
 464
 465 void
 466 HttpRequest::adaptHistoryImport(const HttpRequest &them)
 467 {
 468     if (!adaptHistory_) {
 469         adaptHistory_ = them.adaptHistory_; // may be nil
 470     } else {
 471         // check that histories did not diverge
 472         Must(!them.adaptHistory_ || them.adaptHistory_ == adaptHistory_);
 473     }
 474 }
 475
 476 #endif
 477
 478 bool
 479 HttpRequest::multipartRangeRequest() const
 480 {
 481     return (range && range->specs.count > 1);
 482 }
 483
 484 bool
 485 HttpRequest::bodyNibbled() const
 486 {
 487     return body_pipe != NULL && body_pipe->consumedSize() > 0;
 488 }
 489
 490 void
 491 HttpRequest::detailError(err_type aType, int aDetail)
 492 {
 493     if (errType || errDetail)
 494         debugs(11, 5, HERE << "old error details: " << errType << '/' << errDetail);
 495     debugs(11, 5, HERE << "current error details: " << aType << '/' << aDetail);
 496     // checking type and detail separately may cause inconsistency, but
 497     // may result in more details available if they only become available later
 498     if (!errType)
 499         errType = aType;
 500     if (!errDetail)
 501         errDetail = aDetail;
 502 }
 503
 504 void
 505 HttpRequest::clearError()
 506 {
 507     debugs(11, 7, HERE << "old error details: " << errType << '/' << errDetail);
 508     errType = ERR_NONE;
 509     errDetail = ERR_DETAIL_NONE;
 510 }
 511
 512 const char *HttpRequest::packableURI(bool full_uri) const
 513 {
 514     if (full_uri)
 515         return urlCanonical((HttpRequest*)this);
 516
 517     if (urlpath.size())
 518         return urlpath.termedBuf();
 519
 520     return "/";
 521 }
 522
 523 void HttpRequest::packFirstLineInto(Packer * p, bool full_uri) const
 524 {
 525     // form HTTP request-line
 526     packerPrintf(p, "%s %s HTTP/%d.%d\r\n",
 527                  RequestMethodStr(method),
 528                  packableURI(full_uri),
 529                  http_ver.major, http_ver.minor);
 530 }
 531
 532 /*
 533  * Indicate whether or not we would expect an entity-body
 534  * along with this request
 535  */
 536 bool
 537 HttpRequest::expectingBody(const HttpRequestMethod& unused, int64_t& theSize) const
 538 {
 539     bool expectBody = false;
 540
 541     /*
 542      * Note: Checks for message validity is in clientIsContentLengthValid().
 543      * this just checks if a entity-body is expected based on HTTP message syntax
 544      */
 545     if (header.chunked()) {
 546         expectBody = true;
 547         theSize = -1;
 548     } else if (content_length >= 0) {
 549         expectBody = true;
 550         theSize = content_length;
 551     } else {
 552         expectBody = false;
 553         // theSize undefined
 554     }
 555
 556     return expectBody;
 557 }
 558
 559 /*
 560  * Create a Request from a URL and METHOD.
 561  *
 562  * If the METHOD is CONNECT, then a host:port pair is looked for instead of a URL.
 563  * If the request cannot be created cleanly, NULL is returned
 564  */
 565 HttpRequest *
 566 HttpRequest::CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method)
 567 {
 568     return urlParse(method, url, NULL);
 569 }
 570
 571 /*
 572  * Create a Request from a URL.
 573  *
 574  * If the request cannot be created cleanly, NULL is returned
 575  */
 576 HttpRequest *
 577 HttpRequest::CreateFromUrl(char * url)
 578 {
 579     return urlParse(Http::METHOD_GET, url, NULL);
 580 }
 581
 582 /**
 583  * Are responses to this request possible cacheable ?
 584  * If false then no matter what the response must not be cached.
 585  */
 586 bool
 587 HttpRequest::maybeCacheable()
 588 {
 589     // Intercepted request with Host: header which cannot be trusted.
 590     // Because it failed verification, or someone bypassed the security tests
 591     // we cannot cache the reponse for sharing between clients.
 592     // TODO: update cache to store for particular clients only (going to same Host: and destination IP)
 593     if (!flags.hostVerified && (flags.intercepted || flags.interceptTproxy))
 594         return false;
 595
 596     switch (protocol) {
 597     case AnyP::PROTO_HTTP:
 598         if (!method.respMaybeCacheable())
 599             return false;
 600
 601         // XXX: this would seem the correct place to detect request cache-controls
 602         //      no-store, private and related which block cacheability
 603         break;
 604
 605     case AnyP::PROTO_GOPHER:
 606         if (!gopherCachable(this))
 607             return false;
 608         break;
 609
 610     case AnyP::PROTO_CACHE_OBJECT:
 611         return false;
 612
 613         //case AnyP::PROTO_FTP:
 614     default:
 615         break;
 616     }
 617
 618     return true;
 619 }
 620
 621 bool
 622 HttpRequest::conditional() const
 623 {
 624     return flags.ims ||
 625            header.has(HDR_IF_MATCH) ||
 626            header.has(HDR_IF_NONE_MATCH);
 627 }
 628
 629 void
 630 HttpRequest::recordLookup(const DnsLookupDetails &dns)
 631 {
 632     if (dns.wait >= 0) { // known delay
 633         if (dnsWait >= 0) // have recorded DNS wait before
 634             dnsWait += dns.wait;
 635         else
 636             dnsWait = dns.wait;
 637     }
 638 }
 639
 640 int64_t
 641 HttpRequest::getRangeOffsetLimit()
 642 {
 643     /* -2 is the starting value of rangeOffsetLimit.
 644      * If it is -2, that means we haven't checked it yet.
 645      *  Otherwise, return the current value */
 646     if (rangeOffsetLimit != -2)
 647         return rangeOffsetLimit;
 648
 649     rangeOffsetLimit = 0; // default value for rangeOffsetLimit
 650
 651     ACLFilledChecklist ch(NULL, this, NULL);
 652     ch.src_addr = client_addr;
 653     ch.my_addr =  my_addr;
 654
 655     for (AclSizeLimit *l = Config.rangeOffsetLimit; l; l = l -> next) {
 656         /* if there is no ACL list or if the ACLs listed match use this limit value */
 657         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
 658             debugs(58, 4, HERE << "rangeOffsetLimit=" << rangeOffsetLimit);
 659             rangeOffsetLimit = l->size; // may be -1
 660             break;
 661         }
 662     }
 663
 664     return rangeOffsetLimit;
 665 }
 666
 667 bool
 668 HttpRequest::canHandle1xx() const
 669 {
 670     // old clients do not support 1xx unless they sent Expect: 100-continue
 671     // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT)
 672     if (http_ver <= Http::ProtocolVersion(1,0) && !header.has(HDR_EXPECT))
 673         return false;
 674
 675     // others must support 1xx control messages
 676     return true;
 677 }
 678
 679 ConnStateData *
 680 HttpRequest::pinnedConnection()
 681 {
 682     if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned)
 683         return clientConnectionManager.get();
 684     return NULL;
 685 }
 686
 687 const char *
 688 HttpRequest::storeId()
 689 {
 690     if (store_id.size() != 0) {
 691         debugs(73, 3, "sent back store_id:" << store_id);
 692
 693         return store_id.termedBuf();
 694     }
 695     debugs(73, 3, "sent back canonicalUrl:" << urlCanonical(this) );
 696
 697     return urlCanonical(this);
 698 }