src/client_side_request.cc

   1
   2 /*
   3  * $Id: client_side_request.cc,v 1.60 2006/04/02 11:58:38 serassio Exp $
   4  *
   5  * DEBUG: section 85    Client-side Request Routines
   6  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  * Squid is the result of efforts by numerous individuals from the Internet
  12  * community; see the CONTRIBUTORS file for full details.   Many organizations
  13  * have provided support for Squid's development; see the SPONSORS file for
  14  * full details.  Squid is Copyrighted (C) 2001 by the Regents of the
  15  * University of California; see the COPYRIGHT file for full details.  Squid
  16  * incorporates software developed and/or copyrighted by other sources; see the
  17  * CREDITS file for full details.
  18  *
  19  * This program is free software; you can redistribute it and/or modify it under
  20  * the terms of the GNU General Public License as published by the Free
  21  * Software Foundation; either version 2 of the License, or (at your option)
  22  * any later version.
  23  *
  24  * This program is distributed in the hope that it will be useful, but WITHOUT ANY
  25  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  26  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
  27  * details.
  28  *
  29  * You should have received a copy of the GNU General Public License along with
  30  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  31  * Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35
  36 /*
  37  * General logic of request processing:
  38  *
  39  * We run a series of tests to determine if access will be permitted, and to do
  40  * any redirection. Then we call into the result clientStream to retrieve data.
  41  * From that point on it's up to reply management.
  42  */
  43
  44 #include "squid.h"
  45 #include "clientStream.h"
  46 #include "client_side_request.h"
  47 #include "AuthUserRequest.h"
  48 #include "HttpRequest.h"
  49 #include "ACLChecklist.h"
  50 #include "ACL.h"
  51 #include "client_side.h"
  52 #include "client_side_reply.h"
  53 #include "Store.h"
  54 #include "HttpReply.h"
  55 #include "MemObject.h"
  56 #include "ClientRequestContext.h"
  57
  58 #if ICAP_CLIENT
  59 #include "ICAP/ICAPClientReqmodPrecache.h"
  60 #include "ICAP/ICAPElements.h"
  61 #include "ICAP/ICAPConfig.h"
  62 static void icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data);
  63 extern ICAPConfig TheICAPConfig;
  64 #endif
  65
  66 #if LINGERING_CLOSE
  67 #define comm_close comm_lingering_close
  68 #endif
  69
  70 static const char *const crlf = "\r\n";
  71
  72 CBDATA_CLASS_INIT(ClientRequestContext);
  73
  74 void *
  75 ClientRequestContext::operator new (size_t size)
  76 {
  77     assert (size == sizeof(ClientRequestContext));
  78     CBDATA_INIT_TYPE(ClientRequestContext);
  79     ClientRequestContext *result = cbdataAlloc(ClientRequestContext);
  80     return result;
  81 }
  82
  83 void
  84 ClientRequestContext::operator delete (void *address)
  85 {
  86     ClientRequestContext *t = static_cast<ClientRequestContext *>(address);
  87     cbdataFree(t);
  88 }
  89
  90 /* Local functions */
  91 /* other */
  92 static void clientAccessCheckDoneWrapper(int, void *);
  93 static int clientCachable(ClientHttpRequest * http);
  94 static int clientHierarchical(ClientHttpRequest * http);
  95 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  96 static RH clientRedirectDoneWrapper;
  97 static PF checkNoCacheDoneWrapper;
  98 extern "C" CSR clientGetMoreData;
  99 extern "C" CSS clientReplyStatus;
 100 extern "C" CSD clientReplyDetach;
 101 static void checkFailureRatio(err_type, hier_code);
 102
 103 ClientRequestContext::~ClientRequestContext()
 104 {
 105     /*
 106      * Release our "lock" on our parent, ClientHttpRequest, if we
 107      * still have one
 108      */
 109
 110     if (http)
 111         cbdataReferenceDone(http);
 112
 113     if (acl_checklist) {
 114         if (acl_checklist->asyncInProgress()) {
 115             acl_checklist->markDeleteWhenDone();
 116         } else {
 117             delete acl_checklist;
 118         }
 119     }
 120 }
 121
 122 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(anHttp), acl_checklist (NULL), redirect_state (REDIRECT_NONE)
 123 {
 124     (void) cbdataReference(http);
 125     http_access_done = false;
 126     redirect_done = false;
 127     no_cache_done = false;
 128     interpreted_req_hdrs = false;
 129 }
 130
 131 CBDATA_CLASS_INIT(ClientHttpRequest);
 132
 133 void *
 134 ClientHttpRequest::operator new (size_t size)
 135 {
 136     assert (size == sizeof (ClientHttpRequest));
 137     CBDATA_INIT_TYPE(ClientHttpRequest);
 138     ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest);
 139     return result;
 140 }
 141
 142 void
 143 ClientHttpRequest::operator delete (void *address)
 144 {
 145     ClientHttpRequest *t = static_cast<ClientHttpRequest *>(address);
 146     cbdataFree(t);
 147 }
 148
 149 ClientHttpRequest::ClientHttpRequest(ConnStateData::Pointer aConn) : loggingEntry_(NULL)
 150 {
 151     start = current_time;
 152     setConn(aConn);
 153     dlinkAdd(this, &active, &ClientActiveRequests);
 154 #if ICAP_CLIENT
 155
 156     request_satisfaction_mode = false;
 157 #endif
 158 }
 159
 160 /*
 161  * returns true if client specified that the object must come from the cache
 162  * without contacting origin server
 163  */
 164 bool
 165 ClientHttpRequest::onlyIfCached()const
 166 {
 167     assert(request);
 168     return request->cache_control &&
 169            EBIT_TEST(request->cache_control->mask, CC_ONLY_IF_CACHED);
 170 }
 171
 172 /*
 173  * This function is designed to serve a fairly specific purpose.
 174  * Occasionally our vBNS-connected caches can talk to each other, but not
 175  * the rest of the world.  Here we try to detect frequent failures which
 176  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 177  * the failure:success ratio goes above 1.0 then we go into "hit only"
 178  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 179  * will only fetch HITs from us if they are using the ICP protocol.  We
 180  * stay in this mode for 5 minutes.
 181  *
 182  * Duane W., Sept 16, 1996
 183  */
 184
 185 #define FAILURE_MODE_TIME 300
 186
 187 static void
 188 checkFailureRatio(err_type etype, hier_code hcode)
 189 {
 190     static double magic_factor = 100.0;
 191     double n_good;
 192     double n_bad;
 193
 194     if (hcode == HIER_NONE)
 195         return;
 196
 197     n_good = magic_factor / (1.0 + request_failure_ratio);
 198
 199     n_bad = magic_factor - n_good;
 200
 201     switch (etype) {
 202
 203     case ERR_DNS_FAIL:
 204
 205     case ERR_CONNECT_FAIL:
 206
 207     case ERR_READ_ERROR:
 208         n_bad++;
 209         break;
 210
 211     default:
 212         n_good++;
 213     }
 214
 215     request_failure_ratio = n_bad / n_good;
 216
 217     if (hit_only_mode_until > squid_curtime)
 218         return;
 219
 220     if (request_failure_ratio < 1.0)
 221         return;
 222
 223     debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio);
 224
 225     debug(33, 0) ("Going into hit-only-mode for %d minutes...\n",
 226                   FAILURE_MODE_TIME / 60);
 227
 228     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 229
 230     request_failure_ratio = 0.8;        /* reset to something less than 1.0 */
 231 }
 232
 233 ClientHttpRequest::~ClientHttpRequest()
 234 {
 235     debug(33, 3) ("httpRequestFree: %s\n", uri);
 236     /* if body_connection !NULL, then ProcessBody has not
 237      * found the end of the body yet
 238      */
 239
 240     if (request && request->body_connection.getRaw() != NULL) {
 241         clientAbortBody(request);       /* abort body transter */
 242         request->body_connection = NULL;
 243     }
 244
 245     /* the ICP check here was erroneous
 246      * - storeReleaseRequest was always called if entry was valid
 247      */
 248     assert(logType < LOG_TYPE_MAX);
 249
 250     logRequest();
 251
 252     loggingEntry(NULL);
 253
 254     if (request)
 255         checkFailureRatio(request->errType, al.hier.code);
 256
 257     freeResources();
 258
 259 #if ICAP_CLIENT
 260
 261     if (icap) {
 262         delete icap;
 263         cbdataReferenceDone(icap);
 264     }
 265
 266 #endif
 267     if (calloutContext)
 268         delete calloutContext;
 269
 270     /* moving to the next connection is handled by the context free */
 271     dlinkDelete(&active, &ClientActiveRequests);
 272 }
 273
 274 /* Create a request and kick it off */
 275 /*
 276  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 277  * determined by the user
 278  */
 279 int                             /* returns nonzero on failure */
 280 clientBeginRequest(method_t method, char const *url, CSCB * streamcallback,
 281                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 282                    char *tailbuf, size_t taillen)
 283 {
 284     size_t url_sz;
 285     HttpVersion http_ver (1, 0);
 286     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 287     HttpRequest *request;
 288     StoreIOBuffer tempBuffer;
 289     http->start = current_time;
 290     /* this is only used to adjust the connection offset in client_side.c */
 291     http->req_sz = 0;
 292     tempBuffer.length = taillen;
 293     tempBuffer.data = tailbuf;
 294     /* client stream setup */
 295     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 296                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 297                      streamdetach, streamdata, tempBuffer);
 298     /* make it visible in the 'current acctive requests list' */
 299     /* Set flags */
 300     /* internal requests only makes sense in an
 301      * accelerator today. TODO: accept flags ? */
 302     http->flags.accel = 1;
 303     /* allow size for url rewriting */
 304     url_sz = strlen(url) + Config.appendDomainLen + 5;
 305     http->uri = (char *)xcalloc(url_sz, 1);
 306     strcpy(http->uri, url);
 307
 308     if ((request = urlParse(method, http->uri)) == NULL) {
 309         debug(85, 5) ("Invalid URL: %s\n", http->uri);
 310         return -1;
 311     }
 312
 313     /*
 314      * now update the headers in request with our supplied headers. urLParse
 315      * should return a blank header set, but we use Update to be sure of
 316      * correctness.
 317      */
 318     if (header)
 319         httpHeaderUpdate(&request->header, header, NULL);
 320
 321     http->log_uri = xstrdup(urlCanonicalClean(request));
 322
 323     /* http struct now ready */
 324
 325     /*
 326      * build new header list *? TODO
 327      */
 328     request->flags.accelerated = http->flags.accel;
 329
 330     request->flags.internalclient = 1;
 331
 332     /* this is an internally created
 333      * request, not subject to acceleration
 334      * target overrides */
 335     /*
 336      * FIXME? Do we want to detect and handle internal requests of internal
 337      * objects ?
 338      */
 339
 340     /* Internally created requests cannot have bodies today */
 341     request->content_length = 0;
 342
 343     request->client_addr = no_addr;
 344
 345     request->my_addr = no_addr; /* undefined for internal requests */
 346
 347     request->my_port = 0;
 348
 349     request->http_ver = http_ver;
 350
 351     http->request = HTTPMSGLOCK(request);
 352
 353     /* optional - skip the access check ? */
 354     http->calloutContext = new ClientRequestContext(http);
 355
 356     http->calloutContext->http_access_done = false;
 357
 358     http->calloutContext->redirect_done = true;
 359
 360     http->calloutContext->no_cache_done = true;
 361
 362     http->doCallouts();
 363
 364     return 0;
 365 }
 366
 367 bool
 368 ClientRequestContext::httpStateIsValid()
 369 {
 370     ClientHttpRequest *http_ = http;
 371
 372     if (cbdataReferenceValid(http_))
 373         return true;
 374
 375     http = NULL;
 376
 377     cbdataReferenceDone(http_);
 378
 379     return false;
 380 }
 381
 382 /* This is the entry point for external users of the client_side routines */
 383 void
 384 ClientRequestContext::clientAccessCheck()
 385 {
 386     acl_checklist =
 387         clientAclChecklistCreate(Config.accessList.http, http);
 388     acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 389 }
 390
 391 void
 392 clientAccessCheckDoneWrapper(int answer, void *data)
 393 {
 394     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 395
 396     if (!calloutContext->httpStateIsValid())
 397         return;
 398
 399     calloutContext->clientAccessCheckDone(answer);
 400 }
 401
 402 void
 403 ClientRequestContext::clientAccessCheckDone(int answer)
 404 {
 405     acl_checklist = NULL;
 406     err_type page_id;
 407     http_status status;
 408     debug(85, 2) ("The request %s %s is %s, because it matched '%s'\n",
 409                   RequestMethodStr[http->request->method], http->uri,
 410                   answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED",
 411                   AclMatchedName ? AclMatchedName : "NO ACL's");
 412     char const *proxy_auth_msg = "<null>";
 413
 414     if (http->getConn().getRaw() != NULL && http->getConn()->auth_user_request != NULL)
 415         proxy_auth_msg = http->getConn()->auth_user_request->denyMessage("<null>");
 416     else if (http->request->auth_user_request != NULL)
 417         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 418
 419     if (answer != ACCESS_ALLOWED) {
 420         /* Send an error */
 421         debug(85, 5) ("Access Denied: %s\n", http->uri);
 422         debug(85, 5) ("AclMatchedName = %s\n",
 423                       AclMatchedName ? AclMatchedName : "<null>");
 424         debug(85, 5) ("Proxy Auth Message = %s\n",
 425                       proxy_auth_msg ? proxy_auth_msg : "<null>");
 426         /*
 427          * NOTE: get page_id here, based on AclMatchedName because if
 428          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 429          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 430          * <pribeiro@isel.pt>
 431          */
 432         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName);
 433         http->logType = LOG_TCP_DENIED;
 434
 435         if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) {
 436             if (!http->flags.accel) {
 437                 /* Proxy authorisation needed */
 438                 status = HTTP_PROXY_AUTHENTICATION_REQUIRED;
 439             } else {
 440                 /* WWW authorisation needed */
 441                 status = HTTP_UNAUTHORIZED;
 442             }
 443
 444             if (page_id == ERR_NONE)
 445                 page_id = ERR_CACHE_ACCESS_DENIED;
 446         } else {
 447             status = HTTP_FORBIDDEN;
 448
 449             if (page_id == ERR_NONE)
 450                 page_id = ERR_ACCESS_DENIED;
 451         }
 452
 453         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 454         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 455         assert (repContext);
 456         repContext->setReplyToError(page_id, status,
 457                                     http->request->method, NULL,
 458                                     http->getConn().getRaw() != NULL ? &http->getConn()->peer.sin_addr : &no_addr, http->request,
 459                                     NULL, http->getConn().getRaw() != NULL
 460                                     && http->getConn()->auth_user_request ? http->getConn()->
 461                                     auth_user_request : http->request->auth_user_request);
 462         node = (clientStreamNode *)http->client_stream.tail->data;
 463         clientStreamRead(node, http, node->readBuffer);
 464         return;
 465     }
 466
 467     /* ACCESS_ALLOWED continues here ... */
 468     safe_free(http->uri);
 469
 470     http->uri = xstrdup(urlCanonical(http->request));
 471
 472     http->doCallouts();
 473 }
 474
 475 #if ICAP_CLIENT
 476 void
 477 ClientRequestContext::icapAccessCheck()
 478 {
 479     ICAPAccessCheck *icap_access_check;
 480
 481     icap_access_check = new ICAPAccessCheck(ICAP::methodReqmod, ICAP::pointPreCache, http->request, NULL, icapAclCheckDoneWrapper, this);
 482
 483     if (icap_access_check != NULL) {
 484         icap_access_check->check();
 485         return;
 486     }
 487
 488     http->doCallouts();
 489 }
 490
 491 static void
 492 icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data)
 493 {
 494     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 495
 496     if (!calloutContext->httpStateIsValid())
 497         return;
 498
 499     calloutContext->icapAclCheckDone(service);
 500 }
 501
 502 void
 503 ClientRequestContext::icapAclCheckDone(ICAPServiceRep::Pointer service)
 504 {
 505     /*
 506      * No matching ICAP service in the config file
 507      */
 508
 509     if (service == NULL) {
 510         http->doCallouts();
 511         return;
 512     }
 513
 514     /*
 515      * Setup ICAP state and such.  If successful, just return.
 516      * We'll get back to doCallouts() after REQMOD is done.
 517      */
 518     if (0 == http->doIcap(service))
 519         return;
 520
 521     /*
 522      * If doIcap() fails, then we have to either return an error
 523      * to the user, or keep going without ICAP.
 524      */
 525     fatal("Fix this case in ClientRequestContext::icapAclCheckDone()");
 526
 527     http->doCallouts();
 528 }
 529
 530 #endif
 531
 532 static void
 533 clientRedirectAccessCheckDone(int answer, void *data)
 534 {
 535     ClientRequestContext *context = (ClientRequestContext *)data;
 536     ClientHttpRequest *http = context->http;
 537     context->acl_checklist = NULL;
 538
 539     if (answer == ACCESS_ALLOWED)
 540         redirectStart(http, clientRedirectDoneWrapper, context);
 541     else
 542         context->clientRedirectDone(NULL);
 543 }
 544
 545 void
 546 ClientRequestContext::clientRedirectStart()
 547 {
 548     debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri);
 549
 550     if (Config.accessList.redirector) {
 551         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 552         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 553     } else
 554         redirectStart(http, clientRedirectDoneWrapper, this);
 555 }
 556
 557 static int
 558 clientCachable(ClientHttpRequest * http)
 559 {
 560     HttpRequest *req = http->request;
 561     method_t method = req->method;
 562
 563     if (req->protocol == PROTO_HTTP)
 564         return httpCachable(method);
 565
 566     /* FTP is always cachable */
 567     if (req->protocol == PROTO_WAIS)
 568         return 0;
 569
 570     /*
 571      * The below looks questionable: what non HTTP protocols use connect,
 572      * trace, put and post? RC
 573      */
 574     if (method == METHOD_CONNECT)
 575         return 0;
 576
 577     if (method == METHOD_TRACE)
 578         return 0;
 579
 580     if (method == METHOD_PUT)
 581         return 0;
 582
 583     if (method == METHOD_POST)
 584         return 0;
 585
 586     /* XXX POST may be cached sometimes.. ignored
 587
 588                                                                  * for now */
 589     if (req->protocol == PROTO_GOPHER)
 590         return gopherCachable(req);
 591
 592     if (req->protocol == PROTO_CACHEOBJ)
 593         return 0;
 594
 595     return 1;
 596 }
 597
 598 static int
 599 clientHierarchical(ClientHttpRequest * http)
 600 {
 601     const char *url = http->uri;
 602     HttpRequest *request = http->request;
 603     method_t method = request->method;
 604     const wordlist *p = NULL;
 605
 606     /*
 607      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 608      * neighbors support private keys
 609      */
 610
 611     if (request->flags.ims && !neighbors_do_private_keys)
 612         return 0;
 613
 614     /*
 615      * This is incorrect: authenticating requests can be sent via a hierarchy
 616      * (they can even be cached if the correct headers are set on the reply)
 617      */
 618     if (request->flags.auth)
 619         return 0;
 620
 621     if (method == METHOD_TRACE)
 622         return 1;
 623
 624     if (method != METHOD_GET)
 625         return 0;
 626
 627     /* scan hierarchy_stoplist */
 628     for (p = Config.hierarchy_stoplist; p; p = p->next)
 629         if (strstr(url, p->key))
 630             return 0;
 631
 632     if (request->flags.loopdetect)
 633         return 0;
 634
 635     if (request->protocol == PROTO_HTTP)
 636         return httpCachable(method);
 637
 638     if (request->protocol == PROTO_GOPHER)
 639         return gopherCachable(request);
 640
 641     if (request->protocol == PROTO_WAIS)
 642         return 0;
 643
 644     if (request->protocol == PROTO_CACHEOBJ)
 645         return 0;
 646
 647     return 1;
 648 }
 649
 650
 651 static void
 652 clientInterpretRequestHeaders(ClientHttpRequest * http)
 653 {
 654     HttpRequest *request = http->request;
 655     const HttpHeader *req_hdr = &request->header;
 656     int no_cache = 0;
 657 #if !(ESI) || defined(USE_USERAGENT_LOG) || defined(USE_REFERER_LOG)
 658
 659     const char *str;
 660 #endif
 661
 662     request->imslen = -1;
 663     request->ims = httpHeaderGetTime(req_hdr, HDR_IF_MODIFIED_SINCE);
 664
 665     if (request->ims > 0)
 666         request->flags.ims = 1;
 667
 668 #if ESI
 669     /*
 670      * We ignore Cache-Control as per the Edge Architecture Section 3. See
 671      * www.esi.org for more information.
 672      */
 673 #else
 674
 675     if (httpHeaderHas(req_hdr, HDR_PRAGMA)) {
 676         String s = httpHeaderGetList(req_hdr, HDR_PRAGMA);
 677
 678         if (strListIsMember(&s, "no-cache", ','))
 679             no_cache++;
 680
 681         s.clean();
 682     }
 683
 684     if (request->cache_control)
 685         if (EBIT_TEST(request->cache_control->mask, CC_NO_CACHE))
 686             no_cache++;
 687
 688     /*
 689     * Work around for supporting the Reload button in IE browsers when Squid
 690     * is used as an accelerator or transparent proxy, by turning accelerated
 691     * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
 692     * actually only fixed in SP1, but we can't tell whether we are talking to
 693     * SP1 or not so all 5.5 versions are treated 'normally').
 694     */
 695     if (Config.onoff.ie_refresh) {
 696         if (http->flags.accel && request->flags.ims) {
 697             if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT))) {
 698                 if (strstr(str, "MSIE 5.01") != NULL)
 699                     no_cache++;
 700                 else if (strstr(str, "MSIE 5.0") != NULL)
 701                     no_cache++;
 702                 else if (strstr(str, "MSIE 4.") != NULL)
 703                     no_cache++;
 704                 else if (strstr(str, "MSIE 3.") != NULL)
 705                     no_cache++;
 706             }
 707         }
 708     }
 709
 710 #endif
 711     if (no_cache) {
 712 #if HTTP_VIOLATIONS
 713
 714         if (Config.onoff.reload_into_ims)
 715             request->flags.nocache_hack = 1;
 716         else if (refresh_nocache_hack)
 717             request->flags.nocache_hack = 1;
 718         else
 719 #endif
 720
 721             request->flags.nocache = 1;
 722     }
 723
 724     /* ignore range header in non-GETs */
 725     if (request->method == METHOD_GET) {
 726         request->range = httpHeaderGetRange(req_hdr);
 727
 728         if (request->range) {
 729             request->flags.range = 1;
 730             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
 731             /* XXX: This is suboptimal. We should give the stream the range set,
 732              * and thereby let the top of the stream set the offset when the
 733              * size becomes known. As it is, we will end up requesting from 0
 734              * for evey -X range specification.
 735              * RBC - this may be somewhat wrong. We should probably set the range
 736              * iter up at this point.
 737              */
 738             node->readBuffer.offset = request->range->lowestOffset(0);
 739             http->range_iter.pos = request->range->begin();
 740             http->range_iter.valid = true;
 741         }
 742     }
 743
 744     if (httpHeaderHas(req_hdr, HDR_AUTHORIZATION))
 745         request->flags.auth = 1;
 746
 747     if (request->login[0] != '\0')
 748         request->flags.auth = 1;
 749
 750     if (httpHeaderHas(req_hdr, HDR_VIA)) {
 751         String s = httpHeaderGetList(req_hdr, HDR_VIA);
 752         /*
 753          * ThisCache cannot be a member of Via header, "1.0 ThisCache" can.
 754          * Note ThisCache2 has a space prepended to the hostname so we don't
 755          * accidentally match super-domains.
 756          */
 757
 758         if (strListIsSubstr(&s, ThisCache2, ',')) {
 759             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
 760                      request, (ObjPackMethod) & httpRequestPack);
 761             request->flags.loopdetect = 1;
 762         }
 763
 764 #if FORW_VIA_DB
 765         fvdbCountVia(s.buf());
 766
 767 #endif
 768
 769         s.clean();
 770     }
 771
 772 #if USE_USERAGENT_LOG
 773     if ((str = httpHeaderGetStr(req_hdr, HDR_USER_AGENT)))
 774         logUserAgent(fqdnFromAddr(http->getConn().getRaw() ? http->getConn()->log_addr : no_addr), str);
 775
 776 #endif
 777 #if USE_REFERER_LOG
 778
 779     if ((str = httpHeaderGetStr(req_hdr, HDR_REFERER)))
 780         logReferer(fqdnFromAddr(http->getConn().getRaw() ? http->getConn()->log_addr : no_addr), str, http->log_uri);
 781
 782 #endif
 783 #if FORW_VIA_DB
 784
 785     if (httpHeaderHas(req_hdr, HDR_X_FORWARDED_FOR)) {
 786         String s = httpHeaderGetList(req_hdr, HDR_X_FORWARDED_FOR);
 787         fvdbCountForw(s.buf());
 788         s.clean();
 789     }
 790
 791 #endif
 792     if (request->method == METHOD_TRACE) {
 793         request->max_forwards = httpHeaderGetInt(req_hdr, HDR_MAX_FORWARDS);
 794     }
 795
 796     if (clientCachable(http))
 797         request->flags.cachable = 1;
 798
 799     if (clientHierarchical(http))
 800         request->flags.hierarchical = 1;
 801
 802     debug(85, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n",
 803                   request->flags.nocache ? "SET" : "NOT SET");
 804
 805     debug(85, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n",
 806                   request->flags.cachable ? "SET" : "NOT SET");
 807
 808     debug(85, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n",
 809                   request->flags.hierarchical ? "SET" : "NOT SET");
 810 }
 811
 812 void
 813 clientRedirectDoneWrapper(void *data, char *result)
 814 {
 815     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 816
 817     if (!calloutContext->httpStateIsValid())
 818         return;
 819
 820     calloutContext->clientRedirectDone(result);
 821 }
 822
 823 void
 824 ClientRequestContext::clientRedirectDone(char *result)
 825 {
 826     HttpRequest *new_request = NULL;
 827     HttpRequest *old_request = http->request;
 828     debug(85, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri,
 829                   result ? result : "NULL");
 830     assert(redirect_state == REDIRECT_PENDING);
 831     redirect_state = REDIRECT_DONE;
 832
 833     if (result) {
 834         http_status status = (http_status) atoi(result);
 835
 836         if (status == HTTP_MOVED_PERMANENTLY
 837                 || status == HTTP_MOVED_TEMPORARILY
 838                 || status == HTTP_SEE_OTHER
 839                 || status == HTTP_TEMPORARY_REDIRECT) {
 840             char *t = result;
 841
 842             if ((t = strchr(result, ':')) != NULL) {
 843                 http->redirect.status = status;
 844                 http->redirect.location = xstrdup(t + 1);
 845             } else {
 846                 debug(85, 1) ("clientRedirectDone: bad input: %s\n", result);
 847             }
 848         } else if (strcmp(result, http->uri))
 849             new_request = urlParse(old_request->method, result);
 850     }
 851
 852     if (new_request) {
 853         safe_free(http->uri);
 854         http->uri = xstrdup(urlCanonical(new_request));
 855         new_request->http_ver = old_request->http_ver;
 856         httpHeaderAppend(&new_request->header, &old_request->header);
 857         new_request->client_addr = old_request->client_addr;
 858         new_request->client_port = old_request->client_port;
 859         new_request->my_addr = old_request->my_addr;
 860         new_request->my_port = old_request->my_port;
 861         new_request->flags = old_request->flags;
 862         new_request->flags.redirected = 1;
 863
 864         if (old_request->auth_user_request) {
 865             new_request->auth_user_request = old_request->auth_user_request;
 866
 867             new_request->auth_user_request->lock()
 868
 869             ;
 870         }
 871
 872         if (old_request->body_connection.getRaw() != NULL) {
 873             new_request->body_connection = old_request->body_connection;
 874             old_request->body_connection = NULL;
 875         }
 876
 877         new_request->content_length = old_request->content_length;
 878         new_request->extacl_user = old_request->extacl_user;
 879         new_request->extacl_passwd = old_request->extacl_passwd;
 880         new_request->flags.proxy_keepalive = old_request->flags.proxy_keepalive;
 881         HTTPMSGUNLOCK(old_request);
 882         http->request = HTTPMSGLOCK(new_request);
 883     }
 884
 885     /* FIXME PIPELINE: This is innacurate during pipelining */
 886
 887     if (http->getConn().getRaw() != NULL)
 888         fd_note(http->getConn()->fd, http->uri);
 889
 890     assert(http->uri);
 891
 892     http->doCallouts();
 893 }
 894
 895 void
 896 ClientRequestContext::checkNoCache()
 897 {
 898     acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
 899     acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
 900 }
 901
 902 static void
 903 checkNoCacheDoneWrapper(int answer, void *data)
 904 {
 905     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 906
 907     if (!calloutContext->httpStateIsValid())
 908         return;
 909
 910     calloutContext->checkNoCacheDone(answer);
 911 }
 912
 913 void
 914 ClientRequestContext::checkNoCacheDone(int answer)
 915 {
 916     acl_checklist = NULL;
 917     http->request->flags.cachable = answer;
 918     http->doCallouts();
 919 }
 920
 921 /*
 922  * Identify requests that do not go through the store and client side stream
 923  * and forward them to the appropriate location. All other requests, request
 924  * them.
 925  */
 926 void
 927 ClientHttpRequest::processRequest()
 928 {
 929     debug(85, 4) ("clientProcessRequest: %s '%s'\n",
 930                   RequestMethodStr[request->method], uri);
 931
 932     if (request->method == METHOD_CONNECT && !redirect.status) {
 933         logType = LOG_TCP_MISS;
 934         sslStart(this, &out.size, &al.http.code);
 935         return;
 936     }
 937
 938     httpStart();
 939 }
 940
 941 void
 942 ClientHttpRequest::httpStart()
 943 {
 944     logType = LOG_TAG_NONE;
 945     debug(85, 4) ("ClientHttpRequest::httpStart: %s for '%s'\n",
 946                   log_tags[logType], uri);
 947     /* no one should have touched this */
 948     assert(out.offset == 0);
 949     /* Use the Stream Luke */
 950     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
 951     clientStreamRead(node, this, node->readBuffer);
 952 }
 953
 954 bool
 955 ClientHttpRequest::gotEnough() const
 956 {
 957     /** TODO: should be querying the stream. */
 958     int contentLength =
 959         memObject()->getReply()->bodySize(request->method);
 960     assert(contentLength >= 0);
 961
 962     if (out.offset < contentLength)
 963         return false;
 964
 965     return true;
 966 }
 967
 968 void
 969 ClientHttpRequest::maxReplyBodySize(ssize_t clen)
 970 {
 971     maxReplyBodySize_ = clen;
 972 }
 973
 974 ssize_t
 975 ClientHttpRequest::maxReplyBodySize() const
 976 {
 977     return maxReplyBodySize_;
 978 }
 979
 980 bool
 981 ClientHttpRequest::isReplyBodyTooLarge(ssize_t clen) const
 982 {
 983     if (0 == maxReplyBodySize())
 984         return 0;       /* disabled */
 985
 986     if (clen < 0)
 987         return 0;       /* unknown */
 988
 989     return clen > maxReplyBodySize();
 990 }
 991
 992 void
 993 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
 994 {
 995     entry_ = newEntry;
 996 }
 997
 998 void
 999 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1000 {
1001     if (loggingEntry_)
1002         storeUnlockObject(loggingEntry_);
1003
1004     loggingEntry_ = newEntry;
1005
1006     if (loggingEntry_)
1007         storeLockObject(loggingEntry_);
1008 }
1009
1010 /*
1011  * doCallouts() - This function controls the order of "callout"
1012  * executions, including non-blocking access control checks, the
1013  * redirector, and ICAP.  Previously, these callouts were chained
1014  * together such that "clientAccessCheckDone()" would call
1015  * "clientRedirectStart()" and so on.
1016  *
1017  * The ClientRequestContext (aka calloutContext) class holds certain
1018  * state data for the callout/callback operations.  Previously
1019  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1020  * for a short time.  ClientRequestContext would then delete itself
1021  * and pass control back to ClientHttpRequest when all callouts
1022  * were finished.
1023  *
1024  * This caused some problems for ICAP because we want to make the
1025  * ICAP callout after checking ACLs, but before checking the no_cache
1026  * list.  We can't stuff the ICAP state into the ClientRequestContext
1027  * class because we still need the ICAP state after ClientRequestContext
1028  * goes away.
1029  *
1030  * Note that ClientRequestContext is created before the first call
1031  * to doCallouts().
1032  *
1033  * If one of the callouts notices that ClientHttpRequest is no
1034  * longer valid, it should call cbdataReferenceDone() so that
1035  * ClientHttpRequest's reference count goes to zero and it will get
1036  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1037  *
1038  * Note that we set the _done flags here before actually starting
1039  * the callout.  This is strictly for convenience.
1040  */
1041
1042 void
1043 ClientHttpRequest::doCallouts()
1044 {
1045     assert(calloutContext);
1046
1047     if (!calloutContext->http_access_done) {
1048         calloutContext->http_access_done = true;
1049         calloutContext->clientAccessCheck();
1050         return;
1051     }
1052
1053 #if ICAP_CLIENT
1054     if (TheICAPConfig.onoff && !calloutContext->icap_acl_check_done) {
1055         calloutContext->icap_acl_check_done = true;
1056         calloutContext->icapAccessCheck();
1057         return;
1058     }
1059
1060 #endif
1061
1062     if (!calloutContext->redirect_done) {
1063         calloutContext->redirect_done = true;
1064         assert(calloutContext->redirect_state == REDIRECT_NONE);
1065
1066         if (Config.Program.redirect) {
1067             calloutContext->redirect_state = REDIRECT_PENDING;
1068             calloutContext->clientRedirectStart();
1069             return;
1070         }
1071     }
1072
1073     if (!calloutContext->interpreted_req_hdrs) {
1074         calloutContext->interpreted_req_hdrs = 1;
1075         clientInterpretRequestHeaders(this);
1076     }
1077
1078     if (!calloutContext->no_cache_done) {
1079         calloutContext->no_cache_done = true;
1080
1081         if (Config.accessList.noCache && request->flags.cachable) {
1082             calloutContext->checkNoCache();
1083             return;
1084         }
1085     }
1086
1087     cbdataReferenceDone(calloutContext->http);
1088     delete calloutContext;
1089     calloutContext = NULL;
1090 #if HEADERS_LOG
1091
1092     headersLog(0, 1, request->method, request);
1093 #endif
1094
1095     processRequest();
1096 }
1097
1098 #ifndef _USE_INLINE_
1099 #include "client_side_request.cci"
1100 #endif
1101
1102 #if ICAP_CLIENT
1103 /*
1104  * Initiate an ICAP transaction.  Return 0 if all is well, or -1 upon error.
1105  * Caller will handle error condition by generating a Squid error message
1106  * or take other action.
1107  */
1108 int
1109 ClientHttpRequest::doIcap(ICAPServiceRep::Pointer service)
1110 {
1111     debug(85,3)("ClientHttpRequest::doIcap() called\n");
1112     assert(NULL == icap);
1113     icap = new ICAPClientReqmodPrecache(service);
1114     (void) cbdataReference(icap);
1115     icap->startReqMod(this, request);
1116     icap->doneSending();
1117     return 0;
1118 }
1119
1120 /*
1121  * Called by ICAPAnchor when it has space available for us.
1122  */
1123 void
1124 ClientHttpRequest::icapSpaceAvailable()
1125 {
1126     debug(85,3)("ClientHttpRequest::icapSpaceAvailable() called\n");
1127 }
1128
1129 void
1130 ClientHttpRequest::takeAdaptedHeaders(HttpMsg *msg)
1131 {
1132     debug(85,3)("ClientHttpRequest::takeAdaptedHeaders() called\n");
1133     assert(cbdataReferenceValid(this));         // indicates bug
1134
1135     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1136         /*
1137          * Replace the old request with the new request.  First,
1138          * Move the "body_connection" over, then unlink old and
1139          * link new to the http state.
1140          */
1141         new_req->body_connection = request->body_connection;
1142         request->body_connection = NULL;
1143         HTTPMSGUNLOCK(request);
1144         request = HTTPMSGLOCK(new_req);
1145         /*
1146          * Store the new URI for logging
1147          */
1148         xfree(uri);
1149         uri = xstrdup(urlCanonical(request));
1150         setLogUri(this, urlCanonicalClean(request));
1151         assert(request->method);
1152     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1153         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1154
1155         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1156         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1157         repContext->createStoreEntry(request->method, request->flags);
1158
1159         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1160         request_satisfaction_mode = true;
1161         request_satisfaction_offset = 0;
1162         storeEntry()->replaceHttpReply(new_rep);
1163         clientGetMoreData(node, this);
1164     }
1165
1166     if (!request_satisfaction_mode)
1167         doCallouts();
1168
1169     debug(85,3)("ClientHttpRequest::takeAdaptedHeaders() finished\n");
1170 }
1171
1172 void
1173 ClientHttpRequest::takeAdaptedBody(MemBuf *buf)
1174 {
1175     debug(85,3)("ClientHttpRequest::takeAdaptedBody() called\n");
1176
1177     if (request_satisfaction_mode) {
1178         storeEntry()->write(StoreIOBuffer(buf, request_satisfaction_offset));
1179         request_satisfaction_offset += buf->contentSize();
1180         buf->consume(buf->contentSize()); // consume everything written
1181     }
1182 }
1183
1184 void
1185 ClientHttpRequest::doneAdapting()
1186 {
1187     debug(85,3)("ClientHttpRequest::doneAdapting() called\n");
1188 }
1189
1190 void
1191 ClientHttpRequest::abortAdapting()
1192 {
1193     debug(85,3)("ClientHttpRequest::abortAdapting() called\n");
1194
1195     if ((NULL == storeEntry()) || storeEntry()->isEmpty()) {
1196         debug(85,3)("WARNING: ICAP REQMOD callout failed, proceeding with original request\n");
1197         doCallouts();
1198 #if ICAP_HARD_ERROR
1199
1200         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1201         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1202         assert (repContext);
1203         // Note if this code is ever used, clientBuildError() should be modified to
1204         // accept an errno arg
1205         repContext->setReplyToError(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR,
1206                                     request->method, NULL,
1207                                     getConn().getRaw() != NULL ? &getConn()->peer.sin_addr : &no_addr, request,
1208                                     NULL, getConn().getRaw() != NULL
1209                                     && getConn()->auth_user_request ? getConn()->
1210                                     auth_user_request : request->auth_user_request, errno);
1211         node = (clientStreamNode *)client_stream.tail->data;
1212         clientStreamRead(node, this, node->readBuffer);
1213 #endif
1214
1215         return;
1216     }
1217
1218     debug(0,0)("write me at %s:%d\n", __FILE__,__LINE__);
1219 }
1220
1221 #endif