src/client_side_request.cc

   1
   2 /*
   3  * $Id: client_side_request.cc,v 1.77 2006/10/31 23:30:57 wessels Exp $
   4  *
   5  * DEBUG: section 85    Client-side Request Routines
   6  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36
  37 /*
  38  * General logic of request processing:
  39  *
  40  * We run a series of tests to determine if access will be permitted, and to do
  41  * any redirection. Then we call into the result clientStream to retrieve data.
  42  * From that point on it's up to reply management.
  43  */
  44
  45 #include "squid.h"
  46 #include "clientStream.h"
  47 #include "client_side_request.h"
  48 #include "AuthUserRequest.h"
  49 #include "HttpRequest.h"
  50 #include "ACLChecklist.h"
  51 #include "ACL.h"
  52 #include "client_side.h"
  53 #include "client_side_reply.h"
  54 #include "Store.h"
  55 #include "HttpReply.h"
  56 #include "MemObject.h"
  57 #include "ClientRequestContext.h"
  58 #include "SquidTime.h"
  59 #include "wordlist.h"
  60
  61 #if ICAP_CLIENT
  62 #include "ICAP/ICAPClientReqmodPrecache.h"
  63 #include "ICAP/ICAPElements.h"
  64 #include "ICAP/ICAPConfig.h"
  65 static void icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data);
  66 extern ICAPConfig TheICAPConfig;
  67 #endif
  68
  69 #if LINGERING_CLOSE
  70 #define comm_close comm_lingering_close
  71 #endif
  72
  73 static const char *const crlf = "\r\n";
  74
  75 CBDATA_CLASS_INIT(ClientRequestContext);
  76
  77 void *
  78 ClientRequestContext::operator new (size_t size)
  79 {
  80     assert (size == sizeof(ClientRequestContext));
  81     CBDATA_INIT_TYPE(ClientRequestContext);
  82     ClientRequestContext *result = cbdataAlloc(ClientRequestContext);
  83     return result;
  84 }
  85
  86 void
  87 ClientRequestContext::operator delete (void *address)
  88 {
  89     ClientRequestContext *t = static_cast<ClientRequestContext *>(address);
  90     cbdataFree(t);
  91 }
  92
  93 /* Local functions */
  94 /* other */
  95 static void clientAccessCheckDoneWrapper(int, void *);
  96 static int clientHierarchical(ClientHttpRequest * http);
  97 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  98 static RH clientRedirectDoneWrapper;
  99 static PF checkNoCacheDoneWrapper;
 100 extern "C" CSR clientGetMoreData;
 101 extern "C" CSS clientReplyStatus;
 102 extern "C" CSD clientReplyDetach;
 103 static void checkFailureRatio(err_type, hier_code);
 104
 105 ClientRequestContext::~ClientRequestContext()
 106 {
 107     /*
 108      * Release our "lock" on our parent, ClientHttpRequest, if we
 109      * still have one
 110      */
 111
 112     if (http)
 113         cbdataReferenceDone(http);
 114
 115     if (acl_checklist) {
 116         if (acl_checklist->asyncInProgress()) {
 117             acl_checklist->markDeleteWhenDone();
 118         } else {
 119             delete acl_checklist;
 120         }
 121     }
 122
 123     debugs(85,3, HERE << this << " ClientHttpRequest destructed");
 124 }
 125
 126 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE)
 127 {
 128     http_access_done = false;
 129     redirect_done = false;
 130     no_cache_done = false;
 131     interpreted_req_hdrs = false;
 132     debugs(85,3, HERE << this << " ClientRequestContext constructed");
 133 }
 134
 135 CBDATA_CLASS_INIT(ClientHttpRequest);
 136
 137 void *
 138 ClientHttpRequest::operator new (size_t size)
 139 {
 140     assert (size == sizeof (ClientHttpRequest));
 141     CBDATA_INIT_TYPE(ClientHttpRequest);
 142     ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest);
 143     return result;
 144 }
 145
 146 void
 147 ClientHttpRequest::operator delete (void *address)
 148 {
 149     ClientHttpRequest *t = static_cast<ClientHttpRequest *>(address);
 150     cbdataFree(t);
 151 }
 152
 153 ClientHttpRequest::ClientHttpRequest(ConnStateData::Pointer aConn) : loggingEntry_(NULL)
 154 {
 155     start = current_time;
 156     setConn(aConn);
 157     dlinkAdd(this, &active, &ClientActiveRequests);
 158 #if ICAP_CLIENT
 159
 160     request_satisfaction_mode = false;
 161 #endif
 162 }
 163
 164 /*
 165  * returns true if client specified that the object must come from the cache
 166  * without contacting origin server
 167  */
 168 bool
 169 ClientHttpRequest::onlyIfCached()const
 170 {
 171     assert(request);
 172     return request->cache_control &&
 173            EBIT_TEST(request->cache_control->mask, CC_ONLY_IF_CACHED);
 174 }
 175
 176 /*
 177  * This function is designed to serve a fairly specific purpose.
 178  * Occasionally our vBNS-connected caches can talk to each other, but not
 179  * the rest of the world.  Here we try to detect frequent failures which
 180  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 181  * the failure:success ratio goes above 1.0 then we go into "hit only"
 182  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 183  * will only fetch HITs from us if they are using the ICP protocol.  We
 184  * stay in this mode for 5 minutes.
 185  *
 186  * Duane W., Sept 16, 1996
 187  */
 188
 189 #define FAILURE_MODE_TIME 300
 190
 191 static void
 192 checkFailureRatio(err_type etype, hier_code hcode)
 193 {
 194     static double magic_factor = 100.0;
 195     double n_good;
 196     double n_bad;
 197
 198     if (hcode == HIER_NONE)
 199         return;
 200
 201     n_good = magic_factor / (1.0 + request_failure_ratio);
 202
 203     n_bad = magic_factor - n_good;
 204
 205     switch (etype) {
 206
 207     case ERR_DNS_FAIL:
 208
 209     case ERR_CONNECT_FAIL:
 210
 211     case ERR_READ_ERROR:
 212         n_bad++;
 213         break;
 214
 215     default:
 216         n_good++;
 217     }
 218
 219     request_failure_ratio = n_bad / n_good;
 220
 221     if (hit_only_mode_until > squid_curtime)
 222         return;
 223
 224     if (request_failure_ratio < 1.0)
 225         return;
 226
 227     debug(33, 0) ("Failure Ratio at %4.2f\n", request_failure_ratio);
 228
 229     debug(33, 0) ("Going into hit-only-mode for %d minutes...\n",
 230                   FAILURE_MODE_TIME / 60);
 231
 232     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 233
 234     request_failure_ratio = 0.8;        /* reset to something less than 1.0 */
 235 }
 236
 237 ClientHttpRequest::~ClientHttpRequest()
 238 {
 239     debug(33, 3) ("httpRequestFree: %s\n", uri);
 240     PROF_start(httpRequestFree);
 241     /* if body_connection !NULL, then ProcessBody has not
 242      * found the end of the body yet
 243      */
 244
 245     if (request && request->body_reader != NULL) {
 246         request->body_reader = NULL;    // refcounted, triggers abort if needed.
 247         debugs(32, 3, HERE << "setting body_reader = NULL for request " << request);
 248     }
 249
 250     /* the ICP check here was erroneous
 251      * - storeReleaseRequest was always called if entry was valid
 252      */
 253     assert(logType < LOG_TYPE_MAX);
 254     logRequest();
 255     loggingEntry(NULL);
 256
 257     if (request)
 258         checkFailureRatio(request->errType, al.hier.code);
 259
 260     freeResources();
 261
 262 #if ICAP_CLIENT
 263     if (icap)
 264         delete icap;
 265 #endif
 266     if (calloutContext)
 267         delete calloutContext;
 268
 269     /* moving to the next connection is handled by the context free */
 270     dlinkDelete(&active, &ClientActiveRequests);
 271     PROF_stop(httpRequestFree);
 272 }
 273
 274 /* Create a request and kick it off */
 275 /*
 276  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 277  * determined by the user
 278  */
 279 int                             /* returns nonzero on failure */
 280 clientBeginRequest(method_t method, char const *url, CSCB * streamcallback,
 281                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 282                    char *tailbuf, size_t taillen)
 283 {
 284     size_t url_sz;
 285     HttpVersion http_ver (1, 0);
 286     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 287     HttpRequest *request;
 288     StoreIOBuffer tempBuffer;
 289     http->start = current_time;
 290     /* this is only used to adjust the connection offset in client_side.c */
 291     http->req_sz = 0;
 292     tempBuffer.length = taillen;
 293     tempBuffer.data = tailbuf;
 294     /* client stream setup */
 295     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 296                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 297                      streamdetach, streamdata, tempBuffer);
 298     /* make it visible in the 'current acctive requests list' */
 299     /* Set flags */
 300     /* internal requests only makes sense in an
 301      * accelerator today. TODO: accept flags ? */
 302     http->flags.accel = 1;
 303     /* allow size for url rewriting */
 304     url_sz = strlen(url) + Config.appendDomainLen + 5;
 305     http->uri = (char *)xcalloc(url_sz, 1);
 306     strcpy(http->uri, url);
 307
 308     if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
 309         debug(85, 5) ("Invalid URL: %s\n", http->uri);
 310         return -1;
 311     }
 312
 313     /*
 314      * now update the headers in request with our supplied headers. urLParse
 315      * should return a blank header set, but we use Update to be sure of
 316      * correctness.
 317      */
 318     if (header)
 319         request->header.update(header, NULL);
 320
 321     http->log_uri = xstrdup(urlCanonicalClean(request));
 322
 323     /* http struct now ready */
 324
 325     /*
 326      * build new header list *? TODO
 327      */
 328     request->flags.accelerated = http->flags.accel;
 329
 330     request->flags.internalclient = 1;
 331
 332     /* this is an internally created
 333      * request, not subject to acceleration
 334      * target overrides */
 335     /*
 336      * FIXME? Do we want to detect and handle internal requests of internal
 337      * objects ?
 338      */
 339
 340     /* Internally created requests cannot have bodies today */
 341     request->content_length = 0;
 342
 343     request->client_addr = no_addr;
 344
 345     request->my_addr = no_addr; /* undefined for internal requests */
 346
 347     request->my_port = 0;
 348
 349     request->http_ver = http_ver;
 350
 351     http->request = HTTPMSGLOCK(request);
 352
 353     /* optional - skip the access check ? */
 354     http->calloutContext = new ClientRequestContext(http);
 355
 356     http->calloutContext->http_access_done = false;
 357
 358     http->calloutContext->redirect_done = true;
 359
 360     http->calloutContext->no_cache_done = true;
 361
 362     http->doCallouts();
 363
 364     return 0;
 365 }
 366
 367 bool
 368 ClientRequestContext::httpStateIsValid()
 369 {
 370     ClientHttpRequest *http_ = http;
 371
 372     if (cbdataReferenceValid(http_))
 373         return true;
 374
 375     http = NULL;
 376
 377     cbdataReferenceDone(http_);
 378
 379     return false;
 380 }
 381
 382 /* This is the entry point for external users of the client_side routines */
 383 void
 384 ClientRequestContext::clientAccessCheck()
 385 {
 386     acl_checklist =
 387         clientAclChecklistCreate(Config.accessList.http, http);
 388     acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 389 }
 390
 391 void
 392 clientAccessCheckDoneWrapper(int answer, void *data)
 393 {
 394     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 395
 396     if (!calloutContext->httpStateIsValid())
 397         return;
 398
 399     calloutContext->clientAccessCheckDone(answer);
 400 }
 401
 402 void
 403 ClientRequestContext::clientAccessCheckDone(int answer)
 404 {
 405     acl_checklist = NULL;
 406     err_type page_id;
 407     http_status status;
 408     debug(85, 2) ("The request %s %s is %s, because it matched '%s'\n",
 409                   RequestMethodStr[http->request->method], http->uri,
 410                   answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED",
 411                   AclMatchedName ? AclMatchedName : "NO ACL's");
 412     char const *proxy_auth_msg = "<null>";
 413
 414     if (http->getConn().getRaw() != NULL && http->getConn()->auth_user_request != NULL)
 415         proxy_auth_msg = http->getConn()->auth_user_request->denyMessage("<null>");
 416     else if (http->request->auth_user_request != NULL)
 417         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 418
 419     if (answer != ACCESS_ALLOWED) {
 420         /* Send an error */
 421         debug(85, 5) ("Access Denied: %s\n", http->uri);
 422         debug(85, 5) ("AclMatchedName = %s\n",
 423                       AclMatchedName ? AclMatchedName : "<null>");
 424         debug(85, 5) ("Proxy Auth Message = %s\n",
 425                       proxy_auth_msg ? proxy_auth_msg : "<null>");
 426         /*
 427          * NOTE: get page_id here, based on AclMatchedName because if
 428          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 429          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 430          * <pribeiro@isel.pt>
 431          */
 432         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName);
 433         http->logType = LOG_TCP_DENIED;
 434
 435         if (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName)) {
 436             if (!http->flags.accel) {
 437                 /* Proxy authorisation needed */
 438                 status = HTTP_PROXY_AUTHENTICATION_REQUIRED;
 439             } else {
 440                 /* WWW authorisation needed */
 441                 status = HTTP_UNAUTHORIZED;
 442             }
 443
 444             if (page_id == ERR_NONE)
 445                 page_id = ERR_CACHE_ACCESS_DENIED;
 446         } else {
 447             status = HTTP_FORBIDDEN;
 448
 449             if (page_id == ERR_NONE)
 450                 page_id = ERR_ACCESS_DENIED;
 451         }
 452
 453         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 454         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 455         assert (repContext);
 456         repContext->setReplyToError(page_id, status,
 457                                     http->request->method, NULL,
 458                                     http->getConn().getRaw() != NULL ? &http->getConn()->peer.sin_addr : &no_addr, http->request,
 459                                     NULL, http->getConn().getRaw() != NULL
 460                                     && http->getConn()->auth_user_request ? http->getConn()->
 461                                     auth_user_request : http->request->auth_user_request);
 462         node = (clientStreamNode *)http->client_stream.tail->data;
 463         clientStreamRead(node, http, node->readBuffer);
 464         return;
 465     }
 466
 467     /* ACCESS_ALLOWED continues here ... */
 468     safe_free(http->uri);
 469
 470     http->uri = xstrdup(urlCanonical(http->request));
 471
 472     http->doCallouts();
 473 }
 474
 475 #if ICAP_CLIENT
 476 void
 477 ClientRequestContext::icapAccessCheck()
 478 {
 479     ICAPAccessCheck *icap_access_check;
 480
 481     icap_access_check = new ICAPAccessCheck(ICAP::methodReqmod, ICAP::pointPreCache, http->request, NULL, icapAclCheckDoneWrapper, this);
 482
 483     if (icap_access_check != NULL) {
 484         icap_access_check->check();
 485         return;
 486     }
 487
 488     http->doCallouts();
 489 }
 490
 491 static void
 492 icapAclCheckDoneWrapper(ICAPServiceRep::Pointer service, void *data)
 493 {
 494     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 495
 496     if (!calloutContext->httpStateIsValid())
 497         return;
 498
 499     calloutContext->icapAclCheckDone(service);
 500 }
 501
 502 void
 503 ClientRequestContext::icapAclCheckDone(ICAPServiceRep::Pointer service)
 504 {
 505     debugs(93,3,HERE << this << " icapAclCheckDone called");
 506     /*
 507      * No matching ICAP service in the config file
 508      */
 509
 510     if (service == NULL) {
 511         http->doCallouts();
 512         return;
 513     }
 514
 515     /*
 516      * Setup ICAP state and such.  If successful, just return.
 517      * We'll get back to doCallouts() after REQMOD is done.
 518      */
 519     assert(http);
 520
 521     if (0 == http->doIcap(service))
 522         return;
 523
 524     /*
 525      * If doIcap() fails, then we have to either return an error
 526      * to the user, or keep going without ICAP.
 527      */
 528     fatal("Fix this case in ClientRequestContext::icapAclCheckDone()");
 529     // And when fixed, check whether the service is down in doIcap and
 530     // if it is, abort early, without creating ICAPClientReqmodPrecache.
 531     // See Server::startIcap() and its use.
 532
 533     http->doCallouts();
 534 }
 535
 536 #endif
 537
 538 static void
 539 clientRedirectAccessCheckDone(int answer, void *data)
 540 {
 541     ClientRequestContext *context = (ClientRequestContext *)data;
 542     ClientHttpRequest *http = context->http;
 543     context->acl_checklist = NULL;
 544
 545     if (answer == ACCESS_ALLOWED)
 546         redirectStart(http, clientRedirectDoneWrapper, context);
 547     else
 548         context->clientRedirectDone(NULL);
 549 }
 550
 551 void
 552 ClientRequestContext::clientRedirectStart()
 553 {
 554     debug(33, 5) ("clientRedirectStart: '%s'\n", http->uri);
 555
 556     if (Config.accessList.redirector) {
 557         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 558         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 559     } else
 560         redirectStart(http, clientRedirectDoneWrapper, this);
 561 }
 562
 563 static int
 564 clientHierarchical(ClientHttpRequest * http)
 565 {
 566     const char *url = http->uri;
 567     HttpRequest *request = http->request;
 568     method_t method = request->method;
 569     const wordlist *p = NULL;
 570
 571     /*
 572      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 573      * neighbors support private keys
 574      */
 575
 576     if (request->flags.ims && !neighbors_do_private_keys)
 577         return 0;
 578
 579     /*
 580      * This is incorrect: authenticating requests can be sent via a hierarchy
 581      * (they can even be cached if the correct headers are set on the reply)
 582      */
 583     if (request->flags.auth)
 584         return 0;
 585
 586     if (method == METHOD_TRACE)
 587         return 1;
 588
 589     if (method != METHOD_GET)
 590         return 0;
 591
 592     /* scan hierarchy_stoplist */
 593     for (p = Config.hierarchy_stoplist; p; p = p->next)
 594         if (strstr(url, p->key))
 595             return 0;
 596
 597     if (request->flags.loopdetect)
 598         return 0;
 599
 600     if (request->protocol == PROTO_HTTP)
 601         return httpCachable(method);
 602
 603     if (request->protocol == PROTO_GOPHER)
 604         return gopherCachable(request);
 605
 606     if (request->protocol == PROTO_WAIS)
 607         return 0;
 608
 609     if (request->protocol == PROTO_CACHEOBJ)
 610         return 0;
 611
 612     return 1;
 613 }
 614
 615
 616 static void
 617 clientInterpretRequestHeaders(ClientHttpRequest * http)
 618 {
 619     HttpRequest *request = http->request;
 620     HttpHeader *req_hdr = &request->header;
 621     int no_cache = 0;
 622 #if !(ESI) || defined(USE_USERAGENT_LOG) || defined(USE_REFERER_LOG)
 623
 624     const char *str;
 625 #endif
 626
 627     request->imslen = -1;
 628     request->ims = req_hdr->getTime(HDR_IF_MODIFIED_SINCE);
 629
 630     if (request->ims > 0)
 631         request->flags.ims = 1;
 632
 633 #if ESI
 634     /*
 635      * We ignore Cache-Control as per the Edge Architecture Section 3. See
 636      * www.esi.org for more information.
 637      */
 638 #else
 639
 640     if (req_hdr->has(HDR_PRAGMA)) {
 641         String s = req_hdr->getList(HDR_PRAGMA);
 642
 643         if (strListIsMember(&s, "no-cache", ','))
 644             no_cache++;
 645
 646         s.clean();
 647     }
 648
 649     if (request->cache_control)
 650         if (EBIT_TEST(request->cache_control->mask, CC_NO_CACHE))
 651             no_cache++;
 652
 653     /*
 654     * Work around for supporting the Reload button in IE browsers when Squid
 655     * is used as an accelerator or transparent proxy, by turning accelerated
 656     * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
 657     * actually only fixed in SP1, but we can't tell whether we are talking to
 658     * SP1 or not so all 5.5 versions are treated 'normally').
 659     */
 660     if (Config.onoff.ie_refresh) {
 661         if (http->flags.accel && request->flags.ims) {
 662             if ((str = req_hdr->getStr(HDR_USER_AGENT))) {
 663                 if (strstr(str, "MSIE 5.01") != NULL)
 664                     no_cache++;
 665                 else if (strstr(str, "MSIE 5.0") != NULL)
 666                     no_cache++;
 667                 else if (strstr(str, "MSIE 4.") != NULL)
 668                     no_cache++;
 669                 else if (strstr(str, "MSIE 3.") != NULL)
 670                     no_cache++;
 671             }
 672         }
 673     }
 674
 675 #endif
 676     if (no_cache) {
 677 #if HTTP_VIOLATIONS
 678
 679         if (Config.onoff.reload_into_ims)
 680             request->flags.nocache_hack = 1;
 681         else if (refresh_nocache_hack)
 682             request->flags.nocache_hack = 1;
 683         else
 684 #endif
 685
 686             request->flags.nocache = 1;
 687     }
 688
 689     /* ignore range header in non-GETs or non-HEADs */
 690     if (request->method == METHOD_GET || request->method == METHOD_HEAD) {
 691         request->range = req_hdr->getRange();
 692
 693         if (request->range) {
 694             request->flags.range = 1;
 695             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
 696             /* XXX: This is suboptimal. We should give the stream the range set,
 697              * and thereby let the top of the stream set the offset when the
 698              * size becomes known. As it is, we will end up requesting from 0
 699              * for evey -X range specification.
 700              * RBC - this may be somewhat wrong. We should probably set the range
 701              * iter up at this point.
 702              */
 703             node->readBuffer.offset = request->range->lowestOffset(0);
 704             http->range_iter.pos = request->range->begin();
 705             http->range_iter.valid = true;
 706         }
 707     }
 708
 709     /* Only HEAD and GET requests permit a Range or Request-Range header.
 710      * If these headers appear on any other type of request, delete them now.
 711      */
 712     else {
 713         req_hdr->delById(HDR_RANGE);
 714         req_hdr->delById(HDR_REQUEST_RANGE);
 715         request->range = NULL;
 716     }
 717
 718     if (req_hdr->has(HDR_AUTHORIZATION))
 719         request->flags.auth = 1;
 720
 721     if (request->login[0] != '\0')
 722         request->flags.auth = 1;
 723
 724     if (req_hdr->has(HDR_VIA)) {
 725         String s = req_hdr->getList(HDR_VIA);
 726         /*
 727          * ThisCache cannot be a member of Via header, "1.0 ThisCache" can.
 728          * Note ThisCache2 has a space prepended to the hostname so we don't
 729          * accidentally match super-domains.
 730          */
 731
 732         if (strListIsSubstr(&s, ThisCache2, ',')) {
 733             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
 734                      request, (ObjPackMethod) & httpRequestPack);
 735             request->flags.loopdetect = 1;
 736         }
 737
 738 #if FORW_VIA_DB
 739         fvdbCountVia(s.buf());
 740
 741 #endif
 742
 743         s.clean();
 744     }
 745
 746 #if USE_USERAGENT_LOG
 747     if ((str = req_hdr->getStr(HDR_USER_AGENT)))
 748         logUserAgent(fqdnFromAddr(http->getConn().getRaw() ? http->getConn()->log_addr : no_addr), str);
 749
 750 #endif
 751 #if USE_REFERER_LOG
 752
 753     if ((str = req_hdr->getStr(HDR_REFERER)))
 754         logReferer(fqdnFromAddr(http->getConn().getRaw() ? http->getConn()->log_addr : no_addr), str, http->log_uri);
 755
 756 #endif
 757 #if FORW_VIA_DB
 758
 759     if (req_hdr->has(HDR_X_FORWARDED_FOR)) {
 760         String s = req_hdr->getList(HDR_X_FORWARDED_FOR);
 761         fvdbCountForw(s.buf());
 762         s.clean();
 763     }
 764
 765 #endif
 766     if (request->method == METHOD_TRACE) {
 767         request->max_forwards = req_hdr->getInt(HDR_MAX_FORWARDS);
 768     }
 769
 770     request->flags.cachable = http->request->cacheable();
 771
 772     if (clientHierarchical(http))
 773         request->flags.hierarchical = 1;
 774
 775     debug(85, 5) ("clientInterpretRequestHeaders: REQ_NOCACHE = %s\n",
 776                   request->flags.nocache ? "SET" : "NOT SET");
 777
 778     debug(85, 5) ("clientInterpretRequestHeaders: REQ_CACHABLE = %s\n",
 779                   request->flags.cachable ? "SET" : "NOT SET");
 780
 781     debug(85, 5) ("clientInterpretRequestHeaders: REQ_HIERARCHICAL = %s\n",
 782                   request->flags.hierarchical ? "SET" : "NOT SET");
 783 }
 784
 785 void
 786 clientRedirectDoneWrapper(void *data, char *result)
 787 {
 788     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 789
 790     if (!calloutContext->httpStateIsValid())
 791         return;
 792
 793     calloutContext->clientRedirectDone(result);
 794 }
 795
 796 void
 797 ClientRequestContext::clientRedirectDone(char *result)
 798 {
 799     HttpRequest *new_request = NULL;
 800     HttpRequest *old_request = http->request;
 801     debug(85, 5) ("clientRedirectDone: '%s' result=%s\n", http->uri,
 802                   result ? result : "NULL");
 803     assert(redirect_state == REDIRECT_PENDING);
 804     redirect_state = REDIRECT_DONE;
 805
 806     if (result) {
 807         http_status status = (http_status) atoi(result);
 808
 809         if (status == HTTP_MOVED_PERMANENTLY
 810                 || status == HTTP_MOVED_TEMPORARILY
 811                 || status == HTTP_SEE_OTHER
 812                 || status == HTTP_TEMPORARY_REDIRECT) {
 813             char *t = result;
 814
 815             if ((t = strchr(result, ':')) != NULL) {
 816                 http->redirect.status = status;
 817                 http->redirect.location = xstrdup(t + 1);
 818             } else {
 819                 debug(85, 1) ("clientRedirectDone: bad input: %s\n", result);
 820             }
 821         } else if (strcmp(result, http->uri))
 822             new_request = HttpRequest::CreateFromUrlAndMethod(result, old_request->method);
 823     }
 824
 825     if (new_request) {
 826         safe_free(http->uri);
 827         http->uri = xstrdup(urlCanonical(new_request));
 828         new_request->http_ver = old_request->http_ver;
 829         new_request->header.append(&old_request->header);
 830         new_request->client_addr = old_request->client_addr;
 831         new_request->client_port = old_request->client_port;
 832         new_request->my_addr = old_request->my_addr;
 833         new_request->my_port = old_request->my_port;
 834         new_request->flags = old_request->flags;
 835         new_request->flags.redirected = 1;
 836
 837         if (old_request->auth_user_request) {
 838             new_request->auth_user_request = old_request->auth_user_request;
 839
 840             new_request->auth_user_request->lock()
 841
 842             ;
 843         }
 844
 845         if (old_request->body_reader != NULL) {
 846             new_request->body_reader = old_request->body_reader;
 847             old_request->body_reader = NULL;
 848             debugs(0,0,HERE << "setting body_reader = NULL for request " << old_request);
 849         }
 850
 851         new_request->content_length = old_request->content_length;
 852         new_request->extacl_user = old_request->extacl_user;
 853         new_request->extacl_passwd = old_request->extacl_passwd;
 854         new_request->flags.proxy_keepalive = old_request->flags.proxy_keepalive;
 855         HTTPMSGUNLOCK(old_request);
 856         http->request = HTTPMSGLOCK(new_request);
 857     }
 858
 859     /* FIXME PIPELINE: This is innacurate during pipelining */
 860
 861     if (http->getConn().getRaw() != NULL)
 862         fd_note(http->getConn()->fd, http->uri);
 863
 864     assert(http->uri);
 865
 866     http->doCallouts();
 867 }
 868
 869 void
 870 ClientRequestContext::checkNoCache()
 871 {
 872     acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
 873     acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
 874 }
 875
 876 static void
 877 checkNoCacheDoneWrapper(int answer, void *data)
 878 {
 879     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 880
 881     if (!calloutContext->httpStateIsValid())
 882         return;
 883
 884     calloutContext->checkNoCacheDone(answer);
 885 }
 886
 887 void
 888 ClientRequestContext::checkNoCacheDone(int answer)
 889 {
 890     acl_checklist = NULL;
 891     http->request->flags.cachable = answer;
 892     http->doCallouts();
 893 }
 894
 895 /*
 896  * Identify requests that do not go through the store and client side stream
 897  * and forward them to the appropriate location. All other requests, request
 898  * them.
 899  */
 900 void
 901 ClientHttpRequest::processRequest()
 902 {
 903     debug(85, 4) ("clientProcessRequest: %s '%s'\n",
 904                   RequestMethodStr[request->method], uri);
 905
 906     if (request->method == METHOD_CONNECT && !redirect.status) {
 907         logType = LOG_TCP_MISS;
 908         sslStart(this, &out.size, &al.http.code);
 909         return;
 910     }
 911
 912     httpStart();
 913 }
 914
 915 void
 916 ClientHttpRequest::httpStart()
 917 {
 918     PROF_start(httpStart);
 919     logType = LOG_TAG_NONE;
 920     debug(85, 4) ("ClientHttpRequest::httpStart: %s for '%s'\n",
 921                   log_tags[logType], uri);
 922     /* no one should have touched this */
 923     assert(out.offset == 0);
 924     /* Use the Stream Luke */
 925     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
 926     clientStreamRead(node, this, node->readBuffer);
 927     PROF_stop(httpStart);
 928 }
 929
 930 bool
 931 ClientHttpRequest::gotEnough() const
 932 {
 933     /** TODO: should be querying the stream. */
 934     int contentLength =
 935         memObject()->getReply()->bodySize(request->method);
 936     assert(contentLength >= 0);
 937
 938     if (out.offset < contentLength)
 939         return false;
 940
 941     return true;
 942 }
 943
 944 void
 945 ClientHttpRequest::maxReplyBodySize(ssize_t clen)
 946 {
 947     maxReplyBodySize_ = clen;
 948 }
 949
 950 ssize_t
 951 ClientHttpRequest::maxReplyBodySize() const
 952 {
 953     return maxReplyBodySize_;
 954 }
 955
 956 bool
 957 ClientHttpRequest::isReplyBodyTooLarge(ssize_t clen) const
 958 {
 959     if (0 == maxReplyBodySize())
 960         return 0;       /* disabled */
 961
 962     if (clen < 0)
 963         return 0;       /* unknown */
 964
 965     return clen > maxReplyBodySize();
 966 }
 967
 968 void
 969 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
 970 {
 971     entry_ = newEntry;
 972 }
 973
 974 void
 975 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
 976 {
 977     if (loggingEntry_)
 978         loggingEntry_->unlock();
 979
 980     loggingEntry_ = newEntry;
 981
 982     if (loggingEntry_)
 983         loggingEntry_->lock()
 984
 985         ;
 986 }
 987
 988 /*
 989  * doCallouts() - This function controls the order of "callout"
 990  * executions, including non-blocking access control checks, the
 991  * redirector, and ICAP.  Previously, these callouts were chained
 992  * together such that "clientAccessCheckDone()" would call
 993  * "clientRedirectStart()" and so on.
 994  *
 995  * The ClientRequestContext (aka calloutContext) class holds certain
 996  * state data for the callout/callback operations.  Previously
 997  * ClientHttpRequest would sort of hand off control to ClientRequestContext
 998  * for a short time.  ClientRequestContext would then delete itself
 999  * and pass control back to ClientHttpRequest when all callouts
1000  * were finished.
1001  *
1002  * This caused some problems for ICAP because we want to make the
1003  * ICAP callout after checking ACLs, but before checking the no_cache
1004  * list.  We can't stuff the ICAP state into the ClientRequestContext
1005  * class because we still need the ICAP state after ClientRequestContext
1006  * goes away.
1007  *
1008  * Note that ClientRequestContext is created before the first call
1009  * to doCallouts().
1010  *
1011  * If one of the callouts notices that ClientHttpRequest is no
1012  * longer valid, it should call cbdataReferenceDone() so that
1013  * ClientHttpRequest's reference count goes to zero and it will get
1014  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1015  *
1016  * Note that we set the _done flags here before actually starting
1017  * the callout.  This is strictly for convenience.
1018  */
1019
1020 void
1021 ClientHttpRequest::doCallouts()
1022 {
1023     assert(calloutContext);
1024
1025     if (!calloutContext->http_access_done) {
1026         calloutContext->http_access_done = true;
1027         calloutContext->clientAccessCheck();
1028         return;
1029     }
1030
1031 #if ICAP_CLIENT
1032     if (TheICAPConfig.onoff && !calloutContext->icap_acl_check_done) {
1033         calloutContext->icap_acl_check_done = true;
1034         calloutContext->icapAccessCheck();
1035         return;
1036     }
1037
1038 #endif
1039
1040     if (!calloutContext->redirect_done) {
1041         calloutContext->redirect_done = true;
1042         assert(calloutContext->redirect_state == REDIRECT_NONE);
1043
1044         if (Config.Program.redirect) {
1045             calloutContext->redirect_state = REDIRECT_PENDING;
1046             calloutContext->clientRedirectStart();
1047             return;
1048         }
1049     }
1050
1051     if (!calloutContext->interpreted_req_hdrs) {
1052         calloutContext->interpreted_req_hdrs = 1;
1053         clientInterpretRequestHeaders(this);
1054     }
1055
1056     if (!calloutContext->no_cache_done) {
1057         calloutContext->no_cache_done = true;
1058
1059         if (Config.accessList.noCache && request->flags.cachable) {
1060             calloutContext->checkNoCache();
1061             return;
1062         }
1063     }
1064
1065     cbdataReferenceDone(calloutContext->http);
1066     delete calloutContext;
1067     calloutContext = NULL;
1068 #if HEADERS_LOG
1069
1070     headersLog(0, 1, request->method, request);
1071 #endif
1072
1073     processRequest();
1074 }
1075
1076 #ifndef _USE_INLINE_
1077 #include "client_side_request.cci"
1078 #endif
1079
1080 #if ICAP_CLIENT
1081 /*
1082  * Initiate an ICAP transaction.  Return 0 if all is well, or -1 upon error.
1083  * Caller will handle error condition by generating a Squid error message
1084  * or take other action.
1085  */
1086 int
1087 ClientHttpRequest::doIcap(ICAPServiceRep::Pointer service)
1088 {
1089     debugs(85, 3, HERE << this << " ClientHttpRequest::doIcap() called");
1090     assert(NULL == icap);
1091     icap = new ICAPClientReqmodPrecache(service);
1092     icap->startReqMod(this, request);
1093
1094     if (request->body_reader == NULL) {
1095         debugs(32, 3, HERE << "client request hasnt body...");
1096         icap->doneSending();
1097
1098     }
1099
1100     return 0;
1101 }
1102
1103 /*
1104  * icapSendRequestBodyWrapper
1105  *
1106  * A callback wrapper for ::icapSendRequestBody()
1107  *
1108  * icapSendRequestBodyWrapper is of type CBCB
1109  */
1110 void
1111 ClientHttpRequest::icapSendRequestBodyWrapper(MemBuf &mb, void *data)
1112 {
1113     ClientHttpRequest *chr = static_cast<ClientHttpRequest*>(data);
1114     chr->icapSendRequestBody(mb);
1115 }
1116
1117
1118 /*
1119  * icapSendRequestBody
1120  *
1121  * Sends some chunk of a request body to the ICAP side.  Must make sure
1122  * that the ICAP-side can accept the data we have.  If there is more
1123  * body data to read, then schedule another BodyReader callback.
1124  */
1125 void
1126 ClientHttpRequest::icapSendRequestBody(MemBuf &mb)
1127 {
1128     ssize_t size_to_send  = mb.contentSize();
1129     debugs(32,3,HERE << "have " << mb.contentSize() << " bytes in mb");
1130
1131     if (size_to_send == 0) {
1132         /*
1133          * An error occurred during this transaction.  Tell ICAP that we're done.
1134          */
1135
1136         if (icap)
1137             icap->doneSending();
1138
1139         return;
1140     }
1141
1142     debugs(32,3,HERE << "icap->potentialSpaceSize() = " << icap->potentialSpaceSize());
1143
1144     if (size_to_send > icap->potentialSpaceSize())
1145         size_to_send = icap->potentialSpaceSize();
1146
1147     if (size_to_send) {
1148         debugs(32,3,HERE << "sending " << size_to_send << " body bytes to ICAP");
1149         StoreIOBuffer sbuf(size_to_send, 0, mb.content());
1150         icap->sendMoreData(sbuf);
1151         icap->body_reader->consume(size_to_send);
1152         icap->body_reader->bytes_read += size_to_send;
1153         debugs(32,3," HTTP client body bytes_read=" << icap->body_reader->bytes_read);
1154     } else {
1155         debugs(32,2,HERE << "cannot send body data to ICAP");
1156         debugs(32,2,HERE << "\tBodyReader MemBuf has " << mb.contentSize());
1157         debugs(32,2,HERE << "\tbut icap->potentialSpaceSize() is " << icap->potentialSpaceSize());
1158         return;
1159     }
1160
1161     /*
1162      * If we sent some data this time, and there is more data to
1163      * read, then schedule another read request via BodyReader.
1164      */
1165     if (size_to_send && icap->body_reader->remaining()) {
1166         debugs(32,3,HERE << "calling body_reader->read()");
1167         icap->body_reader->read(icapSendRequestBodyWrapper, this);
1168     } else {
1169         debugs(32,3,HERE << "No more request body bytes to send");
1170         icap->doneSending();
1171     }
1172 }
1173
1174 /*
1175  * Called by ICAPAnchor when it has space available for us.
1176  */
1177 void
1178 ClientHttpRequest::icapSpaceAvailable()
1179 {
1180     debugs(85,3,HERE << this << " ClientHttpRequest::icapSpaceAvailable() called\n");
1181
1182     if (request->body_reader != NULL && icap->body_reader == NULL) {
1183         debugs(32,3,HERE << "reassigning HttpRequest->body_reader to ICAP");
1184         /*
1185          * ICAP hooks on to the BodyReader that gets data from
1186          * ConnStateData.  We'll make a new BodyReader that
1187          * HttpStateData can use if the adapted response has a
1188          * request body.  See ICAPClientReqmodPrecache::noteSourceStart()
1189          */
1190         icap->body_reader = request->body_reader;
1191         request->body_reader = NULL;
1192     }
1193
1194     if (icap->body_reader == NULL)
1195         return;
1196
1197     if (icap->body_reader->callbackPending())
1198         return;
1199
1200     debugs(32,3,HERE << "Calling read() for body data");
1201
1202     icap->body_reader->read(icapSendRequestBodyWrapper, this);
1203 }
1204
1205 void
1206 ClientHttpRequest::takeAdaptedHeaders(HttpMsg *msg)
1207 {
1208     debug(85,3)("ClientHttpRequest::takeAdaptedHeaders() called\n");
1209     assert(cbdataReferenceValid(this));         // indicates bug
1210
1211     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1212         /*
1213          * Replace the old request with the new request.  First,
1214          * Move the "body_connection" over, then unlink old and
1215          * link new to the http state.
1216          */
1217         HTTPMSGUNLOCK(request);
1218         request = HTTPMSGLOCK(new_req);
1219         /*
1220          * Store the new URI for logging
1221          */
1222         xfree(uri);
1223         uri = xstrdup(urlCanonical(request));
1224         setLogUri(this, urlCanonicalClean(request));
1225         assert(request->method);
1226     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1227         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1228
1229         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1230         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1231         repContext->createStoreEntry(request->method, request->flags);
1232
1233         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1234         request_satisfaction_mode = true;
1235         request_satisfaction_offset = 0;
1236         storeEntry()->replaceHttpReply(new_rep);
1237         clientGetMoreData(node, this);
1238     }
1239
1240     if (!request_satisfaction_mode)
1241         doCallouts();
1242
1243     debug(85,3)("ClientHttpRequest::takeAdaptedHeaders() finished\n");
1244 }
1245
1246 void
1247 ClientHttpRequest::takeAdaptedBody(MemBuf *buf)
1248 {
1249     debug(85,3)("ClientHttpRequest::takeAdaptedBody() called\n");
1250
1251     if (request_satisfaction_mode) {
1252         storeEntry()->write(StoreIOBuffer(buf, request_satisfaction_offset));
1253         request_satisfaction_offset += buf->contentSize();
1254         buf->consume(buf->contentSize()); // consume everything written
1255     } else {
1256         debug(85,0)("Unexpected call to takeAdaptedBody when "
1257                     "not in request_satisfaction_mode");
1258     }
1259 }
1260
1261 void
1262 ClientHttpRequest::doneAdapting()
1263 {
1264     debug(85,3)("ClientHttpRequest::doneAdapting() called\n");
1265 }
1266
1267 void
1268 ClientHttpRequest::abortAdapting()
1269 {
1270     debug(85,3)("ClientHttpRequest::abortAdapting() called\n");
1271
1272     if ((NULL == storeEntry()) || storeEntry()->isEmpty()) {
1273         debug(85,3)("WARNING: ICAP REQMOD callout failed, proceeding with original request\n");
1274
1275         if (calloutContext)
1276             doCallouts();
1277
1278 #if ICAP_HARD_ERROR
1279
1280         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1281
1282         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1283
1284         assert (repContext);
1285
1286         // Note if this code is ever used, clientBuildError() should be modified to
1287         // accept an errno arg
1288         repContext->setReplyToError(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR,
1289                                     request->method, NULL,
1290                                     getConn().getRaw() != NULL ? &getConn()->peer.sin_addr : &no_addr, request,
1291                                     NULL, getConn().getRaw() != NULL
1292                                     && getConn()->auth_user_request ? getConn()->
1293                                     auth_user_request : request->auth_user_request, errno);
1294
1295         node = (clientStreamNode *)client_stream.tail->data;
1296
1297         clientStreamRead(node, this, node->readBuffer);
1298
1299 #endif
1300
1301         return;
1302     }
1303
1304     debug(0,0)("write me at %s:%d\n", __FILE__,__LINE__);
1305 }
1306
1307 #endif