src/client_side_request.cc

   1
   2 /*
   3  * $Id$
   4  *
   5  * DEBUG: section 85    Client-side Request Routines
   6  * AUTHOR: Robert Collins (Originally Duane Wessels in client_side.c)
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36
  37 /*
  38  * General logic of request processing:
  39  *
  40  * We run a series of tests to determine if access will be permitted, and to do
  41  * any redirection. Then we call into the result clientStream to retrieve data.
  42  * From that point on it's up to reply management.
  43  */
  44
  45 #include "squid.h"
  46 #include "clientStream.h"
  47 #include "client_side_request.h"
  48 #include "auth/UserRequest.h"
  49 #include "HttpRequest.h"
  50 #include "ProtoPort.h"
  51 #include "acl/FilledChecklist.h"
  52 #include "acl/Gadgets.h"
  53 #include "client_side.h"
  54 #include "client_side_reply.h"
  55 #include "Store.h"
  56 #include "HttpReply.h"
  57 #include "MemObject.h"
  58 #include "ClientRequestContext.h"
  59 #include "SquidTime.h"
  60 #include "wordlist.h"
  61 #include "inet_pton.h"
  62 #include "fde.h"
  63
  64 #if USE_ADAPTATION
  65 #include "adaptation/AccessCheck.h"
  66 #include "adaptation/Iterator.h"
  67 #include "adaptation/Service.h"
  68 #if ICAP_CLIENT
  69 #include "adaptation/icap/History.h"
  70 #endif
  71 //static void adaptationAclCheckDoneWrapper(Adaptation::ServicePointer service, void *data);
  72 #endif
  73
  74
  75
  76 #if LINGERING_CLOSE
  77 #define comm_close comm_lingering_close
  78 #endif
  79
  80 static const char *const crlf = "\r\n";
  81
  82 #if FOLLOW_X_FORWARDED_FOR
  83 static void
  84 clientFollowXForwardedForCheck(int answer, void *data);
  85 #endif /* FOLLOW_X_FORWARDED_FOR */
  86
  87 CBDATA_CLASS_INIT(ClientRequestContext);
  88
  89 void *
  90 ClientRequestContext::operator new (size_t size)
  91 {
  92     assert (size == sizeof(ClientRequestContext));
  93     CBDATA_INIT_TYPE(ClientRequestContext);
  94     ClientRequestContext *result = cbdataAlloc(ClientRequestContext);
  95     return result;
  96 }
  97
  98 void
  99 ClientRequestContext::operator delete (void *address)
 100 {
 101     ClientRequestContext *t = static_cast<ClientRequestContext *>(address);
 102     cbdataFree(t);
 103 }
 104
 105 /* Local functions */
 106 /* other */
 107 static void clientAccessCheckDoneWrapper(int, void *);
 108 static int clientHierarchical(ClientHttpRequest * http);
 109 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
 110 static RH clientRedirectDoneWrapper;
 111 static PF checkNoCacheDoneWrapper;
 112 extern "C" CSR clientGetMoreData;
 113 extern "C" CSS clientReplyStatus;
 114 extern "C" CSD clientReplyDetach;
 115 static void checkFailureRatio(err_type, hier_code);
 116
 117 ClientRequestContext::~ClientRequestContext()
 118 {
 119     /*
 120      * Release our "lock" on our parent, ClientHttpRequest, if we
 121      * still have one
 122      */
 123
 124     if (http)
 125         cbdataReferenceDone(http);
 126
 127     debugs(85,3, HERE << this << " ClientRequestContext destructed");
 128 }
 129
 130 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE)
 131 {
 132     http_access_done = false;
 133     redirect_done = false;
 134     no_cache_done = false;
 135     interpreted_req_hdrs = false;
 136     debugs(85,3, HERE << this << " ClientRequestContext constructed");
 137 }
 138
 139 CBDATA_CLASS_INIT(ClientHttpRequest);
 140
 141 void *
 142 ClientHttpRequest::operator new (size_t size)
 143 {
 144     assert (size == sizeof (ClientHttpRequest));
 145     CBDATA_INIT_TYPE(ClientHttpRequest);
 146     ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest);
 147     return result;
 148 }
 149
 150 void
 151 ClientHttpRequest::operator delete (void *address)
 152 {
 153     ClientHttpRequest *t = static_cast<ClientHttpRequest *>(address);
 154     cbdataFree(t);
 155 }
 156
 157 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 158 #if USE_ADAPTATION
 159         AsyncJob("ClientHttpRequest"),
 160 #endif
 161         loggingEntry_(NULL)
 162 {
 163     start_time = current_time;
 164     setConn(aConn);
 165     dlinkAdd(this, &active, &ClientActiveRequests);
 166 #if USE_ADAPTATION
 167     request_satisfaction_mode = false;
 168 #endif
 169 }
 170
 171 /*
 172  * returns true if client specified that the object must come from the cache
 173  * without contacting origin server
 174  */
 175 bool
 176 ClientHttpRequest::onlyIfCached()const
 177 {
 178     assert(request);
 179     return request->cache_control &&
 180            EBIT_TEST(request->cache_control->mask, CC_ONLY_IF_CACHED);
 181 }
 182
 183 /*
 184  * This function is designed to serve a fairly specific purpose.
 185  * Occasionally our vBNS-connected caches can talk to each other, but not
 186  * the rest of the world.  Here we try to detect frequent failures which
 187  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 188  * the failure:success ratio goes above 1.0 then we go into "hit only"
 189  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 190  * will only fetch HITs from us if they are using the ICP protocol.  We
 191  * stay in this mode for 5 minutes.
 192  *
 193  * Duane W., Sept 16, 1996
 194  */
 195
 196 #define FAILURE_MODE_TIME 300
 197
 198 static void
 199 checkFailureRatio(err_type etype, hier_code hcode)
 200 {
 201     static double magic_factor = 100.0;
 202     double n_good;
 203     double n_bad;
 204
 205     if (hcode == HIER_NONE)
 206         return;
 207
 208     n_good = magic_factor / (1.0 + request_failure_ratio);
 209
 210     n_bad = magic_factor - n_good;
 211
 212     switch (etype) {
 213
 214     case ERR_DNS_FAIL:
 215
 216     case ERR_CONNECT_FAIL:
 217     case ERR_SECURE_CONNECT_FAIL:
 218
 219     case ERR_READ_ERROR:
 220         n_bad++;
 221         break;
 222
 223     default:
 224         n_good++;
 225     }
 226
 227     request_failure_ratio = n_bad / n_good;
 228
 229     if (hit_only_mode_until > squid_curtime)
 230         return;
 231
 232     if (request_failure_ratio < 1.0)
 233         return;
 234
 235     debugs(33, 0, "Failure Ratio at "<< std::setw(4)<<
 236            std::setprecision(3) << request_failure_ratio);
 237
 238     debugs(33, 0, "Going into hit-only-mode for " <<
 239            FAILURE_MODE_TIME / 60 << " minutes...");
 240
 241     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 242
 243     request_failure_ratio = 0.8;        /* reset to something less than 1.0 */
 244 }
 245
 246 ClientHttpRequest::~ClientHttpRequest()
 247 {
 248     debugs(33, 3, "httpRequestFree: " << uri);
 249     PROF_start(httpRequestFree);
 250
 251     // Even though freeResources() below may destroy the request,
 252     // we no longer set request->body_pipe to NULL here
 253     // because we did not initiate that pipe (ConnStateData did)
 254
 255     /* the ICP check here was erroneous
 256      * - StoreEntry::releaseRequest was always called if entry was valid
 257      */
 258     assert(logType < LOG_TYPE_MAX);
 259
 260     logRequest();
 261
 262     loggingEntry(NULL);
 263
 264     if (request)
 265         checkFailureRatio(request->errType, al.hier.code);
 266
 267     freeResources();
 268
 269 #if USE_ADAPTATION
 270     announceInitiatorAbort(virginHeadSource);
 271
 272     if (adaptedBodySource != NULL)
 273         stopConsumingFrom(adaptedBodySource);
 274 #endif
 275
 276     if (calloutContext)
 277         delete calloutContext;
 278
 279     if (conn_)
 280         cbdataReferenceDone(conn_);
 281
 282     /* moving to the next connection is handled by the context free */
 283     dlinkDelete(&active, &ClientActiveRequests);
 284
 285     PROF_stop(httpRequestFree);
 286 }
 287
 288 /* Create a request and kick it off */
 289 /*
 290  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 291  * determined by the user
 292  */
 293 int                             /* returns nonzero on failure */
 294 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 295                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 296                    char *tailbuf, size_t taillen)
 297 {
 298     size_t url_sz;
 299     HttpVersion http_ver (1, 0);
 300     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 301     HttpRequest *request;
 302     StoreIOBuffer tempBuffer;
 303     http->start_time = current_time;
 304     /* this is only used to adjust the connection offset in client_side.c */
 305     http->req_sz = 0;
 306     tempBuffer.length = taillen;
 307     tempBuffer.data = tailbuf;
 308     /* client stream setup */
 309     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 310                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 311                      streamdetach, streamdata, tempBuffer);
 312     /* make it visible in the 'current acctive requests list' */
 313     /* Set flags */
 314     /* internal requests only makes sense in an
 315      * accelerator today. TODO: accept flags ? */
 316     http->flags.accel = 1;
 317     /* allow size for url rewriting */
 318     url_sz = strlen(url) + Config.appendDomainLen + 5;
 319     http->uri = (char *)xcalloc(url_sz, 1);
 320     strcpy(http->uri, url);
 321
 322     if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
 323         debugs(85, 5, "Invalid URL: " << http->uri);
 324         return -1;
 325     }
 326
 327     /*
 328      * now update the headers in request with our supplied headers. urLParse
 329      * should return a blank header set, but we use Update to be sure of
 330      * correctness.
 331      */
 332     if (header)
 333         request->header.update(header, NULL);
 334
 335     http->log_uri = xstrdup(urlCanonicalClean(request));
 336
 337     /* http struct now ready */
 338
 339     /*
 340      * build new header list *? TODO
 341      */
 342     request->flags.accelerated = http->flags.accel;
 343
 344     request->flags.internalclient = 1;
 345
 346     /* this is an internally created
 347      * request, not subject to acceleration
 348      * target overrides */
 349     /*
 350      * FIXME? Do we want to detect and handle internal requests of internal
 351      * objects ?
 352      */
 353
 354     /* Internally created requests cannot have bodies today */
 355     request->content_length = 0;
 356
 357     request->client_addr.SetNoAddr();
 358
 359 #if FOLLOW_X_FORWARDED_FOR
 360     request->indirect_client_addr.SetNoAddr();
 361 #endif /* FOLLOW_X_FORWARDED_FOR */
 362
 363     request->my_addr.SetNoAddr();       /* undefined for internal requests */
 364
 365     request->my_addr.SetPort(0);
 366
 367     request->http_ver = http_ver;
 368
 369     http->request = HTTPMSGLOCK(request);
 370
 371     /* optional - skip the access check ? */
 372     http->calloutContext = new ClientRequestContext(http);
 373
 374     http->calloutContext->http_access_done = false;
 375
 376     http->calloutContext->redirect_done = true;
 377
 378     http->calloutContext->no_cache_done = true;
 379
 380     http->doCallouts();
 381
 382     return 0;
 383 }
 384
 385 bool
 386 ClientRequestContext::httpStateIsValid()
 387 {
 388     ClientHttpRequest *http_ = http;
 389
 390     if (cbdataReferenceValid(http_))
 391         return true;
 392
 393     http = NULL;
 394
 395     cbdataReferenceDone(http_);
 396
 397     return false;
 398 }
 399
 400 #if FOLLOW_X_FORWARDED_FOR
 401 /**
 402  * clientFollowXForwardedForCheck() checks the indirect_client_addr
 403  * against the followXFF ACL, or cleans up and passes control to
 404  * clientAccessCheck().
 405  */
 406
 407 static void
 408 clientFollowXForwardedForCheck(int answer, void *data)
 409 {
 410     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 411     ClientHttpRequest *http = NULL;
 412     HttpRequest *request = NULL;
 413
 414     if (!calloutContext->httpStateIsValid())
 415         return;
 416
 417     http = calloutContext->http;
 418     request = http->request;
 419     /*
 420      * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
 421      * called as a result of ACL checks, or -1 if we are called when
 422      * there's nothing left to do.
 423      */
 424     if (answer == ACCESS_ALLOWED &&
 425             request->x_forwarded_for_iterator.size () != 0) {
 426         /*
 427         * The IP address currently in request->indirect_client_addr
 428         * is trusted to use X-Forwarded-For.  Remove the last
 429         * comma-delimited element from x_forwarded_for_iterator and use
 430         * it to to replace indirect_client_addr, then repeat the cycle.
 431         */
 432         const char *p;
 433         const char *asciiaddr;
 434         int l;
 435         struct in_addr addr;
 436         p = request->x_forwarded_for_iterator.termedBuf();
 437         l = request->x_forwarded_for_iterator.size();
 438
 439         /*
 440         * XXX x_forwarded_for_iterator should really be a list of
 441         * IP addresses, but it's a String instead.  We have to
 442         * walk backwards through the String, biting off the last
 443         * comma-delimited part each time.  As long as the data is in
 444         * a String, we should probably implement and use a variant of
 445         * strListGetItem() that walks backwards instead of forwards
 446         * through a comma-separated list.  But we don't even do that;
 447         * we just do the work in-line here.
 448         */
 449         /* skip trailing space and commas */
 450         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 451             l--;
 452         request->x_forwarded_for_iterator.cut(l);
 453         /* look for start of last item in list */
 454         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 455             l--;
 456         asciiaddr = p+l;
 457         if (xinet_pton(AF_INET, asciiaddr, &addr) != 0) {
 458             request->indirect_client_addr = addr;
 459             request->x_forwarded_for_iterator.cut(l);
 460             if (! Config.onoff.acl_uses_indirect_client) {
 461                 /*
 462                 * If acl_uses_indirect_client is off, then it's impossible
 463                 * to follow more than one level of X-Forwarded-For.
 464                 */
 465                 request->x_forwarded_for_iterator.clean();
 466             }
 467             calloutContext->acl_checklist =
 468                 clientAclChecklistCreate(Config.accessList.followXFF, http);
 469             calloutContext->acl_checklist->
 470             nonBlockingCheck(clientFollowXForwardedForCheck, data);
 471             return;
 472         }
 473     } /*if (answer == ACCESS_ALLOWED &&
 474         request->x_forwarded_for_iterator.size () != 0)*/
 475
 476     /* clean up, and pass control to clientAccessCheck */
 477     if (Config.onoff.log_uses_indirect_client) {
 478         /*
 479         * Ensure that the access log shows the indirect client
 480         * instead of the direct client.
 481         */
 482         ConnStateData *conn = http->getConn();
 483         conn->log_addr = request->indirect_client_addr;
 484     }
 485     request->x_forwarded_for_iterator.clean();
 486     request->flags.done_follow_x_forwarded_for = 1;
 487
 488     /* If follow XFF is denied, we reset the indirect_client_addr
 489        to the direct client. Thats the one we are configured to check for */
 490     if (answer == ACCESS_DENIED) {
 491         request->indirect_client_addr = request->client_addr;
 492     }
 493     /* on a failure, leave it as undefined state ?? */
 494     else if (answer != ACCESS_ALLOWED) {
 495         debugs(28, DBG_CRITICAL, "Follow X-Forwarded-For encountered an error. Ignoring address: " << request->indirect_client_addr );
 496         request->indirect_client_addr = request->client_addr;
 497     }
 498
 499     /* process actual access ACL as normal. */
 500     calloutContext->clientAccessCheck();
 501 }
 502 #endif /* FOLLOW_X_FORWARDED_FOR */
 503
 504 /* This is the entry point for external users of the client_side routines */
 505 void
 506 ClientRequestContext::clientAccessCheck()
 507 {
 508 #if FOLLOW_X_FORWARDED_FOR
 509     if (!http->request->flags.done_follow_x_forwarded_for &&
 510             Config.accessList.followXFF &&
 511             http->request->header.has(HDR_X_FORWARDED_FOR)) {
 512         http->request->x_forwarded_for_iterator =
 513             http->request->header.getList(HDR_X_FORWARDED_FOR);
 514         clientFollowXForwardedForCheck(ACCESS_ALLOWED, this);
 515         return;
 516     }
 517 #endif /* FOLLOW_X_FORWARDED_FOR */
 518
 519     if (Config.accessList.http) {
 520         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 521         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 522     } else {
 523         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 524         clientAccessCheckDone(ACCESS_DENIED);
 525     }
 526 }
 527
 528 void
 529 clientAccessCheckDoneWrapper(int answer, void *data)
 530 {
 531     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 532
 533     if (!calloutContext->httpStateIsValid())
 534         return;
 535
 536     calloutContext->clientAccessCheckDone(answer);
 537 }
 538
 539 void
 540 ClientRequestContext::clientAccessCheckDone(int answer)
 541 {
 542     acl_checklist = NULL;
 543     err_type page_id;
 544     http_status status;
 545     debugs(85, 2, "The request " <<
 546            RequestMethodStr(http->request->method) << " " <<
 547            http->uri << " is " <<
 548            (answer == ACCESS_ALLOWED ? "ALLOWED" : "DENIED") <<
 549            ", because it matched '" <<
 550            (AclMatchedName ? AclMatchedName : "NO ACL's") << "'" );
 551     char const *proxy_auth_msg = "<null>";
 552
 553     if (http->getConn() != NULL && http->getConn()->auth_user_request != NULL)
 554         proxy_auth_msg = http->getConn()->auth_user_request->denyMessage("<null>");
 555     else if (http->request->auth_user_request != NULL)
 556         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 557
 558     if (answer != ACCESS_ALLOWED) {
 559         /* Send an error */
 560         int require_auth = (answer == ACCESS_REQ_PROXY_AUTH || aclIsProxyAuth(AclMatchedName));
 561         debugs(85, 5, "Access Denied: " << http->uri);
 562         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 563
 564         if (require_auth)
 565             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 566
 567         /*
 568          * NOTE: get page_id here, based on AclMatchedName because if
 569          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 570          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 571          * <pribeiro@isel.pt>
 572          */
 573         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_REQ_PROXY_AUTH);
 574
 575         http->logType = LOG_TCP_DENIED;
 576
 577         if (require_auth) {
 578             if (!http->flags.accel) {
 579                 /* Proxy authorisation needed */
 580                 status = HTTP_PROXY_AUTHENTICATION_REQUIRED;
 581             } else {
 582                 /* WWW authorisation needed */
 583                 status = HTTP_UNAUTHORIZED;
 584             }
 585
 586             if (page_id == ERR_NONE)
 587                 page_id = ERR_CACHE_ACCESS_DENIED;
 588         } else {
 589             status = HTTP_FORBIDDEN;
 590
 591             if (page_id == ERR_NONE)
 592                 page_id = ERR_ACCESS_DENIED;
 593         }
 594
 595         clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 596         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 597         assert (repContext);
 598         IpAddress tmpnoaddr;
 599         tmpnoaddr.SetNoAddr();
 600         repContext->setReplyToError(page_id, status,
 601                                     http->request->method, NULL,
 602                                     http->getConn() != NULL ? http->getConn()->peer : tmpnoaddr,
 603                                     http->request,
 604                                     NULL,
 605                                     http->getConn() != NULL && http->getConn()->auth_user_request ?
 606                                     http->getConn()->auth_user_request : http->request->auth_user_request);
 607
 608         node = (clientStreamNode *)http->client_stream.tail->data;
 609         clientStreamRead(node, http, node->readBuffer);
 610         return;
 611     }
 612
 613     /* ACCESS_ALLOWED continues here ... */
 614     safe_free(http->uri);
 615
 616     http->uri = xstrdup(urlCanonical(http->request));
 617
 618     http->doCallouts();
 619 }
 620
 621 #if USE_ADAPTATION
 622 static void
 623 adaptationAclCheckDoneWrapper(Adaptation::ServiceGroupPointer g, void *data)
 624 {
 625     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 626
 627     if (!calloutContext->httpStateIsValid())
 628         return;
 629
 630     calloutContext->adaptationAclCheckDone(g);
 631 }
 632
 633 void
 634 ClientRequestContext::adaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 635 {
 636     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 637     assert(http);
 638
 639 #if ICAP_CLIENT
 640     Adaptation::Icap::History::Pointer ih = http->request->icapHistory();
 641     if (ih != NULL) {
 642         if (http->getConn() != NULL) {
 643             ih->rfc931 = http->getConn()->rfc931;
 644 #if USE_SSL
 645             ih->ssluser = sslGetUserEmail(fd_table[http->getConn()->fd].ssl);
 646 #endif
 647         }
 648         ih->log_uri = http->log_uri;
 649         ih->req_sz = http->req_sz;
 650     }
 651 #endif
 652
 653     if (!g) {
 654         debugs(85,3, HERE << "no adaptation needed");
 655         http->doCallouts();
 656         return;
 657     }
 658
 659     http->startAdaptation(g);
 660 }
 661
 662 #endif
 663
 664 static void
 665 clientRedirectAccessCheckDone(int answer, void *data)
 666 {
 667     ClientRequestContext *context = (ClientRequestContext *)data;
 668     ClientHttpRequest *http = context->http;
 669     context->acl_checklist = NULL;
 670
 671     if (answer == ACCESS_ALLOWED)
 672         redirectStart(http, clientRedirectDoneWrapper, context);
 673     else
 674         context->clientRedirectDone(NULL);
 675 }
 676
 677 void
 678 ClientRequestContext::clientRedirectStart()
 679 {
 680     debugs(33, 5, "clientRedirectStart: '" << http->uri << "'");
 681
 682     if (Config.accessList.redirector) {
 683         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 684         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 685     } else
 686         redirectStart(http, clientRedirectDoneWrapper, this);
 687 }
 688
 689 static int
 690 clientHierarchical(ClientHttpRequest * http)
 691 {
 692     const char *url = http->uri;
 693     HttpRequest *request = http->request;
 694     HttpRequestMethod method = request->method;
 695     const wordlist *p = NULL;
 696
 697     /*
 698      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 699      * neighbors support private keys
 700      */
 701
 702     if (request->flags.ims && !neighbors_do_private_keys)
 703         return 0;
 704
 705     /*
 706      * This is incorrect: authenticating requests can be sent via a hierarchy
 707      * (they can even be cached if the correct headers are set on the reply)
 708      */
 709     if (request->flags.auth)
 710         return 0;
 711
 712     if (method == METHOD_TRACE)
 713         return 1;
 714
 715     if (method != METHOD_GET)
 716         return 0;
 717
 718     /* scan hierarchy_stoplist */
 719     for (p = Config.hierarchy_stoplist; p; p = p->next)
 720         if (strstr(url, p->key))
 721             return 0;
 722
 723     if (request->flags.loopdetect)
 724         return 0;
 725
 726     if (request->protocol == PROTO_HTTP)
 727         return httpCachable(method);
 728
 729     if (request->protocol == PROTO_GOPHER)
 730         return gopherCachable(request);
 731
 732     if (request->protocol == PROTO_CACHEOBJ)
 733         return 0;
 734
 735     return 1;
 736 }
 737
 738
 739 static void
 740 clientCheckPinning(ClientHttpRequest * http)
 741 {
 742     HttpRequest *request = http->request;
 743     HttpHeader *req_hdr = &request->header;
 744     ConnStateData *http_conn = http->getConn();
 745
 746     /* Internal requests such as those from ESI includes may be without
 747      * a client connection
 748      */
 749     if (!http_conn)
 750         return;
 751
 752     request->flags.connection_auth_disabled = http_conn->port->connection_auth_disabled;
 753     if (!request->flags.connection_auth_disabled) {
 754         if (http_conn->pinning.fd != -1) {
 755             if (http_conn->pinning.auth) {
 756                 request->flags.connection_auth = 1;
 757                 request->flags.auth = 1;
 758             } else {
 759                 request->flags.connection_proxy_auth = 1;
 760             }
 761             request->setPinnedConnection(http_conn);
 762         }
 763     }
 764
 765     /* check if connection auth is used, and flag as candidate for pinning
 766      * in such case.
 767      * Note: we may need to set flags.connection_auth even if the connection
 768      * is already pinned if it was pinned earlier due to proxy auth
 769      */
 770     if (!request->flags.connection_auth) {
 771         if (req_hdr->has(HDR_AUTHORIZATION) || req_hdr->has(HDR_PROXY_AUTHORIZATION)) {
 772             HttpHeaderPos pos = HttpHeaderInitPos;
 773             HttpHeaderEntry *e;
 774             int may_pin = 0;
 775             while ((e = req_hdr->getEntry(&pos))) {
 776                 if (e->id == HDR_AUTHORIZATION || e->id == HDR_PROXY_AUTHORIZATION) {
 777                     const char *value = e->value.rawBuf();
 778                     if (strncasecmp(value, "NTLM ", 5) == 0
 779                             ||
 780                             strncasecmp(value, "Negotiate ", 10) == 0
 781                             ||
 782                             strncasecmp(value, "Kerberos ", 9) == 0) {
 783                         if (e->id == HDR_AUTHORIZATION) {
 784                             request->flags.connection_auth = 1;
 785                             may_pin = 1;
 786                         } else {
 787                             request->flags.connection_proxy_auth = 1;
 788                             may_pin = 1;
 789                         }
 790                     }
 791                 }
 792             }
 793             if (may_pin && !request->pinnedConnection()) {
 794                 request->setPinnedConnection(http->getConn());
 795             }
 796         }
 797     }
 798 }
 799
 800 static void
 801 clientInterpretRequestHeaders(ClientHttpRequest * http)
 802 {
 803     HttpRequest *request = http->request;
 804     HttpHeader *req_hdr = &request->header;
 805     int no_cache = 0;
 806 #if !(USE_SQUID_ESI) || defined(USE_USERAGENT_LOG) || defined(USE_REFERER_LOG)
 807
 808     const char *str;
 809 #endif
 810
 811     request->imslen = -1;
 812     request->ims = req_hdr->getTime(HDR_IF_MODIFIED_SINCE);
 813
 814     if (request->ims > 0)
 815         request->flags.ims = 1;
 816
 817 #if USE_SQUID_ESI
 818     /*
 819      * We ignore Cache-Control as per the Edge Architecture Section 3. See
 820      * www.esi.org for more information.
 821      */
 822 #else
 823
 824     if (req_hdr->has(HDR_PRAGMA)) {
 825         String s = req_hdr->getList(HDR_PRAGMA);
 826
 827         if (strListIsMember(&s, "no-cache", ','))
 828             no_cache++;
 829
 830         s.clean();
 831     }
 832
 833     if (request->cache_control)
 834         if (EBIT_TEST(request->cache_control->mask, CC_NO_CACHE))
 835             no_cache++;
 836
 837     /*
 838     * Work around for supporting the Reload button in IE browsers when Squid
 839     * is used as an accelerator or transparent proxy, by turning accelerated
 840     * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
 841     * actually only fixed in SP1, but we can't tell whether we are talking to
 842     * SP1 or not so all 5.5 versions are treated 'normally').
 843     */
 844     if (Config.onoff.ie_refresh) {
 845         if (http->flags.accel && request->flags.ims) {
 846             if ((str = req_hdr->getStr(HDR_USER_AGENT))) {
 847                 if (strstr(str, "MSIE 5.01") != NULL)
 848                     no_cache++;
 849                 else if (strstr(str, "MSIE 5.0") != NULL)
 850                     no_cache++;
 851                 else if (strstr(str, "MSIE 4.") != NULL)
 852                     no_cache++;
 853                 else if (strstr(str, "MSIE 3.") != NULL)
 854                     no_cache++;
 855             }
 856         }
 857     }
 858
 859     if (request->method == METHOD_OTHER) {
 860         no_cache++;
 861     }
 862
 863 #endif
 864     if (no_cache) {
 865 #if HTTP_VIOLATIONS
 866
 867         if (Config.onoff.reload_into_ims)
 868             request->flags.nocache_hack = 1;
 869         else if (refresh_nocache_hack)
 870             request->flags.nocache_hack = 1;
 871         else
 872 #endif
 873
 874             request->flags.nocache = 1;
 875     }
 876
 877     /* ignore range header in non-GETs or non-HEADs */
 878     if (request->method == METHOD_GET || request->method == METHOD_HEAD) {
 879         request->range = req_hdr->getRange();
 880
 881         if (request->range) {
 882             request->flags.range = 1;
 883             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
 884             /* XXX: This is suboptimal. We should give the stream the range set,
 885              * and thereby let the top of the stream set the offset when the
 886              * size becomes known. As it is, we will end up requesting from 0
 887              * for evey -X range specification.
 888              * RBC - this may be somewhat wrong. We should probably set the range
 889              * iter up at this point.
 890              */
 891             node->readBuffer.offset = request->range->lowestOffset(0);
 892             http->range_iter.pos = request->range->begin();
 893             http->range_iter.valid = true;
 894         }
 895     }
 896
 897     /* Only HEAD and GET requests permit a Range or Request-Range header.
 898      * If these headers appear on any other type of request, delete them now.
 899      */
 900     else {
 901         req_hdr->delById(HDR_RANGE);
 902         req_hdr->delById(HDR_REQUEST_RANGE);
 903         request->range = NULL;
 904     }
 905
 906     if (req_hdr->has(HDR_AUTHORIZATION))
 907         request->flags.auth = 1;
 908
 909     clientCheckPinning(http);
 910
 911     if (request->login[0] != '\0')
 912         request->flags.auth = 1;
 913
 914     if (req_hdr->has(HDR_VIA)) {
 915         String s = req_hdr->getList(HDR_VIA);
 916         /*
 917          * ThisCache cannot be a member of Via header, "1.0 ThisCache" can.
 918          * Note ThisCache2 has a space prepended to the hostname so we don't
 919          * accidentally match super-domains.
 920          */
 921
 922         if (strListIsSubstr(&s, ThisCache2, ',')) {
 923             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
 924                      request, (ObjPackMethod) & httpRequestPack);
 925             request->flags.loopdetect = 1;
 926         }
 927
 928 #if FORW_VIA_DB
 929         fvdbCountVia(s.termedBuf());
 930
 931 #endif
 932
 933         s.clean();
 934     }
 935
 936     /**
 937      \todo  --enable-useragent-log and --enable-referer-log. We should
 938             probably drop those two as the custom log formats accomplish pretty much the same thing..
 939     */
 940 #if USE_USERAGENT_LOG
 941     if ((str = req_hdr->getStr(HDR_USER_AGENT)))
 942         logUserAgent(fqdnFromAddr(http->getConn()->log_addr), str);
 943
 944 #endif
 945 #if USE_REFERER_LOG
 946
 947     if ((str = req_hdr->getStr(HDR_REFERER)))
 948         logReferer(fqdnFromAddr(http->getConn()->log_addr), str, http->log_uri);
 949
 950 #endif
 951 #if FORW_VIA_DB
 952
 953     if (req_hdr->has(HDR_X_FORWARDED_FOR)) {
 954         String s = req_hdr->getList(HDR_X_FORWARDED_FOR);
 955         fvdbCountForw(s.termedBuf());
 956         s.clean();
 957     }
 958
 959 #endif
 960     if (request->method == METHOD_TRACE || request->method == METHOD_OPTIONS) {
 961         request->max_forwards = req_hdr->getInt64(HDR_MAX_FORWARDS);
 962     }
 963
 964     request->flags.cachable = http->request->cacheable();
 965
 966     if (clientHierarchical(http))
 967         request->flags.hierarchical = 1;
 968
 969     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
 970            (request->flags.nocache ? "SET" : "NOT SET"));
 971     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
 972            (request->flags.cachable ? "SET" : "NOT SET"));
 973     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
 974            (request->flags.hierarchical ? "SET" : "NOT SET"));
 975
 976 }
 977
 978 void
 979 clientRedirectDoneWrapper(void *data, char *result)
 980 {
 981     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
 982
 983     if (!calloutContext->httpStateIsValid())
 984         return;
 985
 986     calloutContext->clientRedirectDone(result);
 987 }
 988
 989 void
 990 ClientRequestContext::clientRedirectDone(char *result)
 991 {
 992     HttpRequest *new_request = NULL;
 993     HttpRequest *old_request = http->request;
 994     debugs(85, 5, "clientRedirectDone: '" << http->uri << "' result=" << (result ? result : "NULL"));
 995     assert(redirect_state == REDIRECT_PENDING);
 996     redirect_state = REDIRECT_DONE;
 997
 998     if (result) {
 999         http_status status = (http_status) atoi(result);
1000
1001         if (status == HTTP_MOVED_PERMANENTLY
1002                 || status == HTTP_MOVED_TEMPORARILY
1003                 || status == HTTP_SEE_OTHER
1004                 || status == HTTP_TEMPORARY_REDIRECT) {
1005             char *t = result;
1006
1007             if ((t = strchr(result, ':')) != NULL) {
1008                 http->redirect.status = status;
1009                 http->redirect.location = xstrdup(t + 1);
1010             } else {
1011                 debugs(85, 1, "clientRedirectDone: bad input: " << result);
1012             }
1013         } else if (strcmp(result, http->uri))
1014             new_request = HttpRequest::CreateFromUrlAndMethod(result, old_request->method);
1015     }
1016
1017     if (new_request) {
1018         safe_free(http->uri);
1019         http->uri = xstrdup(urlCanonical(new_request));
1020         new_request->http_ver = old_request->http_ver;
1021         new_request->header.append(&old_request->header);
1022         new_request->client_addr = old_request->client_addr;
1023 #if FOLLOW_X_FORWARDED_FOR
1024         new_request->indirect_client_addr = old_request->indirect_client_addr;
1025 #endif /* FOLLOW_X_FORWARDED_FOR */
1026         new_request->my_addr = old_request->my_addr;
1027         new_request->flags = old_request->flags;
1028         new_request->flags.redirected = 1;
1029
1030         if (old_request->auth_user_request) {
1031             new_request->auth_user_request = old_request->auth_user_request;
1032             AUTHUSERREQUESTLOCK(new_request->auth_user_request, "new request");
1033         }
1034
1035         if (old_request->body_pipe != NULL) {
1036             new_request->body_pipe = old_request->body_pipe;
1037             old_request->body_pipe = NULL;
1038             debugs(0,0,HERE << "redirecting body_pipe " << new_request->body_pipe << " from request " << old_request << " to " << new_request);
1039         }
1040
1041         new_request->content_length = old_request->content_length;
1042         new_request->extacl_user = old_request->extacl_user;
1043         new_request->extacl_passwd = old_request->extacl_passwd;
1044         new_request->flags.proxy_keepalive = old_request->flags.proxy_keepalive;
1045         HTTPMSGUNLOCK(old_request);
1046         http->request = HTTPMSGLOCK(new_request);
1047     }
1048
1049     /* FIXME PIPELINE: This is innacurate during pipelining */
1050
1051     if (http->getConn() != NULL)
1052         fd_note(http->getConn()->fd, http->uri);
1053
1054     assert(http->uri);
1055
1056     http->doCallouts();
1057 }
1058
1059 /** Test cache allow/deny configuration
1060  *  Sets flags.cachable=1 if caching is not denied.
1061  */
1062 void
1063 ClientRequestContext::checkNoCache()
1064 {
1065     if (Config.accessList.noCache) {
1066         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1067         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1068     } else {
1069         /* unless otherwise specified, we try to cache. */
1070         checkNoCacheDone(1);
1071     }
1072 }
1073
1074 static void
1075 checkNoCacheDoneWrapper(int answer, void *data)
1076 {
1077     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1078
1079     if (!calloutContext->httpStateIsValid())
1080         return;
1081
1082     calloutContext->checkNoCacheDone(answer);
1083 }
1084
1085 void
1086 ClientRequestContext::checkNoCacheDone(int answer)
1087 {
1088     acl_checklist = NULL;
1089     http->request->flags.cachable = answer;
1090     http->doCallouts();
1091 }
1092
1093 /*
1094  * Identify requests that do not go through the store and client side stream
1095  * and forward them to the appropriate location. All other requests, request
1096  * them.
1097  */
1098 void
1099 ClientHttpRequest::processRequest()
1100 {
1101     debugs(85, 4, "clientProcessRequest: " << RequestMethodStr(request->method) << " '" << uri << "'");
1102
1103 #if USE_SSL
1104     if (request->method == METHOD_CONNECT && sslBumpNeeded()) {
1105         sslBumpStart();
1106         return;
1107     }
1108 #endif
1109
1110     if (request->method == METHOD_CONNECT && !redirect.status) {
1111         logType = LOG_TCP_MISS;
1112         tunnelStart(this, &out.size, &al.http.code);
1113         return;
1114     }
1115
1116     httpStart();
1117 }
1118
1119 void
1120 ClientHttpRequest::httpStart()
1121 {
1122     PROF_start(httpStart);
1123     logType = LOG_TAG_NONE;
1124     debugs(85, 4, "ClientHttpRequest::httpStart: " << log_tags[logType] << " for '" << uri << "'");
1125
1126     /* no one should have touched this */
1127     assert(out.offset == 0);
1128     /* Use the Stream Luke */
1129     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1130     clientStreamRead(node, this, node->readBuffer);
1131     PROF_stop(httpStart);
1132 }
1133
1134 #if USE_SSL
1135
1136 // determines whether we should bump the CONNECT request
1137 bool
1138 ClientHttpRequest::sslBumpNeeded() const
1139 {
1140     if (!getConn()->port->sslBump || !Config.accessList.ssl_bump)
1141         return false;
1142
1143     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1144
1145     ACLFilledChecklist check(Config.accessList.ssl_bump, request, NULL);
1146     check.src_addr = request->client_addr;
1147     check.my_addr = request->my_addr;
1148     return check.fastCheck() == 1;
1149 }
1150
1151 // called when comm_write has completed
1152 static void
1153 SslBumpEstablish(int, char *, size_t, comm_err_t errflag, int, void *data)
1154 {
1155     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1156     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1157
1158     assert(r && cbdataReferenceValid(r));
1159     r->sslBumpEstablish(errflag);
1160 }
1161
1162 void
1163 ClientHttpRequest::sslBumpEstablish(comm_err_t errflag)
1164 {
1165     // Bail out quickly on COMM_ERR_CLOSING - close handlers will tidy up
1166     if (errflag == COMM_ERR_CLOSING)
1167         return;
1168
1169     if (errflag) {
1170         getConn()->startClosing("CONNECT response failure in SslBump");
1171         return;
1172     }
1173
1174     getConn()->switchToHttps();
1175 }
1176
1177 void
1178 ClientHttpRequest::sslBumpStart()
1179 {
1180     debugs(85, 5, HERE << "ClientHttpRequest::sslBumpStart");
1181
1182     // send an HTTP 200 response to kick client SSL negotiation
1183     const int fd = getConn()->fd;
1184     debugs(33, 7, HERE << "Confirming CONNECT tunnel on FD " << fd);
1185
1186     // TODO: Unify with tunnel.cc and add a Server(?) header
1187     static const char *const conn_established =
1188         "HTTP/1.0 200 Connection established\r\n\r\n";
1189     comm_write(fd, conn_established, strlen(conn_established),
1190                &SslBumpEstablish, this, NULL);
1191 }
1192
1193 #endif
1194
1195 bool
1196 ClientHttpRequest::gotEnough() const
1197 {
1198     /** TODO: should be querying the stream. */
1199     int64_t contentLength =
1200         memObject()->getReply()->bodySize(request->method);
1201     assert(contentLength >= 0);
1202
1203     if (out.offset < contentLength)
1204         return false;
1205
1206     return true;
1207 }
1208
1209 void
1210 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1211 {
1212     entry_ = newEntry;
1213 }
1214
1215 void
1216 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1217 {
1218     if (loggingEntry_)
1219         loggingEntry_->unlock();
1220
1221     loggingEntry_ = newEntry;
1222
1223     if (loggingEntry_)
1224         loggingEntry_->lock();
1225 }
1226
1227 /*
1228  * doCallouts() - This function controls the order of "callout"
1229  * executions, including non-blocking access control checks, the
1230  * redirector, and ICAP.  Previously, these callouts were chained
1231  * together such that "clientAccessCheckDone()" would call
1232  * "clientRedirectStart()" and so on.
1233  *
1234  * The ClientRequestContext (aka calloutContext) class holds certain
1235  * state data for the callout/callback operations.  Previously
1236  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1237  * for a short time.  ClientRequestContext would then delete itself
1238  * and pass control back to ClientHttpRequest when all callouts
1239  * were finished.
1240  *
1241  * This caused some problems for ICAP because we want to make the
1242  * ICAP callout after checking ACLs, but before checking the no_cache
1243  * list.  We can't stuff the ICAP state into the ClientRequestContext
1244  * class because we still need the ICAP state after ClientRequestContext
1245  * goes away.
1246  *
1247  * Note that ClientRequestContext is created before the first call
1248  * to doCallouts().
1249  *
1250  * If one of the callouts notices that ClientHttpRequest is no
1251  * longer valid, it should call cbdataReferenceDone() so that
1252  * ClientHttpRequest's reference count goes to zero and it will get
1253  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1254  *
1255  * Note that we set the _done flags here before actually starting
1256  * the callout.  This is strictly for convenience.
1257  */
1258
1259 extern int aclMapTOS (acl_tos * head, ACLChecklist * ch);
1260
1261 void
1262 ClientHttpRequest::doCallouts()
1263 {
1264     assert(calloutContext);
1265
1266     if (!calloutContext->http_access_done) {
1267         debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1268         calloutContext->http_access_done = true;
1269         calloutContext->clientAccessCheck();
1270         return;
1271     }
1272
1273 #if USE_ADAPTATION
1274     if (!calloutContext->adaptation_acl_check_done) {
1275         calloutContext->adaptation_acl_check_done = true;
1276         if (Adaptation::AccessCheck::Start(
1277                     Adaptation::methodReqmod, Adaptation::pointPreCache,
1278                     request, NULL, adaptationAclCheckDoneWrapper, calloutContext))
1279             return; // will call callback
1280     }
1281 #endif
1282
1283     if (!calloutContext->redirect_done) {
1284         calloutContext->redirect_done = true;
1285         assert(calloutContext->redirect_state == REDIRECT_NONE);
1286
1287         if (Config.Program.redirect) {
1288             debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1289             calloutContext->redirect_state = REDIRECT_PENDING;
1290             calloutContext->clientRedirectStart();
1291             return;
1292         }
1293     }
1294
1295     if (!calloutContext->interpreted_req_hdrs) {
1296         debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1297         calloutContext->interpreted_req_hdrs = 1;
1298         clientInterpretRequestHeaders(this);
1299     }
1300
1301     if (!calloutContext->no_cache_done) {
1302         calloutContext->no_cache_done = true;
1303
1304         if (Config.accessList.noCache && request->flags.cachable) {
1305             debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1306             calloutContext->checkNoCache();
1307             return;
1308         }
1309     }
1310
1311     if (!calloutContext->clientside_tos_done) {
1312         calloutContext->clientside_tos_done = true;
1313         if (getConn() != NULL) {
1314             ACLFilledChecklist ch(NULL, request, NULL);
1315             ch.src_addr = request->client_addr;
1316             ch.my_addr = request->my_addr;
1317             int tos = aclMapTOS(Config.accessList.clientside_tos, &ch);
1318             if (tos)
1319                 comm_set_tos(getConn()->fd, tos);
1320         }
1321     }
1322
1323     cbdataReferenceDone(calloutContext->http);
1324     delete calloutContext;
1325     calloutContext = NULL;
1326 #if HEADERS_LOG
1327
1328     headersLog(0, 1, request->method, request);
1329 #endif
1330
1331     debugs(83, 3, HERE << "calling processRequest()");
1332     processRequest();
1333
1334 #if ICAP_CLIENT
1335     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1336     if (ih != NULL)
1337         ih->logType = logType;
1338 #endif
1339 }
1340
1341 #ifndef _USE_INLINE_
1342 #include "client_side_request.cci"
1343 #endif
1344
1345 #if USE_ADAPTATION
1346 /// Initiate an asynchronous adaptation transaction which will call us back.
1347 void
1348 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1349 {
1350     debugs(85, 3, HERE << "adaptation needed for " << this);
1351     assert(!virginHeadSource);
1352     assert(!adaptedBodySource);
1353     virginHeadSource = initiateAdaptation(
1354                            new Adaptation::Iterator(this, request, NULL, g));
1355
1356     // we could try to guess whether we can bypass this adaptation
1357     // initiation failure, but it should not really happen
1358     assert(virginHeadSource != NULL); // Must, really
1359 }
1360
1361 void
1362 ClientHttpRequest::noteAdaptationAnswer(HttpMsg *msg)
1363 {
1364     assert(cbdataReferenceValid(this));         // indicates bug
1365     assert(msg);
1366
1367     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1368         /*
1369          * Replace the old request with the new request.
1370          */
1371         HTTPMSGUNLOCK(request);
1372         request = HTTPMSGLOCK(new_req);
1373         /*
1374          * Store the new URI for logging
1375          */
1376         xfree(uri);
1377         uri = xstrdup(urlCanonical(request));
1378         setLogUri(this, urlCanonicalClean(request));
1379         assert(request->method.id());
1380     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1381         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1382
1383         // subscribe to receive reply body
1384         if (new_rep->body_pipe != NULL) {
1385             adaptedBodySource = new_rep->body_pipe;
1386             assert(adaptedBodySource->setConsumerIfNotLate(this));
1387         }
1388
1389         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1390         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1391         repContext->createStoreEntry(request->method, request->flags);
1392
1393         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1394         request_satisfaction_mode = true;
1395         request_satisfaction_offset = 0;
1396         storeEntry()->replaceHttpReply(new_rep);
1397
1398         if (!adaptedBodySource) // no body
1399             storeEntry()->complete();
1400         clientGetMoreData(node, this);
1401     }
1402
1403     // we are done with getting headers (but may be receiving body)
1404     clearAdaptation(virginHeadSource);
1405
1406     if (!request_satisfaction_mode)
1407         doCallouts();
1408 }
1409
1410 void
1411 ClientHttpRequest::noteAdaptationQueryAbort(bool final)
1412 {
1413     clearAdaptation(virginHeadSource);
1414     assert(!adaptedBodySource);
1415     handleAdaptationFailure(!final);
1416 }
1417
1418 void
1419 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1420 {
1421     assert(request_satisfaction_mode);
1422     assert(adaptedBodySource != NULL);
1423
1424     if (const size_t contentSize = adaptedBodySource->buf().contentSize()) {
1425         BodyPipeCheckout bpc(*adaptedBodySource);
1426         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset);
1427         storeEntry()->write(ioBuf);
1428         // assume can write everything
1429         request_satisfaction_offset += contentSize;
1430         bpc.buf.consume(contentSize);
1431         bpc.checkIn();
1432     }
1433
1434     if (adaptedBodySource->exhausted())
1435         endRequestSatisfaction();
1436     // else wait for more body data
1437 }
1438
1439 void
1440 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
1441 {
1442     assert(!virginHeadSource);
1443     if (adaptedBodySource != NULL) { // did not end request satisfaction yet
1444         // We do not expect more because noteMoreBodyDataAvailable always
1445         // consumes everything. We do not even have a mechanism to consume
1446         // leftovers after noteMoreBodyDataAvailable notifications seize.
1447         assert(adaptedBodySource->exhausted());
1448         endRequestSatisfaction();
1449     }
1450 }
1451
1452 void
1453 ClientHttpRequest::endRequestSatisfaction()
1454 {
1455     debugs(85,4, HERE << this << " ends request satisfaction");
1456     assert(request_satisfaction_mode);
1457     stopConsumingFrom(adaptedBodySource);
1458
1459     // TODO: anything else needed to end store entry formation correctly?
1460     storeEntry()->complete();
1461 }
1462
1463 void
1464 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
1465 {
1466     assert(!virginHeadSource);
1467     stopConsumingFrom(adaptedBodySource);
1468     handleAdaptationFailure();
1469 }
1470
1471 void
1472 ClientHttpRequest::handleAdaptationFailure(bool bypassable)
1473 {
1474     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
1475
1476     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
1477     const bool usedPipe = request->body_pipe != NULL &&
1478                           request->body_pipe->consumedSize() > 0;
1479
1480     if (bypassable && !usedStore && !usedPipe) {
1481         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
1482         if (calloutContext)
1483             doCallouts();
1484         return;
1485     }
1486
1487     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
1488
1489     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1490     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1491     assert(repContext);
1492
1493     // The original author of the code also wanted to pass an errno to
1494     // setReplyToError, but it seems unlikely that the errno reflects the
1495     // true cause of the error at this point, so I did not pass it.
1496     IpAddress noAddr;
1497     noAddr.SetNoAddr();
1498     ConnStateData * c = getConn();
1499     repContext->setReplyToError(ERR_ICAP_FAILURE, HTTP_INTERNAL_SERVER_ERROR,
1500                                 request->method, NULL,
1501                                 (c != NULL ? c->peer : noAddr), request, NULL,
1502                                 (c != NULL && c->auth_user_request ?
1503                                  c->auth_user_request : request->auth_user_request));
1504
1505     node = (clientStreamNode *)client_stream.tail->data;
1506     clientStreamRead(node, this, node->readBuffer);
1507 }
1508
1509 #endif