src/client_side_request.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 85    Client-side Request Routines */
  10
  11 /*
  12  * General logic of request processing:
  13  *
  14  * We run a series of tests to determine if access will be permitted, and to do
  15  * any redirection. Then we call into the result clientStream to retrieve data.
  16  * From that point on it's up to reply management.
  17  */
  18
  19 #include "squid.h"
  20 #include "acl/FilledChecklist.h"
  21 #include "acl/Gadgets.h"
  22 #include "anyp/PortCfg.h"
  23 #include "base/AsyncJobCalls.h"
  24 #include "client_side.h"
  25 #include "client_side_reply.h"
  26 #include "client_side_request.h"
  27 #include "ClientRequestContext.h"
  28 #include "clientStream.h"
  29 #include "comm/Connection.h"
  30 #include "comm/Write.h"
  31 #include "err_detail_type.h"
  32 #include "errorpage.h"
  33 #include "fd.h"
  34 #include "fde.h"
  35 #include "format/Token.h"
  36 #include "gopher.h"
  37 #include "helper.h"
  38 #include "helper/Reply.h"
  39 #include "http.h"
  40 #include "http/Stream.h"
  41 #include "HttpHdrCc.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequest.h"
  44 #include "ip/QosConfig.h"
  45 #include "ipcache.h"
  46 #include "log/access_log.h"
  47 #include "MemObject.h"
  48 #include "Parsing.h"
  49 #include "profiler/Profiler.h"
  50 #include "redirect.h"
  51 #include "SquidConfig.h"
  52 #include "SquidTime.h"
  53 #include "Store.h"
  54 #include "StrList.h"
  55 #include "tools.h"
  56 #include "URL.h"
  57 #include "wordlist.h"
  58 #if USE_AUTH
  59 #include "auth/UserRequest.h"
  60 #endif
  61 #if USE_ADAPTATION
  62 #include "adaptation/AccessCheck.h"
  63 #include "adaptation/Answer.h"
  64 #include "adaptation/Iterator.h"
  65 #include "adaptation/Service.h"
  66 #if ICAP_CLIENT
  67 #include "adaptation/icap/History.h"
  68 #endif
  69 #endif
  70 #if USE_OPENSSL
  71 #include "ssl/ServerBump.h"
  72 #include "ssl/support.h"
  73 #endif
  74
  75 #if LINGERING_CLOSE
  76 #define comm_close comm_lingering_close
  77 #endif
  78
  79 static const char *const crlf = "\r\n";
  80
  81 #if FOLLOW_X_FORWARDED_FOR
  82 static void clientFollowXForwardedForCheck(allow_t answer, void *data);
  83 #endif /* FOLLOW_X_FORWARDED_FOR */
  84
  85 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
  86
  87 CBDATA_CLASS_INIT(ClientRequestContext);
  88
  89 /* Local functions */
  90 /* other */
  91 static void clientAccessCheckDoneWrapper(allow_t, void *);
  92 #if USE_OPENSSL
  93 static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
  94 #endif
  95 static int clientHierarchical(ClientHttpRequest * http);
  96 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  97 static HLPCB clientRedirectDoneWrapper;
  98 static HLPCB clientStoreIdDoneWrapper;
  99 static void checkNoCacheDoneWrapper(allow_t, void *);
 100 SQUIDCEXTERN CSR clientGetMoreData;
 101 SQUIDCEXTERN CSS clientReplyStatus;
 102 SQUIDCEXTERN CSD clientReplyDetach;
 103 static void checkFailureRatio(err_type, hier_code);
 104
 105 ClientRequestContext::~ClientRequestContext()
 106 {
 107     /*
 108      * Release our "lock" on our parent, ClientHttpRequest, if we
 109      * still have one
 110      */
 111
 112     if (http)
 113         cbdataReferenceDone(http);
 114
 115     delete error;
 116     debugs(85,3, "ClientRequestContext destructed, this=" << this);
 117 }
 118
 119 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) :
 120     http(cbdataReference(anHttp)),
 121     acl_checklist(NULL),
 122     redirect_state(REDIRECT_NONE),
 123     store_id_state(REDIRECT_NONE),
 124     host_header_verify_done(false),
 125     http_access_done(false),
 126     adapted_http_access_done(false),
 127 #if USE_ADAPTATION
 128     adaptation_acl_check_done(false),
 129 #endif
 130     redirect_done(false),
 131     store_id_done(false),
 132     no_cache_done(false),
 133     interpreted_req_hdrs(false),
 134     tosToClientDone(false),
 135     nfmarkToClientDone(false),
 136 #if USE_OPENSSL
 137     sslBumpCheckDone(false),
 138 #endif
 139     error(NULL),
 140     readNextRequest(false)
 141 {
 142     debugs(85, 3, "ClientRequestContext constructed, this=" << this);
 143 }
 144
 145 CBDATA_CLASS_INIT(ClientHttpRequest);
 146
 147 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 148 #if USE_ADAPTATION
 149     AsyncJob("ClientHttpRequest"),
 150 #endif
 151     request(NULL),
 152     uri(NULL),
 153     log_uri(NULL),
 154     req_sz(0),
 155     logType(LOG_TAG_NONE),
 156     calloutContext(NULL),
 157     maxReplyBodySize_(0),
 158     entry_(NULL),
 159     loggingEntry_(NULL),
 160     conn_(NULL)
 161 #if USE_OPENSSL
 162     , sslBumpNeed_(Ssl::bumpEnd)
 163 #endif
 164 #if USE_ADAPTATION
 165     , request_satisfaction_mode(false)
 166     , request_satisfaction_offset(0)
 167 #endif
 168 {
 169     setConn(aConn);
 170     al = new AccessLogEntry;
 171     al->cache.start_time = current_time;
 172     al->tcpClient = clientConnection = aConn->clientConnection;
 173     al->cache.port = aConn->port;
 174     al->cache.caddr = aConn->log_addr;
 175
 176 #if USE_OPENSSL
 177     if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
 178         if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get())
 179             al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
 180     }
 181 #endif
 182     dlinkAdd(this, &active, &ClientActiveRequests);
 183 }
 184
 185 /*
 186  * returns true if client specified that the object must come from the cache
 187  * without contacting origin server
 188  */
 189 bool
 190 ClientHttpRequest::onlyIfCached()const
 191 {
 192     assert(request);
 193     return request->cache_control &&
 194            request->cache_control->onlyIfCached();
 195 }
 196
 197 /**
 198  * This function is designed to serve a fairly specific purpose.
 199  * Occasionally our vBNS-connected caches can talk to each other, but not
 200  * the rest of the world.  Here we try to detect frequent failures which
 201  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 202  * the failure:success ratio goes above 1.0 then we go into "hit only"
 203  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 204  * will only fetch HITs from us if they are using the ICP protocol.  We
 205  * stay in this mode for 5 minutes.
 206  *
 207  * Duane W., Sept 16, 1996
 208  */
 209 static void
 210 checkFailureRatio(err_type etype, hier_code hcode)
 211 {
 212     // Can be set at compile time with -D compiler flag
 213 #ifndef FAILURE_MODE_TIME
 214 #define FAILURE_MODE_TIME 300
 215 #endif
 216
 217     if (hcode == HIER_NONE)
 218         return;
 219
 220     // don't bother when ICP is disabled.
 221     if (Config.Port.icp <= 0)
 222         return;
 223
 224     static double magic_factor = 100.0;
 225     double n_good;
 226     double n_bad;
 227
 228     n_good = magic_factor / (1.0 + request_failure_ratio);
 229
 230     n_bad = magic_factor - n_good;
 231
 232     switch (etype) {
 233
 234     case ERR_DNS_FAIL:
 235
 236     case ERR_CONNECT_FAIL:
 237     case ERR_SECURE_CONNECT_FAIL:
 238
 239     case ERR_READ_ERROR:
 240         ++n_bad;
 241         break;
 242
 243     default:
 244         ++n_good;
 245     }
 246
 247     request_failure_ratio = n_bad / n_good;
 248
 249     if (hit_only_mode_until > squid_curtime)
 250         return;
 251
 252     if (request_failure_ratio < 1.0)
 253         return;
 254
 255     debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
 256            std::setprecision(3) << request_failure_ratio);
 257
 258     debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
 259            FAILURE_MODE_TIME / 60 << " minutes...");
 260
 261     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 262
 263     request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
 264 }
 265
 266 ClientHttpRequest::~ClientHttpRequest()
 267 {
 268     debugs(33, 3, "httpRequestFree: " << uri);
 269     PROF_start(httpRequestFree);
 270
 271     // Even though freeResources() below may destroy the request,
 272     // we no longer set request->body_pipe to NULL here
 273     // because we did not initiate that pipe (ConnStateData did)
 274
 275     /* the ICP check here was erroneous
 276      * - StoreEntry::releaseRequest was always called if entry was valid
 277      */
 278
 279     logRequest();
 280
 281     loggingEntry(NULL);
 282
 283     if (request)
 284         checkFailureRatio(request->errType, al->hier.code);
 285
 286     freeResources();
 287
 288 #if USE_ADAPTATION
 289     announceInitiatorAbort(virginHeadSource);
 290
 291     if (adaptedBodySource != NULL)
 292         stopConsumingFrom(adaptedBodySource);
 293 #endif
 294
 295     if (calloutContext)
 296         delete calloutContext;
 297
 298     clientConnection = NULL;
 299
 300     if (conn_)
 301         cbdataReferenceDone(conn_);
 302
 303     /* moving to the next connection is handled by the context free */
 304     dlinkDelete(&active, &ClientActiveRequests);
 305
 306     PROF_stop(httpRequestFree);
 307 }
 308
 309 /**
 310  * Create a request and kick it off
 311  *
 312  * \retval 0     success
 313  * \retval -1    failure
 314  *
 315  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 316  * determined by the user
 317  */
 318 int
 319 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 320                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 321                    char *tailbuf, size_t taillen)
 322 {
 323     size_t url_sz;
 324     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 325     HttpRequest *request;
 326     StoreIOBuffer tempBuffer;
 327     if (http->al != NULL)
 328         http->al->cache.start_time = current_time;
 329     /* this is only used to adjust the connection offset in client_side.c */
 330     http->req_sz = 0;
 331     tempBuffer.length = taillen;
 332     tempBuffer.data = tailbuf;
 333     /* client stream setup */
 334     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 335                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 336                      streamdetach, streamdata, tempBuffer);
 337     /* make it visible in the 'current acctive requests list' */
 338     /* Set flags */
 339     /* internal requests only makes sense in an
 340      * accelerator today. TODO: accept flags ? */
 341     http->flags.accel = true;
 342     /* allow size for url rewriting */
 343     url_sz = strlen(url) + Config.appendDomainLen + 5;
 344     http->uri = (char *)xcalloc(url_sz, 1);
 345     strcpy(http->uri, url);
 346
 347     if ((request = HttpRequest::CreateFromUrl(http->uri, method)) == NULL) {
 348         debugs(85, 5, "Invalid URL: " << http->uri);
 349         return -1;
 350     }
 351
 352     /*
 353      * now update the headers in request with our supplied headers. urlParse
 354      * should return a blank header set, but we use Update to be sure of
 355      * correctness.
 356      */
 357     if (header)
 358         request->header.update(header, NULL);
 359
 360     http->log_uri = xstrdup(urlCanonicalClean(request));
 361
 362     /* http struct now ready */
 363
 364     /*
 365      * build new header list *? TODO
 366      */
 367     request->flags.accelerated = http->flags.accel;
 368
 369     request->flags.internalClient = true;
 370
 371     /* this is an internally created
 372      * request, not subject to acceleration
 373      * target overrides */
 374     /*
 375      * FIXME? Do we want to detect and handle internal requests of internal
 376      * objects ?
 377      */
 378
 379     /* Internally created requests cannot have bodies today */
 380     request->content_length = 0;
 381
 382     request->client_addr.setNoAddr();
 383
 384 #if FOLLOW_X_FORWARDED_FOR
 385     request->indirect_client_addr.setNoAddr();
 386 #endif /* FOLLOW_X_FORWARDED_FOR */
 387
 388     request->my_addr.setNoAddr();   /* undefined for internal requests */
 389
 390     request->my_addr.port(0);
 391
 392     request->http_ver = Http::ProtocolVersion();
 393
 394     http->request = request;
 395     HTTPMSGLOCK(http->request);
 396
 397     /* optional - skip the access check ? */
 398     http->calloutContext = new ClientRequestContext(http);
 399
 400     http->calloutContext->http_access_done = false;
 401
 402     http->calloutContext->redirect_done = true;
 403
 404     http->calloutContext->no_cache_done = true;
 405
 406     http->doCallouts();
 407
 408     return 0;
 409 }
 410
 411 bool
 412 ClientRequestContext::httpStateIsValid()
 413 {
 414     ClientHttpRequest *http_ = http;
 415
 416     if (cbdataReferenceValid(http_))
 417         return true;
 418
 419     http = NULL;
 420
 421     cbdataReferenceDone(http_);
 422
 423     return false;
 424 }
 425
 426 #if FOLLOW_X_FORWARDED_FOR
 427 /**
 428  * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
 429  * against the followXFF ACL, or cleans up and passes control to
 430  * clientAccessCheck().
 431  *
 432  * The trust model here is a little ambiguous. So to clarify the logic:
 433  * - we may always use the direct client address as the client IP.
 434  * - these trust tests merey tell whether we trust given IP enough to believe the
 435  *   IP string which it appended to the X-Forwarded-For: header.
 436  * - if at any point we don't trust what an IP adds we stop looking.
 437  * - at that point the current contents of indirect_client_addr are the value set
 438  *   by the last previously trusted IP.
 439  * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
 440  */
 441 static void
 442 clientFollowXForwardedForCheck(allow_t answer, void *data)
 443 {
 444     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 445
 446     if (!calloutContext->httpStateIsValid())
 447         return;
 448
 449     ClientHttpRequest *http = calloutContext->http;
 450     HttpRequest *request = http->request;
 451
 452     /*
 453      * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
 454      * called as a result of ACL checks, or -1 if we are called when
 455      * there's nothing left to do.
 456      */
 457     if (answer == ACCESS_ALLOWED &&
 458             request->x_forwarded_for_iterator.size () != 0) {
 459
 460         /*
 461          * Remove the last comma-delimited element from the
 462          * x_forwarded_for_iterator and use it to repeat the cycle.
 463          */
 464         const char *p;
 465         const char *asciiaddr;
 466         int l;
 467         Ip::Address addr;
 468         p = request->x_forwarded_for_iterator.termedBuf();
 469         l = request->x_forwarded_for_iterator.size();
 470
 471         /*
 472         * XXX x_forwarded_for_iterator should really be a list of
 473         * IP addresses, but it's a String instead.  We have to
 474         * walk backwards through the String, biting off the last
 475         * comma-delimited part each time.  As long as the data is in
 476         * a String, we should probably implement and use a variant of
 477         * strListGetItem() that walks backwards instead of forwards
 478         * through a comma-separated list.  But we don't even do that;
 479         * we just do the work in-line here.
 480         */
 481         /* skip trailing space and commas */
 482         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 483             --l;
 484         request->x_forwarded_for_iterator.cut(l);
 485         /* look for start of last item in list */
 486         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 487             --l;
 488         asciiaddr = p+l;
 489         if ((addr = asciiaddr)) {
 490             request->indirect_client_addr = addr;
 491             request->x_forwarded_for_iterator.cut(l);
 492             calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 493             if (!Config.onoff.acl_uses_indirect_client) {
 494                 /* override the default src_addr tested if we have to go deeper than one level into XFF */
 495                 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
 496             }
 497             calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
 498             return;
 499         }
 500     } /*if (answer == ACCESS_ALLOWED &&
 501         request->x_forwarded_for_iterator.size () != 0)*/
 502
 503     /* clean up, and pass control to clientAccessCheck */
 504     if (Config.onoff.log_uses_indirect_client) {
 505         /*
 506         * Ensure that the access log shows the indirect client
 507         * instead of the direct client.
 508         */
 509         ConnStateData *conn = http->getConn();
 510         conn->log_addr = request->indirect_client_addr;
 511         http->al->cache.caddr = conn->log_addr;
 512     }
 513     request->x_forwarded_for_iterator.clean();
 514     request->flags.done_follow_x_forwarded_for = true;
 515
 516     if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
 517         debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
 518     }
 519
 520     /* process actual access ACL as normal. */
 521     calloutContext->clientAccessCheck();
 522 }
 523 #endif /* FOLLOW_X_FORWARDED_FOR */
 524
 525 static void
 526 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const Dns::LookupDetails &dns, void *data)
 527 {
 528     ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
 529     c->hostHeaderIpVerify(ia, dns);
 530 }
 531
 532 void
 533 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const Dns::LookupDetails &dns)
 534 {
 535     Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
 536
 537     // note the DNS details for the transaction stats.
 538     http->request->recordLookup(dns);
 539
 540     if (ia != NULL && ia->count > 0) {
 541         // Is the NAT destination IP in DNS?
 542         for (int i = 0; i < ia->count; ++i) {
 543             if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
 544                 debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
 545                 http->request->flags.hostVerified = true;
 546                 http->doCallouts();
 547                 return;
 548             }
 549             debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
 550         }
 551     }
 552     debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
 553     hostHeaderVerifyFailed("local IP", "any domain IP");
 554 }
 555
 556 void
 557 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
 558 {
 559     // IP address validation for Host: failed. Admin wants to ignore them.
 560     // NP: we do not yet handle CONNECT tunnels well, so ignore for them
 561     if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
 562         debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
 563                " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
 564
 565         // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
 566         // The problems here are about WRITE for new cache content, which means flags.cachable
 567         http->request->flags.cachable = false; // MUST NOT cache (for now)
 568         // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
 569         http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
 570         // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
 571         http->doCallouts();
 572         return;
 573     }
 574
 575     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
 576            http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
 577     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << http->request->header.getStr(Http::HdrType::USER_AGENT));
 578     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
 579
 580     // IP address validation for Host: failed. reject the connection.
 581     clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 582     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 583     assert (repContext);
 584     repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
 585                                 http->request->method, NULL,
 586                                 http->getConn()->clientConnection->remote,
 587                                 http->request,
 588                                 NULL,
 589 #if USE_AUTH
 590                                 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 591                                 http->getConn()->getAuth() : http->request->auth_user_request);
 592 #else
 593                                 NULL);
 594 #endif
 595     node = (clientStreamNode *)http->client_stream.tail->data;
 596     clientStreamRead(node, http, node->readBuffer);
 597 }
 598
 599 void
 600 ClientRequestContext::hostHeaderVerify()
 601 {
 602     // Require a Host: header.
 603     const char *host = http->request->header.getStr(Http::HdrType::HOST);
 604
 605     if (!host) {
 606         // TODO: dump out the HTTP/1.1 error about missing host header.
 607         // otherwise this is fine, can't forge a header value when its not even set.
 608         debugs(85, 3, HERE << "validate skipped with no Host: header present.");
 609         http->doCallouts();
 610         return;
 611     }
 612
 613     if (http->request->flags.internal) {
 614         // TODO: kill this when URL handling allows partial URLs out of accel mode
 615         //       and we no longer screw with the URL just to add our internal host there
 616         debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
 617         http->doCallouts();
 618         return;
 619     }
 620
 621     // Locate if there is a port attached, strip ready for IP lookup
 622     char *portStr = NULL;
 623     char *hostB = xstrdup(host);
 624     host = hostB;
 625     if (host[0] == '[') {
 626         // IPv6 literal.
 627         portStr = strchr(hostB, ']');
 628         if (portStr && *(++portStr) != ':') {
 629             portStr = NULL;
 630         }
 631     } else {
 632         // Domain or IPv4 literal with port
 633         portStr = strrchr(hostB, ':');
 634     }
 635
 636     uint16_t port = 0;
 637     if (portStr) {
 638         *portStr = '\0'; // strip the ':'
 639         if (*(++portStr) != '\0') {
 640             char *end = NULL;
 641             int64_t ret = strtoll(portStr, &end, 10);
 642             if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
 643                 // invalid port details. Replace the ':'
 644                 *(--portStr) = ':';
 645                 portStr = NULL;
 646             } else
 647                 port = (ret & 0xFFFF);
 648         }
 649     }
 650
 651     debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
 652     if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
 653         // verify the Host: port (if any) matches the apparent destination
 654         if (portStr && port != http->getConn()->clientConnection->local.port()) {
 655             debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
 656                    " matches Host: port " << port << " (" << portStr << ")");
 657             hostHeaderVerifyFailed("intercepted port", portStr);
 658         } else {
 659             // XXX: match the scheme default port against the apparent destination
 660
 661             // verify the destination DNS is one of the Host: headers IPs
 662             ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
 663         }
 664     } else if (!Config.onoff.hostStrictVerify) {
 665         debugs(85, 3, "validate skipped.");
 666         http->doCallouts();
 667     } else if (strlen(host) != strlen(http->request->url.host())) {
 668         // Verify forward-proxy requested URL domain matches the Host: header
 669         debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
 670         hostHeaderVerifyFailed(host, http->request->url.host());
 671     } else if (matchDomainName(host, http->request->url.host()) != 0) {
 672         // Verify forward-proxy requested URL domain matches the Host: header
 673         debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
 674         hostHeaderVerifyFailed(host, http->request->url.host());
 675     } else if (portStr && port != http->request->url.port()) {
 676         // Verify forward-proxy requested URL domain matches the Host: header
 677         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
 678         hostHeaderVerifyFailed("URL port", portStr);
 679     } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
 680         // Verify forward-proxy requested URL domain matches the Host: header
 681         // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
 682         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
 683         hostHeaderVerifyFailed("URL port", "default port");
 684     } else {
 685         // Okay no problem.
 686         debugs(85, 3, "validate passed.");
 687         http->request->flags.hostVerified = true;
 688         http->doCallouts();
 689     }
 690     safe_free(hostB);
 691 }
 692
 693 /* This is the entry point for external users of the client_side routines */
 694 void
 695 ClientRequestContext::clientAccessCheck()
 696 {
 697 #if FOLLOW_X_FORWARDED_FOR
 698     if (!http->request->flags.doneFollowXff() &&
 699             Config.accessList.followXFF &&
 700             http->request->header.has(Http::HdrType::X_FORWARDED_FOR)) {
 701
 702         /* we always trust the direct client address for actual use */
 703         http->request->indirect_client_addr = http->request->client_addr;
 704         http->request->indirect_client_addr.port(0);
 705
 706         /* setup the XFF iterator for processing */
 707         http->request->x_forwarded_for_iterator = http->request->header.getList(Http::HdrType::X_FORWARDED_FOR);
 708
 709         /* begin by checking to see if we trust direct client enough to walk XFF */
 710         acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 711         acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
 712         return;
 713     }
 714 #endif
 715
 716     if (Config.accessList.http) {
 717         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 718         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 719     } else {
 720         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 721         clientAccessCheckDone(ACCESS_DENIED);
 722     }
 723 }
 724
 725 /**
 726  * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
 727  * The default here is to allow all. Since the earlier http_access should do a default deny all.
 728  * This check is just for a last-minute denial based on adapted request headers.
 729  */
 730 void
 731 ClientRequestContext::clientAccessCheck2()
 732 {
 733     if (Config.accessList.adapted_http) {
 734         acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
 735         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 736     } else {
 737         debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
 738         clientAccessCheckDone(ACCESS_ALLOWED);
 739     }
 740 }
 741
 742 void
 743 clientAccessCheckDoneWrapper(allow_t answer, void *data)
 744 {
 745     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 746
 747     if (!calloutContext->httpStateIsValid())
 748         return;
 749
 750     calloutContext->clientAccessCheckDone(answer);
 751 }
 752
 753 void
 754 ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
 755 {
 756     acl_checklist = NULL;
 757     err_type page_id;
 758     Http::StatusCode status;
 759     debugs(85, 2, "The request " << http->request->method << ' ' <<
 760            http->uri << " is " << answer <<
 761            "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
 762
 763 #if USE_AUTH
 764     char const *proxy_auth_msg = "<null>";
 765     if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
 766         proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
 767     else if (http->request->auth_user_request != NULL)
 768         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 769 #endif
 770
 771     if (answer != ACCESS_ALLOWED) {
 772         // auth has a grace period where credentials can be expired but okay not to challenge.
 773
 774         /* Send an auth challenge or error */
 775         // XXX: do we still need aclIsProxyAuth() ?
 776         bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
 777         debugs(85, 5, "Access Denied: " << http->uri);
 778         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 779 #if USE_AUTH
 780         if (auth_challenge)
 781             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 782 #endif
 783
 784         /*
 785          * NOTE: get page_id here, based on AclMatchedName because if
 786          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 787          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 788          * <pribeiro@isel.pt>
 789          */
 790         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
 791
 792         http->logType = LOG_TCP_DENIED;
 793
 794         if (auth_challenge) {
 795 #if USE_AUTH
 796             if (http->request->flags.sslBumped) {
 797                 /*SSL Bumped request, authentication is not possible*/
 798                 status = Http::scForbidden;
 799             } else if (!http->flags.accel) {
 800                 /* Proxy authorisation needed */
 801                 status = Http::scProxyAuthenticationRequired;
 802             } else {
 803                 /* WWW authorisation needed */
 804                 status = Http::scUnauthorized;
 805             }
 806 #else
 807             // need auth, but not possible to do.
 808             status = Http::scForbidden;
 809 #endif
 810             if (page_id == ERR_NONE)
 811                 page_id = ERR_CACHE_ACCESS_DENIED;
 812         } else {
 813             status = Http::scForbidden;
 814
 815             if (page_id == ERR_NONE)
 816                 page_id = ERR_ACCESS_DENIED;
 817         }
 818
 819         Ip::Address tmpnoaddr;
 820         tmpnoaddr.setNoAddr();
 821         error = clientBuildError(page_id, status,
 822                                  NULL,
 823                                  http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
 824                                  http->request
 825                                 );
 826
 827 #if USE_AUTH
 828         error->auth_user_request =
 829             http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 830             http->getConn()->getAuth() : http->request->auth_user_request;
 831 #endif
 832
 833         readNextRequest = true;
 834     }
 835
 836     /* ACCESS_ALLOWED continues here ... */
 837     xfree(http->uri);
 838     http->uri = SBufToCstring(http->request->effectiveRequestUri());
 839     http->doCallouts();
 840 }
 841
 842 #if USE_ADAPTATION
 843 void
 844 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 845 {
 846     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 847
 848 #if ICAP_CLIENT
 849     Adaptation::Icap::History::Pointer ih = request->icapHistory();
 850     if (ih != NULL) {
 851         if (getConn() != NULL && getConn()->clientConnection != NULL) {
 852             ih->rfc931 = getConn()->clientConnection->rfc931;
 853 #if USE_OPENSSL
 854             if (getConn()->clientConnection->isOpen()) {
 855                 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl.get());
 856             }
 857 #endif
 858         }
 859         ih->log_uri = log_uri;
 860         ih->req_sz = req_sz;
 861     }
 862 #endif
 863
 864     if (!g) {
 865         debugs(85,3, HERE << "no adaptation needed");
 866         doCallouts();
 867         return;
 868     }
 869
 870     startAdaptation(g);
 871 }
 872
 873 #endif
 874
 875 static void
 876 clientRedirectAccessCheckDone(allow_t answer, void *data)
 877 {
 878     ClientRequestContext *context = (ClientRequestContext *)data;
 879     ClientHttpRequest *http = context->http;
 880     context->acl_checklist = NULL;
 881
 882     if (answer == ACCESS_ALLOWED)
 883         redirectStart(http, clientRedirectDoneWrapper, context);
 884     else {
 885         Helper::Reply nilReply;
 886         nilReply.result = Helper::Error;
 887         context->clientRedirectDone(nilReply);
 888     }
 889 }
 890
 891 void
 892 ClientRequestContext::clientRedirectStart()
 893 {
 894     debugs(33, 5, HERE << "'" << http->uri << "'");
 895     (void)SyncNotes(*http->al, *http->request);
 896     if (Config.accessList.redirector) {
 897         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 898         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 899     } else
 900         redirectStart(http, clientRedirectDoneWrapper, this);
 901 }
 902
 903 /**
 904  * This methods handles Access checks result of StoreId access list.
 905  * Will handle as "ERR" (no change) in a case Access is not allowed.
 906  */
 907 static void
 908 clientStoreIdAccessCheckDone(allow_t answer, void *data)
 909 {
 910     ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
 911     ClientHttpRequest *http = context->http;
 912     context->acl_checklist = NULL;
 913
 914     if (answer == ACCESS_ALLOWED)
 915         storeIdStart(http, clientStoreIdDoneWrapper, context);
 916     else {
 917         debugs(85, 3, "access denied expected ERR reply handling: " << answer);
 918         Helper::Reply nilReply;
 919         nilReply.result = Helper::Error;
 920         context->clientStoreIdDone(nilReply);
 921     }
 922 }
 923
 924 /**
 925  * Start locating an alternative storeage ID string (if any) from admin
 926  * configured helper program. This is an asynchronous operation terminating in
 927  * ClientRequestContext::clientStoreIdDone() when completed.
 928  */
 929 void
 930 ClientRequestContext::clientStoreIdStart()
 931 {
 932     debugs(33, 5,"'" << http->uri << "'");
 933
 934     if (Config.accessList.store_id) {
 935         acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
 936         acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
 937     } else
 938         storeIdStart(http, clientStoreIdDoneWrapper, this);
 939 }
 940
 941 static int
 942 clientHierarchical(ClientHttpRequest * http)
 943 {
 944     HttpRequest *request = http->request;
 945     HttpRequestMethod method = request->method;
 946
 947     // intercepted requests MUST NOT (yet) be sent to peers unless verified
 948     if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
 949         return 0;
 950
 951     /*
 952      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 953      * neighbors support private keys
 954      */
 955
 956     if (request->flags.ims && !neighbors_do_private_keys)
 957         return 0;
 958
 959     /*
 960      * This is incorrect: authenticating requests can be sent via a hierarchy
 961      * (they can even be cached if the correct headers are set on the reply)
 962      */
 963     if (request->flags.auth)
 964         return 0;
 965
 966     if (method == Http::METHOD_TRACE)
 967         return 1;
 968
 969     if (method != Http::METHOD_GET)
 970         return 0;
 971
 972     if (request->flags.loopDetected)
 973         return 0;
 974
 975     if (request->url.getScheme() == AnyP::PROTO_HTTP)
 976         return method.respMaybeCacheable();
 977
 978     if (request->url.getScheme() == AnyP::PROTO_GOPHER)
 979         return gopherCachable(request);
 980
 981     if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
 982         return 0;
 983
 984     return 1;
 985 }
 986
 987 static void
 988 clientCheckPinning(ClientHttpRequest * http)
 989 {
 990     HttpRequest *request = http->request;
 991     HttpHeader *req_hdr = &request->header;
 992     ConnStateData *http_conn = http->getConn();
 993
 994     /* Internal requests such as those from ESI includes may be without
 995      * a client connection
 996      */
 997     if (!http_conn)
 998         return;
 999
1000     // Internal requests such as those from Downloader does not have
1001     // local port.
1002     if (!http_conn->port)
1003         return;
1004
1005     request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
1006     if (!request->flags.connectionAuthDisabled) {
1007         if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
1008             if (http_conn->pinning.auth) {
1009                 request->flags.connectionAuth = true;
1010                 request->flags.auth = true;
1011             } else {
1012                 request->flags.connectionProxyAuth = true;
1013             }
1014             // These should already be linked correctly.
1015             assert(request->clientConnectionManager == http_conn);
1016         }
1017     }
1018
1019     /* check if connection auth is used, and flag as candidate for pinning
1020      * in such case.
1021      * Note: we may need to set flags.connectionAuth even if the connection
1022      * is already pinned if it was pinned earlier due to proxy auth
1023      */
1024     if (!request->flags.connectionAuth) {
1025         if (req_hdr->has(Http::HdrType::AUTHORIZATION) || req_hdr->has(Http::HdrType::PROXY_AUTHORIZATION)) {
1026             HttpHeaderPos pos = HttpHeaderInitPos;
1027             HttpHeaderEntry *e;
1028             int may_pin = 0;
1029             while ((e = req_hdr->getEntry(&pos))) {
1030                 if (e->id == Http::HdrType::AUTHORIZATION || e->id == Http::HdrType::PROXY_AUTHORIZATION) {
1031                     const char *value = e->value.rawBuf();
1032                     if (strncasecmp(value, "NTLM ", 5) == 0
1033                             ||
1034                             strncasecmp(value, "Negotiate ", 10) == 0
1035                             ||
1036                             strncasecmp(value, "Kerberos ", 9) == 0) {
1037                         if (e->id == Http::HdrType::AUTHORIZATION) {
1038                             request->flags.connectionAuth = true;
1039                             may_pin = 1;
1040                         } else {
1041                             request->flags.connectionProxyAuth = true;
1042                             may_pin = 1;
1043                         }
1044                     }
1045                 }
1046             }
1047             if (may_pin && !request->pinnedConnection()) {
1048                 // These should already be linked correctly. Just need the ServerConnection to pinn.
1049                 assert(request->clientConnectionManager == http_conn);
1050             }
1051         }
1052     }
1053 }
1054
1055 static void
1056 clientInterpretRequestHeaders(ClientHttpRequest * http)
1057 {
1058     HttpRequest *request = http->request;
1059     HttpHeader *req_hdr = &request->header;
1060     bool no_cache = false;
1061     const char *str;
1062
1063     request->imslen = -1;
1064     request->ims = req_hdr->getTime(Http::HdrType::IF_MODIFIED_SINCE);
1065
1066     if (request->ims > 0)
1067         request->flags.ims = true;
1068
1069     if (!request->flags.ignoreCc) {
1070         if (request->cache_control) {
1071             if (request->cache_control->hasNoCache())
1072                 no_cache=true;
1073
1074             // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1075         } else if (req_hdr->has(Http::HdrType::PRAGMA))
1076             no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1077
1078         /*
1079         * Work around for supporting the Reload button in IE browsers when Squid
1080         * is used as an accelerator or transparent proxy, by turning accelerated
1081         * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1082         * actually only fixed in SP1, but we can't tell whether we are talking to
1083         * SP1 or not so all 5.5 versions are treated 'normally').
1084         */
1085         if (Config.onoff.ie_refresh) {
1086             if (http->flags.accel && request->flags.ims) {
1087                 if ((str = req_hdr->getStr(Http::HdrType::USER_AGENT))) {
1088                     if (strstr(str, "MSIE 5.01") != NULL)
1089                         no_cache=true;
1090                     else if (strstr(str, "MSIE 5.0") != NULL)
1091                         no_cache=true;
1092                     else if (strstr(str, "MSIE 4.") != NULL)
1093                         no_cache=true;
1094                     else if (strstr(str, "MSIE 3.") != NULL)
1095                         no_cache=true;
1096                 }
1097             }
1098         }
1099     }
1100
1101     if (request->method == Http::METHOD_OTHER) {
1102         no_cache=true;
1103     }
1104
1105     if (no_cache) {
1106 #if USE_HTTP_VIOLATIONS
1107
1108         if (Config.onoff.reload_into_ims)
1109             request->flags.nocacheHack = true;
1110         else if (refresh_nocache_hack)
1111             request->flags.nocacheHack = true;
1112         else
1113 #endif
1114
1115             request->flags.noCache = true;
1116     }
1117
1118     /* ignore range header in non-GETs or non-HEADs */
1119     if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1120         // XXX: initialize if we got here without HttpRequest::parseHeader()
1121         if (!request->range)
1122             request->range = req_hdr->getRange();
1123
1124         if (request->range) {
1125             request->flags.isRanged = true;
1126             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1127             /* XXX: This is suboptimal. We should give the stream the range set,
1128              * and thereby let the top of the stream set the offset when the
1129              * size becomes known. As it is, we will end up requesting from 0
1130              * for evey -X range specification.
1131              * RBC - this may be somewhat wrong. We should probably set the range
1132              * iter up at this point.
1133              */
1134             node->readBuffer.offset = request->range->lowestOffset(0);
1135             http->range_iter.pos = request->range->begin();
1136             http->range_iter.end = request->range->end();
1137             http->range_iter.valid = true;
1138         }
1139     }
1140
1141     /* Only HEAD and GET requests permit a Range or Request-Range header.
1142      * If these headers appear on any other type of request, delete them now.
1143      */
1144     else {
1145         req_hdr->delById(Http::HdrType::RANGE);
1146         req_hdr->delById(Http::HdrType::REQUEST_RANGE);
1147         request->ignoreRange("neither HEAD nor GET");
1148     }
1149
1150     if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1151         request->flags.auth = true;
1152
1153     clientCheckPinning(http);
1154
1155     if (!request->url.userInfo().isEmpty())
1156         request->flags.auth = true;
1157
1158     if (req_hdr->has(Http::HdrType::VIA)) {
1159         String s = req_hdr->getList(Http::HdrType::VIA);
1160         /*
1161          * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1162          * Note ThisCache2 has a space prepended to the hostname so we don't
1163          * accidentally match super-domains.
1164          */
1165
1166         if (strListIsSubstr(&s, ThisCache2, ',')) {
1167             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1168                      request, (ObjPackMethod) & httpRequestPack);
1169             request->flags.loopDetected = true;
1170         }
1171
1172 #if USE_FORW_VIA_DB
1173         fvdbCountVia(s.termedBuf());
1174
1175 #endif
1176
1177         s.clean();
1178     }
1179
1180 #if USE_FORW_VIA_DB
1181
1182     if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1183         String s = req_hdr->getList(Http::HdrType::X_FORWARDED_FOR);
1184         fvdbCountForw(s.termedBuf());
1185         s.clean();
1186     }
1187
1188 #endif
1189
1190     request->flags.cachable = http->request->maybeCacheable();
1191
1192     if (clientHierarchical(http))
1193         request->flags.hierarchical = true;
1194
1195     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1196            (request->flags.noCache ? "SET" : "NOT SET"));
1197     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1198            (request->flags.cachable ? "SET" : "NOT SET"));
1199     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1200            (request->flags.hierarchical ? "SET" : "NOT SET"));
1201
1202 }
1203
1204 void
1205 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1206 {
1207     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1208
1209     if (!calloutContext->httpStateIsValid())
1210         return;
1211
1212     calloutContext->clientRedirectDone(result);
1213 }
1214
1215 void
1216 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1217 {
1218     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1219
1220     if (!calloutContext->httpStateIsValid())
1221         return;
1222
1223     calloutContext->clientStoreIdDone(result);
1224 }
1225
1226 void
1227 ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1228 {
1229     HttpRequest *old_request = http->request;
1230     debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1231     assert(redirect_state == REDIRECT_PENDING);
1232     redirect_state = REDIRECT_DONE;
1233
1234     // Put helper response Notes into the transaction state record (ALE) eventually
1235     // do it early to ensure that no matter what the outcome the notes are present.
1236     if (http->al != NULL)
1237         (void)SyncNotes(*http->al, *old_request);
1238
1239     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1240
1241     switch (reply.result) {
1242     case Helper::TimedOut:
1243         if (Config.onUrlRewriteTimeout.action != toutActBypass) {
1244             http->calloutsError(ERR_GATEWAY_FAILURE, ERR_DETAIL_REDIRECTOR_TIMEDOUT);
1245             debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1246         }
1247         break;
1248
1249     case Helper::Unknown:
1250     case Helper::TT:
1251         // Handler in redirect.cc should have already mapped Unknown
1252         // IF it contained valid entry for the old URL-rewrite helper protocol
1253         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1254         break;
1255
1256     case Helper::BrokenHelper:
1257         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1258         break;
1259
1260     case Helper::Error:
1261         // no change to be done.
1262         break;
1263
1264     case Helper::Okay: {
1265         // #1: redirect with a specific status code    OK status=NNN url="..."
1266         // #2: redirect with a default status code     OK url="..."
1267         // #3: re-write the URL                        OK rewrite-url="..."
1268
1269         const char *statusNote = reply.notes.findFirst("status");
1270         const char *urlNote = reply.notes.findFirst("url");
1271
1272         if (urlNote != NULL) {
1273             // HTTP protocol redirect to be done.
1274
1275             // TODO: change default redirect status for appropriate requests
1276             // Squid defaults to 302 status for now for better compatibility with old clients.
1277             // HTTP/1.0 client should get 302 (Http::scFound)
1278             // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1279             // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1280             Http::StatusCode status = Http::scFound;
1281             if (statusNote != NULL) {
1282                 const char * result = statusNote;
1283                 status = static_cast<Http::StatusCode>(atoi(result));
1284             }
1285
1286             if (status == Http::scMovedPermanently
1287                     || status == Http::scFound
1288                     || status == Http::scSeeOther
1289                     || status == Http::scPermanentRedirect
1290                     || status == Http::scTemporaryRedirect) {
1291                 http->redirect.status = status;
1292                 http->redirect.location = xstrdup(urlNote);
1293                 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1294             } else {
1295                 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1296             }
1297         } else {
1298             // URL-rewrite wanted. Ew.
1299             urlNote = reply.notes.findFirst("rewrite-url");
1300
1301             // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1302             if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1303                 // XXX: validate the URL properly *without* generating a whole new request object right here.
1304                 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1305                 HttpRequest *new_request = old_request->clone();
1306                 if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1307                     debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1308
1309                     // update the new request to flag the re-writing was done on it
1310                     new_request->flags.redirected = true;
1311
1312                     // unlink bodypipe from the old request. Not needed there any longer.
1313                     if (old_request->body_pipe != NULL) {
1314                         old_request->body_pipe = NULL;
1315                         debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1316                                " from request " << old_request << " to " << new_request);
1317                     }
1318
1319                     // update the current working ClientHttpRequest fields
1320                     xfree(http->uri);
1321                     http->uri = SBufToCstring(new_request->effectiveRequestUri());
1322                     HTTPMSGUNLOCK(old_request);
1323                     http->request = new_request;
1324                     HTTPMSGLOCK(http->request);
1325                 } else {
1326                     debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1327                            old_request->method << " " << urlNote << " " << old_request->http_ver);
1328                     delete new_request;
1329                 }
1330             }
1331         }
1332     }
1333     break;
1334     }
1335
1336     /* FIXME PIPELINE: This is innacurate during pipelining */
1337
1338     if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1339         fd_note(http->getConn()->clientConnection->fd, http->uri);
1340
1341     assert(http->uri);
1342
1343     http->doCallouts();
1344 }
1345
1346 /**
1347  * This method handles the different replies from StoreID helper.
1348  */
1349 void
1350 ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1351 {
1352     HttpRequest *old_request = http->request;
1353     debugs(85, 5, "'" << http->uri << "' result=" << reply);
1354     assert(store_id_state == REDIRECT_PENDING);
1355     store_id_state = REDIRECT_DONE;
1356
1357     // Put helper response Notes into the transaction state record (ALE) eventually
1358     // do it early to ensure that no matter what the outcome the notes are present.
1359     if (http->al != NULL)
1360         (void)SyncNotes(*http->al, *old_request);
1361
1362     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1363
1364     switch (reply.result) {
1365     case Helper::Unknown:
1366     case Helper::TT:
1367         // Handler in redirect.cc should have already mapped Unknown
1368         // IF it contained valid entry for the old helper protocol
1369         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1370         break;
1371
1372     case Helper::TimedOut:
1373     // Timeouts for storeID are not implemented
1374     case Helper::BrokenHelper:
1375         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1376         break;
1377
1378     case Helper::Error:
1379         // no change to be done.
1380         break;
1381
1382     case Helper::Okay: {
1383         const char *urlNote = reply.notes.findFirst("store-id");
1384
1385         // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1386         if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1387             // Debug section required for some very specific cases.
1388             debugs(85, 9, "Setting storeID with: " << urlNote );
1389             http->request->store_id = urlNote;
1390             http->store_id = urlNote;
1391         }
1392     }
1393     break;
1394     }
1395
1396     http->doCallouts();
1397 }
1398
1399 /** Test cache allow/deny configuration
1400  *  Sets flags.cachable=1 if caching is not denied.
1401  */
1402 void
1403 ClientRequestContext::checkNoCache()
1404 {
1405     if (Config.accessList.noCache) {
1406         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1407         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1408     } else {
1409         /* unless otherwise specified, we try to cache. */
1410         checkNoCacheDone(ACCESS_ALLOWED);
1411     }
1412 }
1413
1414 static void
1415 checkNoCacheDoneWrapper(allow_t answer, void *data)
1416 {
1417     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1418
1419     if (!calloutContext->httpStateIsValid())
1420         return;
1421
1422     calloutContext->checkNoCacheDone(answer);
1423 }
1424
1425 void
1426 ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1427 {
1428     acl_checklist = NULL;
1429     http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1430     http->doCallouts();
1431 }
1432
1433 #if USE_OPENSSL
1434 bool
1435 ClientRequestContext::sslBumpAccessCheck()
1436 {
1437     // If SSL connection tunneling or bumping decision has been made, obey it.
1438     const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1439     if (bumpMode != Ssl::bumpEnd) {
1440         debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1441                "), " << "ignoring ssl_bump for " << http->getConn());
1442         if (!http->getConn()->serverBump())
1443             http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1444         http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1445         return false;
1446     }
1447
1448     // If we have not decided yet, decide whether to bump now.
1449
1450     // Bumping here can only start with a CONNECT request on a bumping port
1451     // (bumping of intercepted SSL conns is decided before we get 1st request).
1452     // We also do not bump redirected CONNECT requests.
1453     if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1454             !Config.accessList.ssl_bump ||
1455             !http->getConn()->port->flags.tunnelSslBumping) {
1456         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1457         debugs(85, 5, HERE << "cannot SslBump this request");
1458         return false;
1459     }
1460
1461     // Do not bump during authentication: clients would not proxy-authenticate
1462     // if we delay a 407 response and respond with 200 OK to CONNECT.
1463     if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1464         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1465         debugs(85, 5, HERE << "no SslBump during proxy authentication");
1466         return false;
1467     }
1468
1469     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1470
1471     ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1472     aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1473     return true;
1474 }
1475
1476 /**
1477  * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1478  * as ACLFilledChecklist callback
1479  */
1480 static void
1481 sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1482 {
1483     ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1484
1485     if (!calloutContext->httpStateIsValid())
1486         return;
1487     calloutContext->sslBumpAccessCheckDone(answer);
1488 }
1489
1490 void
1491 ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1492 {
1493     if (!httpStateIsValid())
1494         return;
1495
1496     const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1497                                    static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1498     http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1499     http->al->ssl.bumpMode = bumpMode; // for logging
1500
1501     http->doCallouts();
1502 }
1503 #endif
1504
1505 /*
1506  * Identify requests that do not go through the store and client side stream
1507  * and forward them to the appropriate location. All other requests, request
1508  * them.
1509  */
1510 void
1511 ClientHttpRequest::processRequest()
1512 {
1513     debugs(85, 4, request->method << ' ' << uri);
1514
1515     if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1516 #if USE_OPENSSL
1517         if (sslBumpNeeded()) {
1518             sslBumpStart();
1519             return;
1520         }
1521 #endif
1522         getConn()->stopReading(); // tunnels read for themselves
1523         tunnelStart(this);
1524         return;
1525     }
1526
1527     httpStart();
1528 }
1529
1530 void
1531 ClientHttpRequest::httpStart()
1532 {
1533     PROF_start(httpStart);
1534     logType = LOG_TAG_NONE;
1535     debugs(85, 4, logType.c_str() << " for '" << uri << "'");
1536
1537     /* no one should have touched this */
1538     assert(out.offset == 0);
1539     /* Use the Stream Luke */
1540     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1541     clientStreamRead(node, this, node->readBuffer);
1542     PROF_stop(httpStart);
1543 }
1544
1545 #if USE_OPENSSL
1546
1547 void
1548 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1549 {
1550     debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1551     sslBumpNeed_ = mode;
1552 }
1553
1554 // called when comm_write has completed
1555 static void
1556 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1557 {
1558     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1559     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1560
1561     assert(r && cbdataReferenceValid(r));
1562     r->sslBumpEstablish(errflag);
1563 }
1564
1565 void
1566 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1567 {
1568     // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1569     if (errflag == Comm::ERR_CLOSING)
1570         return;
1571
1572     if (errflag) {
1573         debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1574         getConn()->clientConnection->close();
1575         return;
1576     }
1577
1578     // We lack HttpReply which logRequest() uses to log the status code.
1579     // TODO: Use HttpReply instead of the "200 Connection established" string.
1580     al->http.code = 200;
1581
1582 #if USE_AUTH
1583     // Preserve authentication info for the ssl-bumped request
1584     if (request->auth_user_request != NULL)
1585         getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1586 #endif
1587
1588     assert(sslBumpNeeded());
1589     getConn()->switchToHttps(request, sslBumpNeed_);
1590 }
1591
1592 void
1593 ClientHttpRequest::sslBumpStart()
1594 {
1595     debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1596            "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1597     getConn()->sslBumpMode = sslBumpNeed_;
1598
1599     AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1600                                   CommIoCbPtrFun(&SslBumpEstablish, this));
1601
1602     if (request->flags.interceptTproxy || request->flags.intercepted) {
1603         CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1604         params.flag = Comm::OK;
1605         params.conn = getConn()->clientConnection;
1606         ScheduleCallHere(bumpCall);
1607         return;
1608     }
1609
1610     // send an HTTP 200 response to kick client SSL negotiation
1611     // TODO: Unify with tunnel.cc and add a Server(?) header
1612     static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1613     Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1614 }
1615
1616 #endif
1617
1618 bool
1619 ClientHttpRequest::gotEnough() const
1620 {
1621     /** TODO: should be querying the stream. */
1622     int64_t contentLength =
1623         memObject()->getReply()->bodySize(request->method);
1624     assert(contentLength >= 0);
1625
1626     if (out.offset < contentLength)
1627         return false;
1628
1629     return true;
1630 }
1631
1632 void
1633 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1634 {
1635     entry_ = newEntry;
1636 }
1637
1638 void
1639 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1640 {
1641     if (loggingEntry_)
1642         loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1643
1644     loggingEntry_ = newEntry;
1645
1646     if (loggingEntry_)
1647         loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1648 }
1649
1650 /*
1651  * doCallouts() - This function controls the order of "callout"
1652  * executions, including non-blocking access control checks, the
1653  * redirector, and ICAP.  Previously, these callouts were chained
1654  * together such that "clientAccessCheckDone()" would call
1655  * "clientRedirectStart()" and so on.
1656  *
1657  * The ClientRequestContext (aka calloutContext) class holds certain
1658  * state data for the callout/callback operations.  Previously
1659  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1660  * for a short time.  ClientRequestContext would then delete itself
1661  * and pass control back to ClientHttpRequest when all callouts
1662  * were finished.
1663  *
1664  * This caused some problems for ICAP because we want to make the
1665  * ICAP callout after checking ACLs, but before checking the no_cache
1666  * list.  We can't stuff the ICAP state into the ClientRequestContext
1667  * class because we still need the ICAP state after ClientRequestContext
1668  * goes away.
1669  *
1670  * Note that ClientRequestContext is created before the first call
1671  * to doCallouts().
1672  *
1673  * If one of the callouts notices that ClientHttpRequest is no
1674  * longer valid, it should call cbdataReferenceDone() so that
1675  * ClientHttpRequest's reference count goes to zero and it will get
1676  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1677  *
1678  * Note that we set the _done flags here before actually starting
1679  * the callout.  This is strictly for convenience.
1680  */
1681
1682 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1683 nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1684
1685 void
1686 ClientHttpRequest::doCallouts()
1687 {
1688     assert(calloutContext);
1689
1690     /*Save the original request for logging purposes*/
1691     if (!calloutContext->http->al->request) {
1692         calloutContext->http->al->request = request;
1693         HTTPMSGLOCK(calloutContext->http->al->request);
1694
1695         NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1696         // Make the previously set client connection ID available as annotation.
1697         if (ConnStateData *csd = calloutContext->http->getConn()) {
1698             if (!csd->connectionTag().isEmpty())
1699                 notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1700         }
1701     }
1702
1703     if (!calloutContext->error) {
1704         // CVE-2009-0801: verify the Host: header is consistent with other known details.
1705         if (!calloutContext->host_header_verify_done) {
1706             debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1707             calloutContext->host_header_verify_done = true;
1708             calloutContext->hostHeaderVerify();
1709             return;
1710         }
1711
1712         if (!calloutContext->http_access_done) {
1713             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1714             calloutContext->http_access_done = true;
1715             calloutContext->clientAccessCheck();
1716             return;
1717         }
1718
1719 #if USE_ADAPTATION
1720         if (!calloutContext->adaptation_acl_check_done) {
1721             calloutContext->adaptation_acl_check_done = true;
1722             if (Adaptation::AccessCheck::Start(
1723                         Adaptation::methodReqmod, Adaptation::pointPreCache,
1724                         request, NULL, calloutContext->http->al, this))
1725                 return; // will call callback
1726         }
1727 #endif
1728
1729         if (!calloutContext->redirect_done) {
1730             calloutContext->redirect_done = true;
1731
1732             if (Config.Program.redirect) {
1733                 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1734                 calloutContext->redirect_state = REDIRECT_PENDING;
1735                 calloutContext->clientRedirectStart();
1736                 return;
1737             }
1738         }
1739
1740         if (!calloutContext->adapted_http_access_done) {
1741             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1742             calloutContext->adapted_http_access_done = true;
1743             calloutContext->clientAccessCheck2();
1744             return;
1745         }
1746
1747         if (!calloutContext->store_id_done) {
1748             calloutContext->store_id_done = true;
1749
1750             if (Config.Program.store_id) {
1751                 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1752                 calloutContext->store_id_state = REDIRECT_PENDING;
1753                 calloutContext->clientStoreIdStart();
1754                 return;
1755             }
1756         }
1757
1758         if (!calloutContext->interpreted_req_hdrs) {
1759             debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1760             calloutContext->interpreted_req_hdrs = 1;
1761             clientInterpretRequestHeaders(this);
1762         }
1763
1764         if (!calloutContext->no_cache_done) {
1765             calloutContext->no_cache_done = true;
1766
1767             if (Config.accessList.noCache && request->flags.cachable) {
1768                 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1769                 calloutContext->checkNoCache();
1770                 return;
1771             }
1772         }
1773     } //  if !calloutContext->error
1774
1775     if (!calloutContext->tosToClientDone) {
1776         calloutContext->tosToClientDone = true;
1777         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1778             ACLFilledChecklist ch(NULL, request, NULL);
1779             ch.src_addr = request->client_addr;
1780             ch.my_addr = request->my_addr;
1781             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1782             if (tos)
1783                 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1784         }
1785     }
1786
1787     if (!calloutContext->nfmarkToClientDone) {
1788         calloutContext->nfmarkToClientDone = true;
1789         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1790             ACLFilledChecklist ch(NULL, request, NULL);
1791             ch.src_addr = request->client_addr;
1792             ch.my_addr = request->my_addr;
1793             nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1794             if (mark)
1795                 Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1796         }
1797     }
1798
1799 #if USE_OPENSSL
1800     // We need to check for SslBump even if the calloutContext->error is set
1801     // because bumping may require delaying the error until after CONNECT.
1802     if (!calloutContext->sslBumpCheckDone) {
1803         calloutContext->sslBumpCheckDone = true;
1804         if (calloutContext->sslBumpAccessCheck())
1805             return;
1806         /* else no ssl bump required*/
1807     }
1808 #endif
1809
1810     if (calloutContext->error) {
1811         // XXX: prformance regression. c_str() reallocates
1812         SBuf storeUri(request->storeId());
1813         StoreEntry *e = storeCreateEntry(storeUri.c_str(), storeUri.c_str(), request->flags, request->method);
1814 #if USE_OPENSSL
1815         if (sslBumpNeeded()) {
1816             // We have to serve an error, so bump the client first.
1817             sslBumpNeed(Ssl::bumpClientFirst);
1818             // set final error but delay sending until we bump
1819             Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1820             errorAppendEntry(e, calloutContext->error);
1821             calloutContext->error = NULL;
1822             getConn()->setServerBump(srvBump);
1823             e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1824         } else
1825 #endif
1826         {
1827             // send the error to the client now
1828             clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1829             clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1830             assert (repContext);
1831             repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1832             errorAppendEntry(e, calloutContext->error);
1833             calloutContext->error = NULL;
1834             if (calloutContext->readNextRequest)
1835                 getConn()->flags.readMore = true; // resume any pipeline reads.
1836             node = (clientStreamNode *)client_stream.tail->data;
1837             clientStreamRead(node, this, node->readBuffer);
1838             e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1839             return;
1840         }
1841     }
1842
1843     cbdataReferenceDone(calloutContext->http);
1844     delete calloutContext;
1845     calloutContext = NULL;
1846 #if HEADERS_LOG
1847
1848     headersLog(0, 1, request->method, request);
1849 #endif
1850
1851     debugs(83, 3, HERE << "calling processRequest()");
1852     processRequest();
1853
1854 #if ICAP_CLIENT
1855     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1856     if (ih != NULL)
1857         ih->logType = logType;
1858 #endif
1859 }
1860
1861 #if !_USE_INLINE_
1862 #include "client_side_request.cci"
1863 #endif
1864
1865 #if USE_ADAPTATION
1866 /// Initiate an asynchronous adaptation transaction which will call us back.
1867 void
1868 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1869 {
1870     debugs(85, 3, HERE << "adaptation needed for " << this);
1871     assert(!virginHeadSource);
1872     assert(!adaptedBodySource);
1873     virginHeadSource = initiateAdaptation(
1874                            new Adaptation::Iterator(request, NULL, al, g));
1875
1876     // we could try to guess whether we can bypass this adaptation
1877     // initiation failure, but it should not really happen
1878     Must(initiated(virginHeadSource));
1879 }
1880
1881 void
1882 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1883 {
1884     assert(cbdataReferenceValid(this));     // indicates bug
1885     clearAdaptation(virginHeadSource);
1886     assert(!adaptedBodySource);
1887
1888     switch (answer.kind) {
1889     case Adaptation::Answer::akForward:
1890         handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1891         break;
1892
1893     case Adaptation::Answer::akBlock:
1894         handleAdaptationBlock(answer);
1895         break;
1896
1897     case Adaptation::Answer::akError:
1898         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1899         break;
1900     }
1901 }
1902
1903 void
1904 ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1905 {
1906     assert(msg);
1907
1908     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1909         /*
1910          * Replace the old request with the new request.
1911          */
1912         HTTPMSGUNLOCK(request);
1913         request = new_req;
1914         HTTPMSGLOCK(request);
1915
1916         // update the new message to flag whether URL re-writing was done on it
1917         if (request->effectiveRequestUri().cmp(uri) != 0)
1918             request->flags.redirected = 1;
1919
1920         /*
1921          * Store the new URI for logging
1922          */
1923         xfree(uri);
1924         uri = SBufToCstring(request->effectiveRequestUri());
1925         setLogUri(this, urlCanonicalClean(request));
1926         assert(request->method.id());
1927     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1928         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1929
1930         // subscribe to receive reply body
1931         if (new_rep->body_pipe != NULL) {
1932             adaptedBodySource = new_rep->body_pipe;
1933             int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1934             assert(consumer_ok);
1935         }
1936
1937         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1938         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1939         assert(repContext);
1940         repContext->createStoreEntry(request->method, request->flags);
1941
1942         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1943         request_satisfaction_mode = true;
1944         request_satisfaction_offset = 0;
1945         storeEntry()->replaceHttpReply(new_rep);
1946         storeEntry()->timestampsSet();
1947
1948         if (!adaptedBodySource) // no body
1949             storeEntry()->complete();
1950         clientGetMoreData(node, this);
1951     }
1952
1953     // we are done with getting headers (but may be receiving body)
1954     clearAdaptation(virginHeadSource);
1955
1956     if (!request_satisfaction_mode)
1957         doCallouts();
1958 }
1959
1960 void
1961 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1962 {
1963     request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1964     AclMatchedName = answer.ruleId.termedBuf();
1965     assert(calloutContext);
1966     calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1967     AclMatchedName = NULL;
1968 }
1969
1970 void
1971 ClientHttpRequest::resumeBodyStorage()
1972 {
1973     if (!adaptedBodySource)
1974         return;
1975
1976     noteMoreBodyDataAvailable(adaptedBodySource);
1977 }
1978
1979 void
1980 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1981 {
1982     assert(request_satisfaction_mode);
1983     assert(adaptedBodySource != NULL);
1984
1985     if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1986         const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1987
1988         if (spaceAvailable < contentSize ) {
1989             // No or partial body data consuming
1990             typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1991             AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1992                                                 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1993             storeEntry()->deferProducer(call);
1994         }
1995
1996         if (!spaceAvailable)
1997             return;
1998
1999         if (spaceAvailable < contentSize )
2000             contentSize = spaceAvailable;
2001
2002         BodyPipeCheckout bpc(*adaptedBodySource);
2003         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
2004         storeEntry()->write(ioBuf);
2005         // assume StoreEntry::write() writes the entire ioBuf
2006         request_satisfaction_offset += ioBuf.length;
2007         bpc.buf.consume(contentSize);
2008         bpc.checkIn();
2009     }
2010
2011     if (adaptedBodySource->exhausted())
2012         endRequestSatisfaction();
2013     // else wait for more body data
2014 }
2015
2016 void
2017 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
2018 {
2019     assert(!virginHeadSource);
2020     // should we end request satisfaction now?
2021     if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
2022         endRequestSatisfaction();
2023 }
2024
2025 void
2026 ClientHttpRequest::endRequestSatisfaction()
2027 {
2028     debugs(85,4, HERE << this << " ends request satisfaction");
2029     assert(request_satisfaction_mode);
2030     stopConsumingFrom(adaptedBodySource);
2031
2032     // TODO: anything else needed to end store entry formation correctly?
2033     storeEntry()->complete();
2034 }
2035
2036 void
2037 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2038 {
2039     assert(!virginHeadSource);
2040     stopConsumingFrom(adaptedBodySource);
2041
2042     debugs(85,3, HERE << "REQMOD body production failed");
2043     if (request_satisfaction_mode) { // too late to recover or serve an error
2044         request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2045         const Comm::ConnectionPointer c = getConn()->clientConnection;
2046         Must(Comm::IsConnOpen(c));
2047         c->close(); // drastic, but we may be writing a response already
2048     } else {
2049         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2050     }
2051 }
2052
2053 void
2054 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2055 {
2056     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2057
2058     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2059     const bool usedPipe = request->body_pipe != NULL &&
2060                           request->body_pipe->consumedSize() > 0;
2061
2062     if (bypassable && !usedStore && !usedPipe) {
2063         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2064         if (calloutContext)
2065             doCallouts();
2066         return;
2067     }
2068
2069     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2070
2071     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2072     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2073     assert(repContext);
2074
2075     calloutsError(ERR_ICAP_FAILURE, errDetail);
2076
2077     if (calloutContext)
2078         doCallouts();
2079 }
2080
2081 #endif
2082
2083 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2084 void
2085 ClientHttpRequest::calloutsError(const err_type error, const int errDetail)
2086 {
2087     // The original author of the code also wanted to pass an errno to
2088     // setReplyToError, but it seems unlikely that the errno reflects the
2089     // true cause of the error at this point, so I did not pass it.
2090     if (calloutContext) {
2091         Ip::Address noAddr;
2092         noAddr.setNoAddr();
2093         ConnStateData * c = getConn();
2094         calloutContext->error = clientBuildError(error, Http::scInternalServerError,
2095                                 NULL,
2096                                 c != NULL ? c->clientConnection->remote : noAddr,
2097                                 request
2098                                                 );
2099 #if USE_AUTH
2100         calloutContext->error->auth_user_request =
2101             c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2102 #endif
2103         calloutContext->error->detailError(errDetail);
2104         calloutContext->readNextRequest = true;
2105         if (c != NULL)
2106             c->expectNoForwarding();
2107     }
2108     //else if(calloutContext == NULL) is it possible?
2109 }
2110