src/client_side_request.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 85    Client-side Request Routines */
  10
  11 /*
  12  * General logic of request processing:
  13  *
  14  * We run a series of tests to determine if access will be permitted, and to do
  15  * any redirection. Then we call into the result clientStream to retrieve data.
  16  * From that point on it's up to reply management.
  17  */
  18
  19 #include "squid.h"
  20 #include "acl/FilledChecklist.h"
  21 #include "acl/Gadgets.h"
  22 #include "anyp/PortCfg.h"
  23 #include "base/AsyncJobCalls.h"
  24 #include "client_side.h"
  25 #include "client_side_reply.h"
  26 #include "client_side_request.h"
  27 #include "ClientRequestContext.h"
  28 #include "clientStream.h"
  29 #include "comm/Connection.h"
  30 #include "comm/Write.h"
  31 #include "err_detail_type.h"
  32 #include "errorpage.h"
  33 #include "fd.h"
  34 #include "fde.h"
  35 #include "format/Token.h"
  36 #include "gopher.h"
  37 #include "helper.h"
  38 #include "helper/Reply.h"
  39 #include "http.h"
  40 #include "http/Stream.h"
  41 #include "HttpHdrCc.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequest.h"
  44 #include "ip/QosConfig.h"
  45 #include "ipcache.h"
  46 #include "log/access_log.h"
  47 #include "MemObject.h"
  48 #include "Parsing.h"
  49 #include "profiler/Profiler.h"
  50 #include "redirect.h"
  51 #include "SquidConfig.h"
  52 #include "SquidTime.h"
  53 #include "Store.h"
  54 #include "StrList.h"
  55 #include "tools.h"
  56 #include "URL.h"
  57 #include "wordlist.h"
  58 #if USE_AUTH
  59 #include "auth/UserRequest.h"
  60 #endif
  61 #if USE_ADAPTATION
  62 #include "adaptation/AccessCheck.h"
  63 #include "adaptation/Answer.h"
  64 #include "adaptation/Iterator.h"
  65 #include "adaptation/Service.h"
  66 #if ICAP_CLIENT
  67 #include "adaptation/icap/History.h"
  68 #endif
  69 #endif
  70 #if USE_OPENSSL
  71 #include "ssl/ServerBump.h"
  72 #include "ssl/support.h"
  73 #endif
  74
  75 #if LINGERING_CLOSE
  76 #define comm_close comm_lingering_close
  77 #endif
  78
  79 static const char *const crlf = "\r\n";
  80
  81 #if FOLLOW_X_FORWARDED_FOR
  82 static void clientFollowXForwardedForCheck(allow_t answer, void *data);
  83 #endif /* FOLLOW_X_FORWARDED_FOR */
  84
  85 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
  86
  87 CBDATA_CLASS_INIT(ClientRequestContext);
  88
  89 /* Local functions */
  90 /* other */
  91 static void clientAccessCheckDoneWrapper(allow_t, void *);
  92 #if USE_OPENSSL
  93 static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
  94 #endif
  95 static int clientHierarchical(ClientHttpRequest * http);
  96 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  97 static HLPCB clientRedirectDoneWrapper;
  98 static HLPCB clientStoreIdDoneWrapper;
  99 static void checkNoCacheDoneWrapper(allow_t, void *);
 100 SQUIDCEXTERN CSR clientGetMoreData;
 101 SQUIDCEXTERN CSS clientReplyStatus;
 102 SQUIDCEXTERN CSD clientReplyDetach;
 103 static void checkFailureRatio(err_type, hier_code);
 104
 105 ClientRequestContext::~ClientRequestContext()
 106 {
 107     /*
 108      * Release our "lock" on our parent, ClientHttpRequest, if we
 109      * still have one
 110      */
 111
 112     if (http)
 113         cbdataReferenceDone(http);
 114
 115     delete error;
 116     debugs(85,3, "ClientRequestContext destructed, this=" << this);
 117 }
 118
 119 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) :
 120     http(cbdataReference(anHttp)),
 121     acl_checklist(NULL),
 122     redirect_state(REDIRECT_NONE),
 123     store_id_state(REDIRECT_NONE),
 124     host_header_verify_done(false),
 125     http_access_done(false),
 126     adapted_http_access_done(false),
 127 #if USE_ADAPTATION
 128     adaptation_acl_check_done(false),
 129 #endif
 130     redirect_done(false),
 131     store_id_done(false),
 132     no_cache_done(false),
 133     interpreted_req_hdrs(false),
 134     tosToClientDone(false),
 135     nfmarkToClientDone(false),
 136 #if USE_OPENSSL
 137     sslBumpCheckDone(false),
 138 #endif
 139     error(NULL),
 140     readNextRequest(false)
 141 {
 142     debugs(85, 3, "ClientRequestContext constructed, this=" << this);
 143 }
 144
 145 CBDATA_CLASS_INIT(ClientHttpRequest);
 146
 147 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 148 #if USE_ADAPTATION
 149     AsyncJob("ClientHttpRequest"),
 150 #endif
 151     request(NULL),
 152     uri(NULL),
 153     log_uri(NULL),
 154     req_sz(0),
 155     logType(LOG_TAG_NONE),
 156     calloutContext(NULL),
 157     maxReplyBodySize_(0),
 158     entry_(NULL),
 159     loggingEntry_(NULL),
 160     conn_(NULL)
 161 #if USE_OPENSSL
 162     , sslBumpNeed_(Ssl::bumpEnd)
 163 #endif
 164 #if USE_ADAPTATION
 165     , request_satisfaction_mode(false)
 166     , request_satisfaction_offset(0)
 167 #endif
 168 {
 169     setConn(aConn);
 170     al = new AccessLogEntry;
 171     al->cache.start_time = current_time;
 172     if (aConn) {
 173         al->tcpClient = clientConnection = aConn->clientConnection;
 174         al->cache.port = aConn->port;
 175         al->cache.caddr = aConn->log_addr;
 176
 177 #if USE_OPENSSL
 178         if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
 179             if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get())
 180                 al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
 181         }
 182 #endif
 183     }
 184     dlinkAdd(this, &active, &ClientActiveRequests);
 185 }
 186
 187 /*
 188  * returns true if client specified that the object must come from the cache
 189  * without contacting origin server
 190  */
 191 bool
 192 ClientHttpRequest::onlyIfCached()const
 193 {
 194     assert(request);
 195     return request->cache_control &&
 196            request->cache_control->onlyIfCached();
 197 }
 198
 199 /**
 200  * This function is designed to serve a fairly specific purpose.
 201  * Occasionally our vBNS-connected caches can talk to each other, but not
 202  * the rest of the world.  Here we try to detect frequent failures which
 203  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 204  * the failure:success ratio goes above 1.0 then we go into "hit only"
 205  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 206  * will only fetch HITs from us if they are using the ICP protocol.  We
 207  * stay in this mode for 5 minutes.
 208  *
 209  * Duane W., Sept 16, 1996
 210  */
 211 static void
 212 checkFailureRatio(err_type etype, hier_code hcode)
 213 {
 214     // Can be set at compile time with -D compiler flag
 215 #ifndef FAILURE_MODE_TIME
 216 #define FAILURE_MODE_TIME 300
 217 #endif
 218
 219     if (hcode == HIER_NONE)
 220         return;
 221
 222     // don't bother when ICP is disabled.
 223     if (Config.Port.icp <= 0)
 224         return;
 225
 226     static double magic_factor = 100.0;
 227     double n_good;
 228     double n_bad;
 229
 230     n_good = magic_factor / (1.0 + request_failure_ratio);
 231
 232     n_bad = magic_factor - n_good;
 233
 234     switch (etype) {
 235
 236     case ERR_DNS_FAIL:
 237
 238     case ERR_CONNECT_FAIL:
 239     case ERR_SECURE_CONNECT_FAIL:
 240
 241     case ERR_READ_ERROR:
 242         ++n_bad;
 243         break;
 244
 245     default:
 246         ++n_good;
 247     }
 248
 249     request_failure_ratio = n_bad / n_good;
 250
 251     if (hit_only_mode_until > squid_curtime)
 252         return;
 253
 254     if (request_failure_ratio < 1.0)
 255         return;
 256
 257     debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
 258            std::setprecision(3) << request_failure_ratio);
 259
 260     debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
 261            FAILURE_MODE_TIME / 60 << " minutes...");
 262
 263     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 264
 265     request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
 266 }
 267
 268 ClientHttpRequest::~ClientHttpRequest()
 269 {
 270     debugs(33, 3, "httpRequestFree: " << uri);
 271     PROF_start(httpRequestFree);
 272
 273     // Even though freeResources() below may destroy the request,
 274     // we no longer set request->body_pipe to NULL here
 275     // because we did not initiate that pipe (ConnStateData did)
 276
 277     /* the ICP check here was erroneous
 278      * - StoreEntry::releaseRequest was always called if entry was valid
 279      */
 280
 281     logRequest();
 282
 283     loggingEntry(NULL);
 284
 285     if (request)
 286         checkFailureRatio(request->errType, al->hier.code);
 287
 288     freeResources();
 289
 290 #if USE_ADAPTATION
 291     announceInitiatorAbort(virginHeadSource);
 292
 293     if (adaptedBodySource != NULL)
 294         stopConsumingFrom(adaptedBodySource);
 295 #endif
 296
 297     if (calloutContext)
 298         delete calloutContext;
 299
 300     clientConnection = NULL;
 301
 302     if (conn_)
 303         cbdataReferenceDone(conn_);
 304
 305     /* moving to the next connection is handled by the context free */
 306     dlinkDelete(&active, &ClientActiveRequests);
 307
 308     PROF_stop(httpRequestFree);
 309 }
 310
 311 /**
 312  * Create a request and kick it off
 313  *
 314  * \retval 0     success
 315  * \retval -1    failure
 316  *
 317  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 318  * determined by the user
 319  */
 320 int
 321 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 322                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 323                    char *tailbuf, size_t taillen)
 324 {
 325     size_t url_sz;
 326     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 327     HttpRequest *request;
 328     StoreIOBuffer tempBuffer;
 329     if (http->al != NULL)
 330         http->al->cache.start_time = current_time;
 331     /* this is only used to adjust the connection offset in client_side.c */
 332     http->req_sz = 0;
 333     tempBuffer.length = taillen;
 334     tempBuffer.data = tailbuf;
 335     /* client stream setup */
 336     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 337                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 338                      streamdetach, streamdata, tempBuffer);
 339     /* make it visible in the 'current acctive requests list' */
 340     /* Set flags */
 341     /* internal requests only makes sense in an
 342      * accelerator today. TODO: accept flags ? */
 343     http->flags.accel = true;
 344     /* allow size for url rewriting */
 345     url_sz = strlen(url) + Config.appendDomainLen + 5;
 346     http->uri = (char *)xcalloc(url_sz, 1);
 347     strcpy(http->uri, url);
 348
 349     if ((request = HttpRequest::CreateFromUrl(http->uri, method)) == NULL) {
 350         debugs(85, 5, "Invalid URL: " << http->uri);
 351         return -1;
 352     }
 353
 354     /*
 355      * now update the headers in request with our supplied headers. urlParse
 356      * should return a blank header set, but we use Update to be sure of
 357      * correctness.
 358      */
 359     if (header)
 360         request->header.update(header);
 361
 362     http->log_uri = xstrdup(urlCanonicalClean(request));
 363
 364     /* http struct now ready */
 365
 366     /*
 367      * build new header list *? TODO
 368      */
 369     request->flags.accelerated = http->flags.accel;
 370
 371     request->flags.internalClient = true;
 372
 373     /* this is an internally created
 374      * request, not subject to acceleration
 375      * target overrides */
 376     /*
 377      * FIXME? Do we want to detect and handle internal requests of internal
 378      * objects ?
 379      */
 380
 381     /* Internally created requests cannot have bodies today */
 382     request->content_length = 0;
 383
 384     request->client_addr.setNoAddr();
 385
 386 #if FOLLOW_X_FORWARDED_FOR
 387     request->indirect_client_addr.setNoAddr();
 388 #endif /* FOLLOW_X_FORWARDED_FOR */
 389
 390     request->my_addr.setNoAddr();   /* undefined for internal requests */
 391
 392     request->my_addr.port(0);
 393
 394     request->http_ver = Http::ProtocolVersion();
 395
 396     http->request = request;
 397     HTTPMSGLOCK(http->request);
 398
 399     /* optional - skip the access check ? */
 400     http->calloutContext = new ClientRequestContext(http);
 401
 402     http->calloutContext->http_access_done = false;
 403
 404     http->calloutContext->redirect_done = true;
 405
 406     http->calloutContext->no_cache_done = true;
 407
 408     http->doCallouts();
 409
 410     return 0;
 411 }
 412
 413 bool
 414 ClientRequestContext::httpStateIsValid()
 415 {
 416     ClientHttpRequest *http_ = http;
 417
 418     if (cbdataReferenceValid(http_))
 419         return true;
 420
 421     http = NULL;
 422
 423     cbdataReferenceDone(http_);
 424
 425     return false;
 426 }
 427
 428 #if FOLLOW_X_FORWARDED_FOR
 429 /**
 430  * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
 431  * against the followXFF ACL, or cleans up and passes control to
 432  * clientAccessCheck().
 433  *
 434  * The trust model here is a little ambiguous. So to clarify the logic:
 435  * - we may always use the direct client address as the client IP.
 436  * - these trust tests merey tell whether we trust given IP enough to believe the
 437  *   IP string which it appended to the X-Forwarded-For: header.
 438  * - if at any point we don't trust what an IP adds we stop looking.
 439  * - at that point the current contents of indirect_client_addr are the value set
 440  *   by the last previously trusted IP.
 441  * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
 442  */
 443 static void
 444 clientFollowXForwardedForCheck(allow_t answer, void *data)
 445 {
 446     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 447
 448     if (!calloutContext->httpStateIsValid())
 449         return;
 450
 451     ClientHttpRequest *http = calloutContext->http;
 452     HttpRequest *request = http->request;
 453
 454     /*
 455      * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
 456      * called as a result of ACL checks, or -1 if we are called when
 457      * there's nothing left to do.
 458      */
 459     if (answer == ACCESS_ALLOWED &&
 460             request->x_forwarded_for_iterator.size () != 0) {
 461
 462         /*
 463          * Remove the last comma-delimited element from the
 464          * x_forwarded_for_iterator and use it to repeat the cycle.
 465          */
 466         const char *p;
 467         const char *asciiaddr;
 468         int l;
 469         Ip::Address addr;
 470         p = request->x_forwarded_for_iterator.termedBuf();
 471         l = request->x_forwarded_for_iterator.size();
 472
 473         /*
 474         * XXX x_forwarded_for_iterator should really be a list of
 475         * IP addresses, but it's a String instead.  We have to
 476         * walk backwards through the String, biting off the last
 477         * comma-delimited part each time.  As long as the data is in
 478         * a String, we should probably implement and use a variant of
 479         * strListGetItem() that walks backwards instead of forwards
 480         * through a comma-separated list.  But we don't even do that;
 481         * we just do the work in-line here.
 482         */
 483         /* skip trailing space and commas */
 484         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 485             --l;
 486         request->x_forwarded_for_iterator.cut(l);
 487         /* look for start of last item in list */
 488         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 489             --l;
 490         asciiaddr = p+l;
 491         if ((addr = asciiaddr)) {
 492             request->indirect_client_addr = addr;
 493             request->x_forwarded_for_iterator.cut(l);
 494             calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 495             if (!Config.onoff.acl_uses_indirect_client) {
 496                 /* override the default src_addr tested if we have to go deeper than one level into XFF */
 497                 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
 498             }
 499             calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
 500             return;
 501         }
 502     } /*if (answer == ACCESS_ALLOWED &&
 503         request->x_forwarded_for_iterator.size () != 0)*/
 504
 505     /* clean up, and pass control to clientAccessCheck */
 506     if (Config.onoff.log_uses_indirect_client) {
 507         /*
 508         * Ensure that the access log shows the indirect client
 509         * instead of the direct client.
 510         */
 511         ConnStateData *conn = http->getConn();
 512         conn->log_addr = request->indirect_client_addr;
 513         http->al->cache.caddr = conn->log_addr;
 514     }
 515     request->x_forwarded_for_iterator.clean();
 516     request->flags.done_follow_x_forwarded_for = true;
 517
 518     if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
 519         debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
 520     }
 521
 522     /* process actual access ACL as normal. */
 523     calloutContext->clientAccessCheck();
 524 }
 525 #endif /* FOLLOW_X_FORWARDED_FOR */
 526
 527 static void
 528 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const Dns::LookupDetails &dns, void *data)
 529 {
 530     ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
 531     c->hostHeaderIpVerify(ia, dns);
 532 }
 533
 534 void
 535 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const Dns::LookupDetails &dns)
 536 {
 537     Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
 538
 539     // note the DNS details for the transaction stats.
 540     http->request->recordLookup(dns);
 541
 542     if (ia != NULL && ia->count > 0) {
 543         // Is the NAT destination IP in DNS?
 544         for (int i = 0; i < ia->count; ++i) {
 545             if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
 546                 debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
 547                 http->request->flags.hostVerified = true;
 548                 http->doCallouts();
 549                 return;
 550             }
 551             debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
 552         }
 553     }
 554     debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
 555     hostHeaderVerifyFailed("local IP", "any domain IP");
 556 }
 557
 558 void
 559 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
 560 {
 561     // IP address validation for Host: failed. Admin wants to ignore them.
 562     // NP: we do not yet handle CONNECT tunnels well, so ignore for them
 563     if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
 564         debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
 565                " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
 566
 567         // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
 568         // The problems here are about WRITE for new cache content, which means flags.cachable
 569         http->request->flags.cachable = false; // MUST NOT cache (for now)
 570         // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
 571         http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
 572         // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
 573         http->doCallouts();
 574         return;
 575     }
 576
 577     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
 578            http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
 579     if (const char *ua = http->request->header.getStr(Http::HdrType::USER_AGENT))
 580         debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << ua);
 581     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
 582
 583     // IP address validation for Host: failed. reject the connection.
 584     clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 585     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 586     assert (repContext);
 587     repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
 588                                 http->request->method, NULL,
 589                                 http->getConn()->clientConnection->remote,
 590                                 http->request,
 591                                 NULL,
 592 #if USE_AUTH
 593                                 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 594                                 http->getConn()->getAuth() : http->request->auth_user_request);
 595 #else
 596                                 NULL);
 597 #endif
 598     node = (clientStreamNode *)http->client_stream.tail->data;
 599     clientStreamRead(node, http, node->readBuffer);
 600 }
 601
 602 void
 603 ClientRequestContext::hostHeaderVerify()
 604 {
 605     // Require a Host: header.
 606     const char *host = http->request->header.getStr(Http::HdrType::HOST);
 607
 608     if (!host) {
 609         // TODO: dump out the HTTP/1.1 error about missing host header.
 610         // otherwise this is fine, can't forge a header value when its not even set.
 611         debugs(85, 3, HERE << "validate skipped with no Host: header present.");
 612         http->doCallouts();
 613         return;
 614     }
 615
 616     if (http->request->flags.internal) {
 617         // TODO: kill this when URL handling allows partial URLs out of accel mode
 618         //       and we no longer screw with the URL just to add our internal host there
 619         debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
 620         http->doCallouts();
 621         return;
 622     }
 623
 624     // Locate if there is a port attached, strip ready for IP lookup
 625     char *portStr = NULL;
 626     char *hostB = xstrdup(host);
 627     host = hostB;
 628     if (host[0] == '[') {
 629         // IPv6 literal.
 630         portStr = strchr(hostB, ']');
 631         if (portStr && *(++portStr) != ':') {
 632             portStr = NULL;
 633         }
 634     } else {
 635         // Domain or IPv4 literal with port
 636         portStr = strrchr(hostB, ':');
 637     }
 638
 639     uint16_t port = 0;
 640     if (portStr) {
 641         *portStr = '\0'; // strip the ':'
 642         if (*(++portStr) != '\0') {
 643             char *end = NULL;
 644             int64_t ret = strtoll(portStr, &end, 10);
 645             if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
 646                 // invalid port details. Replace the ':'
 647                 *(--portStr) = ':';
 648                 portStr = NULL;
 649             } else
 650                 port = (ret & 0xFFFF);
 651         }
 652     }
 653
 654     debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
 655     if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
 656         // verify the Host: port (if any) matches the apparent destination
 657         if (portStr && port != http->getConn()->clientConnection->local.port()) {
 658             debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
 659                    " matches Host: port " << port << " (" << portStr << ")");
 660             hostHeaderVerifyFailed("intercepted port", portStr);
 661         } else {
 662             // XXX: match the scheme default port against the apparent destination
 663
 664             // verify the destination DNS is one of the Host: headers IPs
 665             ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
 666         }
 667     } else if (!Config.onoff.hostStrictVerify) {
 668         debugs(85, 3, "validate skipped.");
 669         http->doCallouts();
 670     } else if (strlen(host) != strlen(http->request->url.host())) {
 671         // Verify forward-proxy requested URL domain matches the Host: header
 672         debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
 673         hostHeaderVerifyFailed(host, http->request->url.host());
 674     } else if (matchDomainName(host, http->request->url.host()) != 0) {
 675         // Verify forward-proxy requested URL domain matches the Host: header
 676         debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
 677         hostHeaderVerifyFailed(host, http->request->url.host());
 678     } else if (portStr && port != http->request->url.port()) {
 679         // Verify forward-proxy requested URL domain matches the Host: header
 680         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
 681         hostHeaderVerifyFailed("URL port", portStr);
 682     } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
 683         // Verify forward-proxy requested URL domain matches the Host: header
 684         // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
 685         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
 686         hostHeaderVerifyFailed("URL port", "default port");
 687     } else {
 688         // Okay no problem.
 689         debugs(85, 3, "validate passed.");
 690         http->request->flags.hostVerified = true;
 691         http->doCallouts();
 692     }
 693     safe_free(hostB);
 694 }
 695
 696 /* This is the entry point for external users of the client_side routines */
 697 void
 698 ClientRequestContext::clientAccessCheck()
 699 {
 700 #if FOLLOW_X_FORWARDED_FOR
 701     if (!http->request->flags.doneFollowXff() &&
 702             Config.accessList.followXFF &&
 703             http->request->header.has(Http::HdrType::X_FORWARDED_FOR)) {
 704
 705         /* we always trust the direct client address for actual use */
 706         http->request->indirect_client_addr = http->request->client_addr;
 707         http->request->indirect_client_addr.port(0);
 708
 709         /* setup the XFF iterator for processing */
 710         http->request->x_forwarded_for_iterator = http->request->header.getList(Http::HdrType::X_FORWARDED_FOR);
 711
 712         /* begin by checking to see if we trust direct client enough to walk XFF */
 713         acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 714         acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
 715         return;
 716     }
 717 #endif
 718
 719     if (Config.accessList.http) {
 720         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 721         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 722     } else {
 723         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 724         clientAccessCheckDone(ACCESS_DENIED);
 725     }
 726 }
 727
 728 /**
 729  * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
 730  * The default here is to allow all. Since the earlier http_access should do a default deny all.
 731  * This check is just for a last-minute denial based on adapted request headers.
 732  */
 733 void
 734 ClientRequestContext::clientAccessCheck2()
 735 {
 736     if (Config.accessList.adapted_http) {
 737         acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
 738         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 739     } else {
 740         debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
 741         clientAccessCheckDone(ACCESS_ALLOWED);
 742     }
 743 }
 744
 745 void
 746 clientAccessCheckDoneWrapper(allow_t answer, void *data)
 747 {
 748     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 749
 750     if (!calloutContext->httpStateIsValid())
 751         return;
 752
 753     calloutContext->clientAccessCheckDone(answer);
 754 }
 755
 756 void
 757 ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
 758 {
 759     acl_checklist = NULL;
 760     err_type page_id;
 761     Http::StatusCode status;
 762     debugs(85, 2, "The request " << http->request->method << ' ' <<
 763            http->uri << " is " << answer <<
 764            "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
 765
 766 #if USE_AUTH
 767     char const *proxy_auth_msg = "<null>";
 768     if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
 769         proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
 770     else if (http->request->auth_user_request != NULL)
 771         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 772 #endif
 773
 774     if (answer != ACCESS_ALLOWED) {
 775         // auth has a grace period where credentials can be expired but okay not to challenge.
 776
 777         /* Send an auth challenge or error */
 778         // XXX: do we still need aclIsProxyAuth() ?
 779         bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
 780         debugs(85, 5, "Access Denied: " << http->uri);
 781         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 782 #if USE_AUTH
 783         if (auth_challenge)
 784             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 785 #endif
 786
 787         /*
 788          * NOTE: get page_id here, based on AclMatchedName because if
 789          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 790          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 791          * <pribeiro@isel.pt>
 792          */
 793         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
 794
 795         http->logType = LOG_TCP_DENIED;
 796
 797         if (auth_challenge) {
 798 #if USE_AUTH
 799             if (http->request->flags.sslBumped) {
 800                 /*SSL Bumped request, authentication is not possible*/
 801                 status = Http::scForbidden;
 802             } else if (!http->flags.accel) {
 803                 /* Proxy authorisation needed */
 804                 status = Http::scProxyAuthenticationRequired;
 805             } else {
 806                 /* WWW authorisation needed */
 807                 status = Http::scUnauthorized;
 808             }
 809 #else
 810             // need auth, but not possible to do.
 811             status = Http::scForbidden;
 812 #endif
 813             if (page_id == ERR_NONE)
 814                 page_id = ERR_CACHE_ACCESS_DENIED;
 815         } else {
 816             status = Http::scForbidden;
 817
 818             if (page_id == ERR_NONE)
 819                 page_id = ERR_ACCESS_DENIED;
 820         }
 821
 822         Ip::Address tmpnoaddr;
 823         tmpnoaddr.setNoAddr();
 824         error = clientBuildError(page_id, status,
 825                                  NULL,
 826                                  http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
 827                                  http->request
 828                                 );
 829
 830 #if USE_AUTH
 831         error->auth_user_request =
 832             http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 833             http->getConn()->getAuth() : http->request->auth_user_request;
 834 #endif
 835
 836         readNextRequest = true;
 837     }
 838
 839     /* ACCESS_ALLOWED continues here ... */
 840     xfree(http->uri);
 841     http->uri = SBufToCstring(http->request->effectiveRequestUri());
 842     http->doCallouts();
 843 }
 844
 845 #if USE_ADAPTATION
 846 void
 847 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 848 {
 849     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 850
 851 #if ICAP_CLIENT
 852     Adaptation::Icap::History::Pointer ih = request->icapHistory();
 853     if (ih != NULL) {
 854         if (getConn() != NULL && getConn()->clientConnection != NULL) {
 855             ih->rfc931 = getConn()->clientConnection->rfc931;
 856 #if USE_OPENSSL
 857             if (getConn()->clientConnection->isOpen()) {
 858                 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl.get());
 859             }
 860 #endif
 861         }
 862         ih->log_uri = log_uri;
 863         ih->req_sz = req_sz;
 864     }
 865 #endif
 866
 867     if (!g) {
 868         debugs(85,3, HERE << "no adaptation needed");
 869         doCallouts();
 870         return;
 871     }
 872
 873     startAdaptation(g);
 874 }
 875
 876 #endif
 877
 878 static void
 879 clientRedirectAccessCheckDone(allow_t answer, void *data)
 880 {
 881     ClientRequestContext *context = (ClientRequestContext *)data;
 882     ClientHttpRequest *http = context->http;
 883     context->acl_checklist = NULL;
 884
 885     if (answer == ACCESS_ALLOWED)
 886         redirectStart(http, clientRedirectDoneWrapper, context);
 887     else {
 888         Helper::Reply nilReply;
 889         nilReply.result = Helper::Error;
 890         context->clientRedirectDone(nilReply);
 891     }
 892 }
 893
 894 void
 895 ClientRequestContext::clientRedirectStart()
 896 {
 897     debugs(33, 5, HERE << "'" << http->uri << "'");
 898     (void)SyncNotes(*http->al, *http->request);
 899     if (Config.accessList.redirector) {
 900         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 901         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 902     } else
 903         redirectStart(http, clientRedirectDoneWrapper, this);
 904 }
 905
 906 /**
 907  * This methods handles Access checks result of StoreId access list.
 908  * Will handle as "ERR" (no change) in a case Access is not allowed.
 909  */
 910 static void
 911 clientStoreIdAccessCheckDone(allow_t answer, void *data)
 912 {
 913     ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
 914     ClientHttpRequest *http = context->http;
 915     context->acl_checklist = NULL;
 916
 917     if (answer == ACCESS_ALLOWED)
 918         storeIdStart(http, clientStoreIdDoneWrapper, context);
 919     else {
 920         debugs(85, 3, "access denied expected ERR reply handling: " << answer);
 921         Helper::Reply nilReply;
 922         nilReply.result = Helper::Error;
 923         context->clientStoreIdDone(nilReply);
 924     }
 925 }
 926
 927 /**
 928  * Start locating an alternative storeage ID string (if any) from admin
 929  * configured helper program. This is an asynchronous operation terminating in
 930  * ClientRequestContext::clientStoreIdDone() when completed.
 931  */
 932 void
 933 ClientRequestContext::clientStoreIdStart()
 934 {
 935     debugs(33, 5,"'" << http->uri << "'");
 936
 937     if (Config.accessList.store_id) {
 938         acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
 939         acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
 940     } else
 941         storeIdStart(http, clientStoreIdDoneWrapper, this);
 942 }
 943
 944 static int
 945 clientHierarchical(ClientHttpRequest * http)
 946 {
 947     HttpRequest *request = http->request;
 948     HttpRequestMethod method = request->method;
 949
 950     // intercepted requests MUST NOT (yet) be sent to peers unless verified
 951     if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
 952         return 0;
 953
 954     /*
 955      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 956      * neighbors support private keys
 957      */
 958
 959     if (request->flags.ims && !neighbors_do_private_keys)
 960         return 0;
 961
 962     /*
 963      * This is incorrect: authenticating requests can be sent via a hierarchy
 964      * (they can even be cached if the correct headers are set on the reply)
 965      */
 966     if (request->flags.auth)
 967         return 0;
 968
 969     if (method == Http::METHOD_TRACE)
 970         return 1;
 971
 972     if (method != Http::METHOD_GET)
 973         return 0;
 974
 975     if (request->flags.loopDetected)
 976         return 0;
 977
 978     if (request->url.getScheme() == AnyP::PROTO_HTTP)
 979         return method.respMaybeCacheable();
 980
 981     if (request->url.getScheme() == AnyP::PROTO_GOPHER)
 982         return gopherCachable(request);
 983
 984     if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
 985         return 0;
 986
 987     return 1;
 988 }
 989
 990 static void
 991 clientCheckPinning(ClientHttpRequest * http)
 992 {
 993     HttpRequest *request = http->request;
 994     HttpHeader *req_hdr = &request->header;
 995     ConnStateData *http_conn = http->getConn();
 996
 997     /* Internal requests such as those from ESI includes may be without
 998      * a client connection
 999      */
1000     if (!http_conn)
1001         return;
1002
1003     request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
1004     if (!request->flags.connectionAuthDisabled) {
1005         if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
1006             if (http_conn->pinning.auth) {
1007                 request->flags.connectionAuth = true;
1008                 request->flags.auth = true;
1009             } else {
1010                 request->flags.connectionProxyAuth = true;
1011             }
1012             // These should already be linked correctly.
1013             assert(request->clientConnectionManager == http_conn);
1014         }
1015     }
1016
1017     /* check if connection auth is used, and flag as candidate for pinning
1018      * in such case.
1019      * Note: we may need to set flags.connectionAuth even if the connection
1020      * is already pinned if it was pinned earlier due to proxy auth
1021      */
1022     if (!request->flags.connectionAuth) {
1023         if (req_hdr->has(Http::HdrType::AUTHORIZATION) || req_hdr->has(Http::HdrType::PROXY_AUTHORIZATION)) {
1024             HttpHeaderPos pos = HttpHeaderInitPos;
1025             HttpHeaderEntry *e;
1026             int may_pin = 0;
1027             while ((e = req_hdr->getEntry(&pos))) {
1028                 if (e->id == Http::HdrType::AUTHORIZATION || e->id == Http::HdrType::PROXY_AUTHORIZATION) {
1029                     const char *value = e->value.rawBuf();
1030                     if (strncasecmp(value, "NTLM ", 5) == 0
1031                             ||
1032                             strncasecmp(value, "Negotiate ", 10) == 0
1033                             ||
1034                             strncasecmp(value, "Kerberos ", 9) == 0) {
1035                         if (e->id == Http::HdrType::AUTHORIZATION) {
1036                             request->flags.connectionAuth = true;
1037                             may_pin = 1;
1038                         } else {
1039                             request->flags.connectionProxyAuth = true;
1040                             may_pin = 1;
1041                         }
1042                     }
1043                 }
1044             }
1045             if (may_pin && !request->pinnedConnection()) {
1046                 // These should already be linked correctly. Just need the ServerConnection to pinn.
1047                 assert(request->clientConnectionManager == http_conn);
1048             }
1049         }
1050     }
1051 }
1052
1053 static void
1054 clientInterpretRequestHeaders(ClientHttpRequest * http)
1055 {
1056     HttpRequest *request = http->request;
1057     HttpHeader *req_hdr = &request->header;
1058     bool no_cache = false;
1059
1060     request->imslen = -1;
1061     request->ims = req_hdr->getTime(Http::HdrType::IF_MODIFIED_SINCE);
1062
1063     if (request->ims > 0)
1064         request->flags.ims = true;
1065
1066     if (!request->flags.ignoreCc) {
1067         if (request->cache_control) {
1068             if (request->cache_control->hasNoCache())
1069                 no_cache=true;
1070
1071             // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1072         } else if (req_hdr->has(Http::HdrType::PRAGMA))
1073             no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1074     }
1075
1076     if (request->method == Http::METHOD_OTHER) {
1077         no_cache=true;
1078     }
1079
1080     if (no_cache) {
1081 #if USE_HTTP_VIOLATIONS
1082
1083         if (Config.onoff.reload_into_ims)
1084             request->flags.nocacheHack = true;
1085         else if (refresh_nocache_hack)
1086             request->flags.nocacheHack = true;
1087         else
1088 #endif
1089
1090             request->flags.noCache = true;
1091     }
1092
1093     /* ignore range header in non-GETs or non-HEADs */
1094     if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1095         // XXX: initialize if we got here without HttpRequest::parseHeader()
1096         if (!request->range)
1097             request->range = req_hdr->getRange();
1098
1099         if (request->range) {
1100             request->flags.isRanged = true;
1101             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1102             /* XXX: This is suboptimal. We should give the stream the range set,
1103              * and thereby let the top of the stream set the offset when the
1104              * size becomes known. As it is, we will end up requesting from 0
1105              * for evey -X range specification.
1106              * RBC - this may be somewhat wrong. We should probably set the range
1107              * iter up at this point.
1108              */
1109             node->readBuffer.offset = request->range->lowestOffset(0);
1110             http->range_iter.pos = request->range->begin();
1111             http->range_iter.end = request->range->end();
1112             http->range_iter.valid = true;
1113         }
1114     }
1115
1116     /* Only HEAD and GET requests permit a Range or Request-Range header.
1117      * If these headers appear on any other type of request, delete them now.
1118      */
1119     else {
1120         req_hdr->delById(Http::HdrType::RANGE);
1121         req_hdr->delById(Http::HdrType::REQUEST_RANGE);
1122         request->ignoreRange("neither HEAD nor GET");
1123     }
1124
1125     if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1126         request->flags.auth = true;
1127
1128     clientCheckPinning(http);
1129
1130     if (!request->url.userInfo().isEmpty())
1131         request->flags.auth = true;
1132
1133     if (req_hdr->has(Http::HdrType::VIA)) {
1134         String s = req_hdr->getList(Http::HdrType::VIA);
1135         /*
1136          * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1137          * Note ThisCache2 has a space prepended to the hostname so we don't
1138          * accidentally match super-domains.
1139          */
1140
1141         if (strListIsSubstr(&s, ThisCache2, ',')) {
1142             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1143                      request, (ObjPackMethod) & httpRequestPack);
1144             request->flags.loopDetected = true;
1145         }
1146
1147 #if USE_FORW_VIA_DB
1148         fvdbCountVia(s.termedBuf());
1149
1150 #endif
1151
1152         s.clean();
1153     }
1154
1155 #if USE_FORW_VIA_DB
1156
1157     if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1158         String s = req_hdr->getList(Http::HdrType::X_FORWARDED_FOR);
1159         fvdbCountForw(s.termedBuf());
1160         s.clean();
1161     }
1162
1163 #endif
1164
1165     request->flags.cachable = http->request->maybeCacheable();
1166
1167     if (clientHierarchical(http))
1168         request->flags.hierarchical = true;
1169
1170     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1171            (request->flags.noCache ? "SET" : "NOT SET"));
1172     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1173            (request->flags.cachable ? "SET" : "NOT SET"));
1174     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1175            (request->flags.hierarchical ? "SET" : "NOT SET"));
1176
1177 }
1178
1179 void
1180 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1181 {
1182     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1183
1184     if (!calloutContext->httpStateIsValid())
1185         return;
1186
1187     calloutContext->clientRedirectDone(result);
1188 }
1189
1190 void
1191 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1192 {
1193     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1194
1195     if (!calloutContext->httpStateIsValid())
1196         return;
1197
1198     calloutContext->clientStoreIdDone(result);
1199 }
1200
1201 void
1202 ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1203 {
1204     HttpRequest *old_request = http->request;
1205     debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1206     assert(redirect_state == REDIRECT_PENDING);
1207     redirect_state = REDIRECT_DONE;
1208
1209     // Put helper response Notes into the transaction state record (ALE) eventually
1210     // do it early to ensure that no matter what the outcome the notes are present.
1211     if (http->al != NULL)
1212         (void)SyncNotes(*http->al, *old_request);
1213
1214     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1215
1216     switch (reply.result) {
1217     case Helper::TimedOut:
1218         if (Config.onUrlRewriteTimeout.action != toutActBypass) {
1219             http->calloutsError(ERR_GATEWAY_FAILURE, ERR_DETAIL_REDIRECTOR_TIMEDOUT);
1220             debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1221         }
1222         break;
1223
1224     case Helper::Unknown:
1225     case Helper::TT:
1226         // Handler in redirect.cc should have already mapped Unknown
1227         // IF it contained valid entry for the old URL-rewrite helper protocol
1228         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1229         break;
1230
1231     case Helper::BrokenHelper:
1232         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1233         break;
1234
1235     case Helper::Error:
1236         // no change to be done.
1237         break;
1238
1239     case Helper::Okay: {
1240         // #1: redirect with a specific status code    OK status=NNN url="..."
1241         // #2: redirect with a default status code     OK url="..."
1242         // #3: re-write the URL                        OK rewrite-url="..."
1243
1244         const char *statusNote = reply.notes.findFirst("status");
1245         const char *urlNote = reply.notes.findFirst("url");
1246
1247         if (urlNote != NULL) {
1248             // HTTP protocol redirect to be done.
1249
1250             // TODO: change default redirect status for appropriate requests
1251             // Squid defaults to 302 status for now for better compatibility with old clients.
1252             // HTTP/1.0 client should get 302 (Http::scFound)
1253             // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1254             // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1255             Http::StatusCode status = Http::scFound;
1256             if (statusNote != NULL) {
1257                 const char * result = statusNote;
1258                 status = static_cast<Http::StatusCode>(atoi(result));
1259             }
1260
1261             if (status == Http::scMovedPermanently
1262                     || status == Http::scFound
1263                     || status == Http::scSeeOther
1264                     || status == Http::scPermanentRedirect
1265                     || status == Http::scTemporaryRedirect) {
1266                 http->redirect.status = status;
1267                 http->redirect.location = xstrdup(urlNote);
1268                 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1269             } else {
1270                 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1271             }
1272         } else {
1273             // URL-rewrite wanted. Ew.
1274             urlNote = reply.notes.findFirst("rewrite-url");
1275
1276             // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1277             if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1278                 // XXX: validate the URL properly *without* generating a whole new request object right here.
1279                 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1280                 HttpRequest *new_request = old_request->clone();
1281                 if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1282                     debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1283
1284                     // update the new request to flag the re-writing was done on it
1285                     new_request->flags.redirected = true;
1286
1287                     // unlink bodypipe from the old request. Not needed there any longer.
1288                     if (old_request->body_pipe != NULL) {
1289                         old_request->body_pipe = NULL;
1290                         debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1291                                " from request " << old_request << " to " << new_request);
1292                     }
1293
1294                     // update the current working ClientHttpRequest fields
1295                     xfree(http->uri);
1296                     http->uri = SBufToCstring(new_request->effectiveRequestUri());
1297                     HTTPMSGUNLOCK(old_request);
1298                     http->request = new_request;
1299                     HTTPMSGLOCK(http->request);
1300                 } else {
1301                     debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1302                            old_request->method << " " << urlNote << " " << old_request->http_ver);
1303                     delete new_request;
1304                 }
1305             }
1306         }
1307     }
1308     break;
1309     }
1310
1311     /* FIXME PIPELINE: This is innacurate during pipelining */
1312
1313     if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1314         fd_note(http->getConn()->clientConnection->fd, http->uri);
1315
1316     assert(http->uri);
1317
1318     http->doCallouts();
1319 }
1320
1321 /**
1322  * This method handles the different replies from StoreID helper.
1323  */
1324 void
1325 ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1326 {
1327     HttpRequest *old_request = http->request;
1328     debugs(85, 5, "'" << http->uri << "' result=" << reply);
1329     assert(store_id_state == REDIRECT_PENDING);
1330     store_id_state = REDIRECT_DONE;
1331
1332     // Put helper response Notes into the transaction state record (ALE) eventually
1333     // do it early to ensure that no matter what the outcome the notes are present.
1334     if (http->al != NULL)
1335         (void)SyncNotes(*http->al, *old_request);
1336
1337     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1338
1339     switch (reply.result) {
1340     case Helper::Unknown:
1341     case Helper::TT:
1342         // Handler in redirect.cc should have already mapped Unknown
1343         // IF it contained valid entry for the old helper protocol
1344         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1345         break;
1346
1347     case Helper::TimedOut:
1348     // Timeouts for storeID are not implemented
1349     case Helper::BrokenHelper:
1350         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1351         break;
1352
1353     case Helper::Error:
1354         // no change to be done.
1355         break;
1356
1357     case Helper::Okay: {
1358         const char *urlNote = reply.notes.findFirst("store-id");
1359
1360         // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1361         if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1362             // Debug section required for some very specific cases.
1363             debugs(85, 9, "Setting storeID with: " << urlNote );
1364             http->request->store_id = urlNote;
1365             http->store_id = urlNote;
1366         }
1367     }
1368     break;
1369     }
1370
1371     http->doCallouts();
1372 }
1373
1374 /** Test cache allow/deny configuration
1375  *  Sets flags.cachable=1 if caching is not denied.
1376  */
1377 void
1378 ClientRequestContext::checkNoCache()
1379 {
1380     if (Config.accessList.noCache) {
1381         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1382         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1383     } else {
1384         /* unless otherwise specified, we try to cache. */
1385         checkNoCacheDone(ACCESS_ALLOWED);
1386     }
1387 }
1388
1389 static void
1390 checkNoCacheDoneWrapper(allow_t answer, void *data)
1391 {
1392     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1393
1394     if (!calloutContext->httpStateIsValid())
1395         return;
1396
1397     calloutContext->checkNoCacheDone(answer);
1398 }
1399
1400 void
1401 ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1402 {
1403     acl_checklist = NULL;
1404     http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1405     http->doCallouts();
1406 }
1407
1408 #if USE_OPENSSL
1409 bool
1410 ClientRequestContext::sslBumpAccessCheck()
1411 {
1412     // If SSL connection tunneling or bumping decision has been made, obey it.
1413     const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1414     if (bumpMode != Ssl::bumpEnd) {
1415         debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1416                "), " << "ignoring ssl_bump for " << http->getConn());
1417         if (!http->getConn()->serverBump())
1418             http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1419         http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1420         return false;
1421     }
1422
1423     // If we have not decided yet, decide whether to bump now.
1424
1425     // Bumping here can only start with a CONNECT request on a bumping port
1426     // (bumping of intercepted SSL conns is decided before we get 1st request).
1427     // We also do not bump redirected CONNECT requests.
1428     if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1429             !Config.accessList.ssl_bump ||
1430             !http->getConn()->port->flags.tunnelSslBumping) {
1431         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1432         debugs(85, 5, HERE << "cannot SslBump this request");
1433         return false;
1434     }
1435
1436     // Do not bump during authentication: clients would not proxy-authenticate
1437     // if we delay a 407 response and respond with 200 OK to CONNECT.
1438     if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1439         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1440         debugs(85, 5, HERE << "no SslBump during proxy authentication");
1441         return false;
1442     }
1443
1444     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1445
1446     ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1447     aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1448     return true;
1449 }
1450
1451 /**
1452  * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1453  * as ACLFilledChecklist callback
1454  */
1455 static void
1456 sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1457 {
1458     ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1459
1460     if (!calloutContext->httpStateIsValid())
1461         return;
1462     calloutContext->sslBumpAccessCheckDone(answer);
1463 }
1464
1465 void
1466 ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1467 {
1468     if (!httpStateIsValid())
1469         return;
1470
1471     const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1472                                    static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1473     http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1474     http->al->ssl.bumpMode = bumpMode; // for logging
1475
1476     http->doCallouts();
1477 }
1478 #endif
1479
1480 /*
1481  * Identify requests that do not go through the store and client side stream
1482  * and forward them to the appropriate location. All other requests, request
1483  * them.
1484  */
1485 void
1486 ClientHttpRequest::processRequest()
1487 {
1488     debugs(85, 4, request->method << ' ' << uri);
1489
1490     if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1491 #if USE_OPENSSL
1492         if (sslBumpNeeded()) {
1493             sslBumpStart();
1494             return;
1495         }
1496 #endif
1497         getConn()->stopReading(); // tunnels read for themselves
1498         tunnelStart(this);
1499         return;
1500     }
1501
1502     httpStart();
1503 }
1504
1505 void
1506 ClientHttpRequest::httpStart()
1507 {
1508     PROF_start(httpStart);
1509     logType = LOG_TAG_NONE;
1510     debugs(85, 4, logType.c_str() << " for '" << uri << "'");
1511
1512     /* no one should have touched this */
1513     assert(out.offset == 0);
1514     /* Use the Stream Luke */
1515     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1516     clientStreamRead(node, this, node->readBuffer);
1517     PROF_stop(httpStart);
1518 }
1519
1520 #if USE_OPENSSL
1521
1522 void
1523 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1524 {
1525     debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1526     sslBumpNeed_ = mode;
1527 }
1528
1529 // called when comm_write has completed
1530 static void
1531 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1532 {
1533     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1534     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1535
1536     assert(r && cbdataReferenceValid(r));
1537     r->sslBumpEstablish(errflag);
1538 }
1539
1540 void
1541 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1542 {
1543     // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1544     if (errflag == Comm::ERR_CLOSING)
1545         return;
1546
1547     if (errflag) {
1548         debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1549         getConn()->clientConnection->close();
1550         return;
1551     }
1552
1553     // We lack HttpReply which logRequest() uses to log the status code.
1554     // TODO: Use HttpReply instead of the "200 Connection established" string.
1555     al->http.code = 200;
1556
1557 #if USE_AUTH
1558     // Preserve authentication info for the ssl-bumped request
1559     if (request->auth_user_request != NULL)
1560         getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1561 #endif
1562
1563     assert(sslBumpNeeded());
1564     getConn()->switchToHttps(request, sslBumpNeed_);
1565 }
1566
1567 void
1568 ClientHttpRequest::sslBumpStart()
1569 {
1570     debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1571            "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1572     getConn()->sslBumpMode = sslBumpNeed_;
1573
1574     AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1575                                   CommIoCbPtrFun(&SslBumpEstablish, this));
1576
1577     if (request->flags.interceptTproxy || request->flags.intercepted) {
1578         CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1579         params.flag = Comm::OK;
1580         params.conn = getConn()->clientConnection;
1581         ScheduleCallHere(bumpCall);
1582         return;
1583     }
1584
1585     // send an HTTP 200 response to kick client SSL negotiation
1586     // TODO: Unify with tunnel.cc and add a Server(?) header
1587     static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1588     Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1589 }
1590
1591 #endif
1592
1593 bool
1594 ClientHttpRequest::gotEnough() const
1595 {
1596     /** TODO: should be querying the stream. */
1597     int64_t contentLength =
1598         memObject()->getReply()->bodySize(request->method);
1599     assert(contentLength >= 0);
1600
1601     if (out.offset < contentLength)
1602         return false;
1603
1604     return true;
1605 }
1606
1607 void
1608 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1609 {
1610     entry_ = newEntry;
1611 }
1612
1613 void
1614 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1615 {
1616     if (loggingEntry_)
1617         loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1618
1619     loggingEntry_ = newEntry;
1620
1621     if (loggingEntry_)
1622         loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1623 }
1624
1625 /*
1626  * doCallouts() - This function controls the order of "callout"
1627  * executions, including non-blocking access control checks, the
1628  * redirector, and ICAP.  Previously, these callouts were chained
1629  * together such that "clientAccessCheckDone()" would call
1630  * "clientRedirectStart()" and so on.
1631  *
1632  * The ClientRequestContext (aka calloutContext) class holds certain
1633  * state data for the callout/callback operations.  Previously
1634  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1635  * for a short time.  ClientRequestContext would then delete itself
1636  * and pass control back to ClientHttpRequest when all callouts
1637  * were finished.
1638  *
1639  * This caused some problems for ICAP because we want to make the
1640  * ICAP callout after checking ACLs, but before checking the no_cache
1641  * list.  We can't stuff the ICAP state into the ClientRequestContext
1642  * class because we still need the ICAP state after ClientRequestContext
1643  * goes away.
1644  *
1645  * Note that ClientRequestContext is created before the first call
1646  * to doCallouts().
1647  *
1648  * If one of the callouts notices that ClientHttpRequest is no
1649  * longer valid, it should call cbdataReferenceDone() so that
1650  * ClientHttpRequest's reference count goes to zero and it will get
1651  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1652  *
1653  * Note that we set the _done flags here before actually starting
1654  * the callout.  This is strictly for convenience.
1655  */
1656
1657 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1658 nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1659
1660 void
1661 ClientHttpRequest::doCallouts()
1662 {
1663     assert(calloutContext);
1664
1665     /*Save the original request for logging purposes*/
1666     if (!calloutContext->http->al->request) {
1667         calloutContext->http->al->request = request;
1668         HTTPMSGLOCK(calloutContext->http->al->request);
1669
1670         NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1671         // Make the previously set client connection ID available as annotation.
1672         if (ConnStateData *csd = calloutContext->http->getConn()) {
1673             if (!csd->connectionTag().isEmpty())
1674                 notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1675         }
1676     }
1677
1678     if (!calloutContext->error) {
1679         // CVE-2009-0801: verify the Host: header is consistent with other known details.
1680         if (!calloutContext->host_header_verify_done) {
1681             debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1682             calloutContext->host_header_verify_done = true;
1683             calloutContext->hostHeaderVerify();
1684             return;
1685         }
1686
1687         if (!calloutContext->http_access_done) {
1688             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1689             calloutContext->http_access_done = true;
1690             calloutContext->clientAccessCheck();
1691             return;
1692         }
1693
1694 #if USE_ADAPTATION
1695         if (!calloutContext->adaptation_acl_check_done) {
1696             calloutContext->adaptation_acl_check_done = true;
1697             if (Adaptation::AccessCheck::Start(
1698                         Adaptation::methodReqmod, Adaptation::pointPreCache,
1699                         request, NULL, calloutContext->http->al, this))
1700                 return; // will call callback
1701         }
1702 #endif
1703
1704         if (!calloutContext->redirect_done) {
1705             calloutContext->redirect_done = true;
1706
1707             if (Config.Program.redirect) {
1708                 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1709                 calloutContext->redirect_state = REDIRECT_PENDING;
1710                 calloutContext->clientRedirectStart();
1711                 return;
1712             }
1713         }
1714
1715         if (!calloutContext->adapted_http_access_done) {
1716             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1717             calloutContext->adapted_http_access_done = true;
1718             calloutContext->clientAccessCheck2();
1719             return;
1720         }
1721
1722         if (!calloutContext->store_id_done) {
1723             calloutContext->store_id_done = true;
1724
1725             if (Config.Program.store_id) {
1726                 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1727                 calloutContext->store_id_state = REDIRECT_PENDING;
1728                 calloutContext->clientStoreIdStart();
1729                 return;
1730             }
1731         }
1732
1733         if (!calloutContext->interpreted_req_hdrs) {
1734             debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1735             calloutContext->interpreted_req_hdrs = 1;
1736             clientInterpretRequestHeaders(this);
1737         }
1738
1739         if (!calloutContext->no_cache_done) {
1740             calloutContext->no_cache_done = true;
1741
1742             if (Config.accessList.noCache && request->flags.cachable) {
1743                 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1744                 calloutContext->checkNoCache();
1745                 return;
1746             }
1747         }
1748     } //  if !calloutContext->error
1749
1750     if (!calloutContext->tosToClientDone) {
1751         calloutContext->tosToClientDone = true;
1752         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1753             ACLFilledChecklist ch(NULL, request, NULL);
1754             ch.src_addr = request->client_addr;
1755             ch.my_addr = request->my_addr;
1756             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1757             if (tos)
1758                 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1759         }
1760     }
1761
1762     if (!calloutContext->nfmarkToClientDone) {
1763         calloutContext->nfmarkToClientDone = true;
1764         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1765             ACLFilledChecklist ch(NULL, request, NULL);
1766             ch.src_addr = request->client_addr;
1767             ch.my_addr = request->my_addr;
1768             nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1769             if (mark)
1770                 Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1771         }
1772     }
1773
1774 #if USE_OPENSSL
1775     // We need to check for SslBump even if the calloutContext->error is set
1776     // because bumping may require delaying the error until after CONNECT.
1777     if (!calloutContext->sslBumpCheckDone) {
1778         calloutContext->sslBumpCheckDone = true;
1779         if (calloutContext->sslBumpAccessCheck())
1780             return;
1781         /* else no ssl bump required*/
1782     }
1783 #endif
1784
1785     if (calloutContext->error) {
1786         // XXX: prformance regression. c_str() reallocates
1787         SBuf storeUri(request->storeId());
1788         StoreEntry *e = storeCreateEntry(storeUri.c_str(), storeUri.c_str(), request->flags, request->method);
1789 #if USE_OPENSSL
1790         if (sslBumpNeeded()) {
1791             // We have to serve an error, so bump the client first.
1792             sslBumpNeed(Ssl::bumpClientFirst);
1793             // set final error but delay sending until we bump
1794             Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1795             errorAppendEntry(e, calloutContext->error);
1796             calloutContext->error = NULL;
1797             getConn()->setServerBump(srvBump);
1798             e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1799         } else
1800 #endif
1801         {
1802             // send the error to the client now
1803             clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1804             clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1805             assert (repContext);
1806             repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1807             errorAppendEntry(e, calloutContext->error);
1808             calloutContext->error = NULL;
1809             if (calloutContext->readNextRequest)
1810                 getConn()->flags.readMore = true; // resume any pipeline reads.
1811             node = (clientStreamNode *)client_stream.tail->data;
1812             clientStreamRead(node, this, node->readBuffer);
1813             e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1814             return;
1815         }
1816     }
1817
1818     cbdataReferenceDone(calloutContext->http);
1819     delete calloutContext;
1820     calloutContext = NULL;
1821 #if HEADERS_LOG
1822
1823     headersLog(0, 1, request->method, request);
1824 #endif
1825
1826     debugs(83, 3, HERE << "calling processRequest()");
1827     processRequest();
1828
1829 #if ICAP_CLIENT
1830     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1831     if (ih != NULL)
1832         ih->logType = logType;
1833 #endif
1834 }
1835
1836 #if !_USE_INLINE_
1837 #include "client_side_request.cci"
1838 #endif
1839
1840 #if USE_ADAPTATION
1841 /// Initiate an asynchronous adaptation transaction which will call us back.
1842 void
1843 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1844 {
1845     debugs(85, 3, HERE << "adaptation needed for " << this);
1846     assert(!virginHeadSource);
1847     assert(!adaptedBodySource);
1848     virginHeadSource = initiateAdaptation(
1849                            new Adaptation::Iterator(request, NULL, al, g));
1850
1851     // we could try to guess whether we can bypass this adaptation
1852     // initiation failure, but it should not really happen
1853     Must(initiated(virginHeadSource));
1854 }
1855
1856 void
1857 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1858 {
1859     assert(cbdataReferenceValid(this));     // indicates bug
1860     clearAdaptation(virginHeadSource);
1861     assert(!adaptedBodySource);
1862
1863     switch (answer.kind) {
1864     case Adaptation::Answer::akForward:
1865         handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1866         break;
1867
1868     case Adaptation::Answer::akBlock:
1869         handleAdaptationBlock(answer);
1870         break;
1871
1872     case Adaptation::Answer::akError:
1873         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1874         break;
1875     }
1876 }
1877
1878 void
1879 ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1880 {
1881     assert(msg);
1882
1883     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1884         /*
1885          * Replace the old request with the new request.
1886          */
1887         HTTPMSGUNLOCK(request);
1888         request = new_req;
1889         HTTPMSGLOCK(request);
1890
1891         // update the new message to flag whether URL re-writing was done on it
1892         if (request->effectiveRequestUri().cmp(uri) != 0)
1893             request->flags.redirected = 1;
1894
1895         /*
1896          * Store the new URI for logging
1897          */
1898         xfree(uri);
1899         uri = SBufToCstring(request->effectiveRequestUri());
1900         setLogUri(this, urlCanonicalClean(request));
1901         assert(request->method.id());
1902     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1903         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1904
1905         // subscribe to receive reply body
1906         if (new_rep->body_pipe != NULL) {
1907             adaptedBodySource = new_rep->body_pipe;
1908             int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1909             assert(consumer_ok);
1910         }
1911
1912         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1913         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1914         assert(repContext);
1915         repContext->createStoreEntry(request->method, request->flags);
1916
1917         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1918         request_satisfaction_mode = true;
1919         request_satisfaction_offset = 0;
1920         storeEntry()->replaceHttpReply(new_rep);
1921         storeEntry()->timestampsSet();
1922
1923         if (!adaptedBodySource) // no body
1924             storeEntry()->complete();
1925         clientGetMoreData(node, this);
1926     }
1927
1928     // we are done with getting headers (but may be receiving body)
1929     clearAdaptation(virginHeadSource);
1930
1931     if (!request_satisfaction_mode)
1932         doCallouts();
1933 }
1934
1935 void
1936 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1937 {
1938     request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1939     AclMatchedName = answer.ruleId.termedBuf();
1940     assert(calloutContext);
1941     calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1942     AclMatchedName = NULL;
1943 }
1944
1945 void
1946 ClientHttpRequest::resumeBodyStorage()
1947 {
1948     if (!adaptedBodySource)
1949         return;
1950
1951     noteMoreBodyDataAvailable(adaptedBodySource);
1952 }
1953
1954 void
1955 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1956 {
1957     assert(request_satisfaction_mode);
1958     assert(adaptedBodySource != NULL);
1959
1960     if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1961         const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1962
1963         if (spaceAvailable < contentSize ) {
1964             // No or partial body data consuming
1965             typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1966             AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1967                                                 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1968             storeEntry()->deferProducer(call);
1969         }
1970
1971         if (!spaceAvailable)
1972             return;
1973
1974         if (spaceAvailable < contentSize )
1975             contentSize = spaceAvailable;
1976
1977         BodyPipeCheckout bpc(*adaptedBodySource);
1978         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
1979         storeEntry()->write(ioBuf);
1980         // assume StoreEntry::write() writes the entire ioBuf
1981         request_satisfaction_offset += ioBuf.length;
1982         bpc.buf.consume(contentSize);
1983         bpc.checkIn();
1984     }
1985
1986     if (adaptedBodySource->exhausted())
1987         endRequestSatisfaction();
1988     // else wait for more body data
1989 }
1990
1991 void
1992 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
1993 {
1994     assert(!virginHeadSource);
1995     // should we end request satisfaction now?
1996     if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
1997         endRequestSatisfaction();
1998 }
1999
2000 void
2001 ClientHttpRequest::endRequestSatisfaction()
2002 {
2003     debugs(85,4, HERE << this << " ends request satisfaction");
2004     assert(request_satisfaction_mode);
2005     stopConsumingFrom(adaptedBodySource);
2006
2007     // TODO: anything else needed to end store entry formation correctly?
2008     storeEntry()->complete();
2009 }
2010
2011 void
2012 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2013 {
2014     assert(!virginHeadSource);
2015     stopConsumingFrom(adaptedBodySource);
2016
2017     debugs(85,3, HERE << "REQMOD body production failed");
2018     if (request_satisfaction_mode) { // too late to recover or serve an error
2019         request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2020         const Comm::ConnectionPointer c = getConn()->clientConnection;
2021         Must(Comm::IsConnOpen(c));
2022         c->close(); // drastic, but we may be writing a response already
2023     } else {
2024         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2025     }
2026 }
2027
2028 void
2029 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2030 {
2031     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2032
2033     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2034     const bool usedPipe = request->body_pipe != NULL &&
2035                           request->body_pipe->consumedSize() > 0;
2036
2037     if (bypassable && !usedStore && !usedPipe) {
2038         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2039         if (calloutContext)
2040             doCallouts();
2041         return;
2042     }
2043
2044     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2045
2046     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2047     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2048     assert(repContext);
2049
2050     calloutsError(ERR_ICAP_FAILURE, errDetail);
2051
2052     if (calloutContext)
2053         doCallouts();
2054 }
2055
2056 #endif
2057
2058 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2059 void
2060 ClientHttpRequest::calloutsError(const err_type error, const int errDetail)
2061 {
2062     // The original author of the code also wanted to pass an errno to
2063     // setReplyToError, but it seems unlikely that the errno reflects the
2064     // true cause of the error at this point, so I did not pass it.
2065     if (calloutContext) {
2066         Ip::Address noAddr;
2067         noAddr.setNoAddr();
2068         ConnStateData * c = getConn();
2069         calloutContext->error = clientBuildError(error, Http::scInternalServerError,
2070                                 NULL,
2071                                 c != NULL ? c->clientConnection->remote : noAddr,
2072                                 request
2073                                                 );
2074 #if USE_AUTH
2075         calloutContext->error->auth_user_request =
2076             c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2077 #endif
2078         calloutContext->error->detailError(errDetail);
2079         calloutContext->readNextRequest = true;
2080         if (c != NULL)
2081             c->expectNoForwarding();
2082     }
2083     //else if(calloutContext == NULL) is it possible?
2084 }
2085