src/client_side_request.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 85    Client-side Request Routines */
  10
  11 /*
  12  * General logic of request processing:
  13  *
  14  * We run a series of tests to determine if access will be permitted, and to do
  15  * any redirection. Then we call into the result clientStream to retrieve data.
  16  * From that point on it's up to reply management.
  17  */
  18
  19 #include "squid.h"
  20 #include "acl/FilledChecklist.h"
  21 #include "acl/Gadgets.h"
  22 #include "anyp/PortCfg.h"
  23 #include "base/AsyncJobCalls.h"
  24 #include "client_side.h"
  25 #include "client_side_reply.h"
  26 #include "client_side_request.h"
  27 #include "ClientRequestContext.h"
  28 #include "clientStream.h"
  29 #include "comm/Connection.h"
  30 #include "comm/Write.h"
  31 #include "err_detail_type.h"
  32 #include "errorpage.h"
  33 #include "fd.h"
  34 #include "fde.h"
  35 #include "format/Token.h"
  36 #include "gopher.h"
  37 #include "helper.h"
  38 #include "helper/Reply.h"
  39 #include "http.h"
  40 #include "http/Stream.h"
  41 #include "HttpHdrCc.h"
  42 #include "HttpReply.h"
  43 #include "HttpRequest.h"
  44 #include "ip/NfMarkConfig.h"
  45 #include "ip/QosConfig.h"
  46 #include "ipcache.h"
  47 #include "log/access_log.h"
  48 #include "MemObject.h"
  49 #include "Parsing.h"
  50 #include "profiler/Profiler.h"
  51 #include "proxyp/Header.h"
  52 #include "redirect.h"
  53 #include "rfc1738.h"
  54 #include "SquidConfig.h"
  55 #include "SquidTime.h"
  56 #include "Store.h"
  57 #include "StrList.h"
  58 #include "tools.h"
  59 #include "wordlist.h"
  60 #if USE_AUTH
  61 #include "auth/UserRequest.h"
  62 #endif
  63 #if USE_ADAPTATION
  64 #include "adaptation/AccessCheck.h"
  65 #include "adaptation/Answer.h"
  66 #include "adaptation/Iterator.h"
  67 #include "adaptation/Service.h"
  68 #if ICAP_CLIENT
  69 #include "adaptation/icap/History.h"
  70 #endif
  71 #endif
  72 #if USE_OPENSSL
  73 #include "ssl/ServerBump.h"
  74 #include "ssl/support.h"
  75 #endif
  76
  77 #if LINGERING_CLOSE
  78 #define comm_close comm_lingering_close
  79 #endif
  80
  81 static const char *const crlf = "\r\n";
  82
  83 #if FOLLOW_X_FORWARDED_FOR
  84 static void clientFollowXForwardedForCheck(Acl::Answer answer, void *data);
  85 #endif /* FOLLOW_X_FORWARDED_FOR */
  86
  87 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *, const AccessLogEntry::Pointer &);
  88
  89 CBDATA_CLASS_INIT(ClientRequestContext);
  90
  91 /* Local functions */
  92 /* other */
  93 static void clientAccessCheckDoneWrapper(Acl::Answer, void *);
  94 #if USE_OPENSSL
  95 static void sslBumpAccessCheckDoneWrapper(Acl::Answer, void *);
  96 #endif
  97 static int clientHierarchical(ClientHttpRequest * http);
  98 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  99 static HLPCB clientRedirectDoneWrapper;
 100 static HLPCB clientStoreIdDoneWrapper;
 101 static void checkNoCacheDoneWrapper(Acl::Answer, void *);
 102 SQUIDCEXTERN CSR clientGetMoreData;
 103 SQUIDCEXTERN CSS clientReplyStatus;
 104 SQUIDCEXTERN CSD clientReplyDetach;
 105 static void checkFailureRatio(err_type, hier_code);
 106
 107 ClientRequestContext::~ClientRequestContext()
 108 {
 109     /*
 110      * Release our "lock" on our parent, ClientHttpRequest, if we
 111      * still have one
 112      */
 113
 114     if (http)
 115         cbdataReferenceDone(http);
 116
 117     delete error;
 118     debugs(85,3, "ClientRequestContext destructed, this=" << this);
 119 }
 120
 121 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) :
 122     http(cbdataReference(anHttp)),
 123     acl_checklist(NULL),
 124     redirect_state(REDIRECT_NONE),
 125     store_id_state(REDIRECT_NONE),
 126     host_header_verify_done(false),
 127     http_access_done(false),
 128     adapted_http_access_done(false),
 129 #if USE_ADAPTATION
 130     adaptation_acl_check_done(false),
 131 #endif
 132     redirect_done(false),
 133     store_id_done(false),
 134     no_cache_done(false),
 135     interpreted_req_hdrs(false),
 136     toClientMarkingDone(false),
 137 #if USE_OPENSSL
 138     sslBumpCheckDone(false),
 139 #endif
 140     error(NULL),
 141     readNextRequest(false)
 142 {
 143     debugs(85, 3, "ClientRequestContext constructed, this=" << this);
 144 }
 145
 146 CBDATA_CLASS_INIT(ClientHttpRequest);
 147
 148 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 149 #if USE_ADAPTATION
 150     AsyncJob("ClientHttpRequest"),
 151 #endif
 152     request(NULL),
 153     uri(NULL),
 154     log_uri(NULL),
 155     req_sz(0),
 156     calloutContext(NULL),
 157     maxReplyBodySize_(0),
 158     entry_(NULL),
 159     loggingEntry_(NULL),
 160     conn_(NULL)
 161 #if USE_OPENSSL
 162     , sslBumpNeed_(Ssl::bumpEnd)
 163 #endif
 164 #if USE_ADAPTATION
 165     , request_satisfaction_mode(false)
 166     , request_satisfaction_offset(0)
 167 #endif
 168 {
 169     setConn(aConn);
 170     al = new AccessLogEntry;
 171     CodeContext::Reset(al);
 172     al->cache.start_time = current_time;
 173     if (aConn) {
 174         al->tcpClient = clientConnection = aConn->clientConnection;
 175         al->cache.port = aConn->port;
 176         al->cache.caddr = aConn->log_addr;
 177         al->proxyProtocolHeader = aConn->proxyProtocolHeader();
 178
 179 #if USE_OPENSSL
 180         if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
 181             if (auto ssl = fd_table[aConn->clientConnection->fd].ssl.get())
 182                 al->cache.sslClientCert.resetWithoutLocking(SSL_get_peer_certificate(ssl));
 183         }
 184 #endif
 185     }
 186     dlinkAdd(this, &active, &ClientActiveRequests);
 187 }
 188
 189 /*
 190  * returns true if client specified that the object must come from the cache
 191  * without contacting origin server
 192  */
 193 bool
 194 ClientHttpRequest::onlyIfCached()const
 195 {
 196     assert(request);
 197     return request->cache_control &&
 198            request->cache_control->hasOnlyIfCached();
 199 }
 200
 201 /**
 202  * This function is designed to serve a fairly specific purpose.
 203  * Occasionally our vBNS-connected caches can talk to each other, but not
 204  * the rest of the world.  Here we try to detect frequent failures which
 205  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 206  * the failure:success ratio goes above 1.0 then we go into "hit only"
 207  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 208  * will only fetch HITs from us if they are using the ICP protocol.  We
 209  * stay in this mode for 5 minutes.
 210  *
 211  * Duane W., Sept 16, 1996
 212  */
 213 static void
 214 checkFailureRatio(err_type etype, hier_code hcode)
 215 {
 216     // Can be set at compile time with -D compiler flag
 217 #ifndef FAILURE_MODE_TIME
 218 #define FAILURE_MODE_TIME 300
 219 #endif
 220
 221     if (hcode == HIER_NONE)
 222         return;
 223
 224     // don't bother when ICP is disabled.
 225     if (Config.Port.icp <= 0)
 226         return;
 227
 228     static double magic_factor = 100.0;
 229     double n_good;
 230     double n_bad;
 231
 232     n_good = magic_factor / (1.0 + request_failure_ratio);
 233
 234     n_bad = magic_factor - n_good;
 235
 236     switch (etype) {
 237
 238     case ERR_DNS_FAIL:
 239
 240     case ERR_CONNECT_FAIL:
 241     case ERR_SECURE_CONNECT_FAIL:
 242
 243     case ERR_READ_ERROR:
 244         ++n_bad;
 245         break;
 246
 247     default:
 248         ++n_good;
 249     }
 250
 251     request_failure_ratio = n_bad / n_good;
 252
 253     if (hit_only_mode_until > squid_curtime)
 254         return;
 255
 256     if (request_failure_ratio < 1.0)
 257         return;
 258
 259     debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
 260            std::setprecision(3) << request_failure_ratio);
 261
 262     debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
 263            FAILURE_MODE_TIME / 60 << " minutes...");
 264
 265     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 266
 267     request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
 268 }
 269
 270 ClientHttpRequest::~ClientHttpRequest()
 271 {
 272     debugs(33, 3, "httpRequestFree: " << uri);
 273     PROF_start(httpRequestFree);
 274
 275     // Even though freeResources() below may destroy the request,
 276     // we no longer set request->body_pipe to NULL here
 277     // because we did not initiate that pipe (ConnStateData did)
 278
 279     /* the ICP check here was erroneous
 280      * - StoreEntry::releaseRequest was always called if entry was valid
 281      */
 282
 283     logRequest();
 284
 285     loggingEntry(NULL);
 286
 287     if (request)
 288         checkFailureRatio(request->errType, al->hier.code);
 289
 290     freeResources();
 291
 292 #if USE_ADAPTATION
 293     announceInitiatorAbort(virginHeadSource);
 294
 295     if (adaptedBodySource != NULL)
 296         stopConsumingFrom(adaptedBodySource);
 297 #endif
 298
 299     if (calloutContext)
 300         delete calloutContext;
 301
 302     clientConnection = NULL;
 303
 304     if (conn_)
 305         cbdataReferenceDone(conn_);
 306
 307     /* moving to the next connection is handled by the context free */
 308     dlinkDelete(&active, &ClientActiveRequests);
 309
 310     PROF_stop(httpRequestFree);
 311 }
 312
 313 /**
 314  * Create a request and kick it off
 315  *
 316  * \retval 0     success
 317  * \retval -1    failure
 318  *
 319  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 320  * determined by the user
 321  */
 322 int
 323 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 324                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 325                    char *tailbuf, size_t taillen, const MasterXaction::Pointer &mx)
 326 {
 327     size_t url_sz;
 328     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 329     HttpRequest *request;
 330     StoreIOBuffer tempBuffer;
 331     if (http->al != NULL)
 332         http->al->cache.start_time = current_time;
 333     /* this is only used to adjust the connection offset in client_side.c */
 334     http->req_sz = 0;
 335     tempBuffer.length = taillen;
 336     tempBuffer.data = tailbuf;
 337     /* client stream setup */
 338     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 339                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 340                      streamdetach, streamdata, tempBuffer);
 341     /* make it visible in the 'current acctive requests list' */
 342     /* Set flags */
 343     /* internal requests only makes sense in an
 344      * accelerator today. TODO: accept flags ? */
 345     http->flags.accel = true;
 346     /* allow size for url rewriting */
 347     url_sz = strlen(url) + Config.appendDomainLen + 5;
 348     http->uri = (char *)xcalloc(url_sz, 1);
 349     strcpy(http->uri, url); // XXX: polluting http->uri before parser validation
 350
 351     request = HttpRequest::FromUrlXXX(http->uri, mx, method);
 352     if (!request) {
 353         debugs(85, 5, "Invalid URL: " << http->uri);
 354         return -1;
 355     }
 356
 357     /*
 358      * now update the headers in request with our supplied headers.
 359      * HttpRequest::FromUrl() should return a blank header set, but
 360      * we use Update to be sure of correctness.
 361      */
 362     if (header)
 363         request->header.update(header);
 364
 365     /* http struct now ready */
 366
 367     /*
 368      * build new header list *? TODO
 369      */
 370     request->flags.accelerated = http->flags.accel;
 371
 372     /* this is an internally created
 373      * request, not subject to acceleration
 374      * target overrides */
 375     /*
 376      * FIXME? Do we want to detect and handle internal requests of internal
 377      * objects ?
 378      */
 379
 380     /* Internally created requests cannot have bodies today */
 381     request->content_length = 0;
 382
 383     request->client_addr.setNoAddr();
 384
 385 #if FOLLOW_X_FORWARDED_FOR
 386     request->indirect_client_addr.setNoAddr();
 387 #endif /* FOLLOW_X_FORWARDED_FOR */
 388
 389     request->my_addr.setNoAddr();   /* undefined for internal requests */
 390
 391     request->my_addr.port(0);
 392
 393     request->http_ver = Http::ProtocolVersion();
 394
 395     http->initRequest(request);
 396
 397     /* optional - skip the access check ? */
 398     http->calloutContext = new ClientRequestContext(http);
 399
 400     http->calloutContext->http_access_done = false;
 401
 402     http->calloutContext->redirect_done = true;
 403
 404     http->calloutContext->no_cache_done = true;
 405
 406     http->doCallouts();
 407
 408     return 0;
 409 }
 410
 411 bool
 412 ClientRequestContext::httpStateIsValid()
 413 {
 414     ClientHttpRequest *http_ = http;
 415
 416     if (cbdataReferenceValid(http_))
 417         return true;
 418
 419     http = NULL;
 420
 421     cbdataReferenceDone(http_);
 422
 423     return false;
 424 }
 425
 426 #if FOLLOW_X_FORWARDED_FOR
 427 /**
 428  * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
 429  * against the followXFF ACL, or cleans up and passes control to
 430  * clientAccessCheck().
 431  *
 432  * The trust model here is a little ambiguous. So to clarify the logic:
 433  * - we may always use the direct client address as the client IP.
 434  * - these trust tests merey tell whether we trust given IP enough to believe the
 435  *   IP string which it appended to the X-Forwarded-For: header.
 436  * - if at any point we don't trust what an IP adds we stop looking.
 437  * - at that point the current contents of indirect_client_addr are the value set
 438  *   by the last previously trusted IP.
 439  * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
 440  */
 441 static void
 442 clientFollowXForwardedForCheck(Acl::Answer answer, void *data)
 443 {
 444     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 445
 446     if (!calloutContext->httpStateIsValid())
 447         return;
 448
 449     ClientHttpRequest *http = calloutContext->http;
 450     HttpRequest *request = http->request;
 451
 452     if (answer.allowed() && request->x_forwarded_for_iterator.size() != 0) {
 453
 454         /*
 455          * Remove the last comma-delimited element from the
 456          * x_forwarded_for_iterator and use it to repeat the cycle.
 457          */
 458         const char *p;
 459         const char *asciiaddr;
 460         int l;
 461         Ip::Address addr;
 462         p = request->x_forwarded_for_iterator.termedBuf();
 463         l = request->x_forwarded_for_iterator.size();
 464
 465         /*
 466         * XXX x_forwarded_for_iterator should really be a list of
 467         * IP addresses, but it's a String instead.  We have to
 468         * walk backwards through the String, biting off the last
 469         * comma-delimited part each time.  As long as the data is in
 470         * a String, we should probably implement and use a variant of
 471         * strListGetItem() that walks backwards instead of forwards
 472         * through a comma-separated list.  But we don't even do that;
 473         * we just do the work in-line here.
 474         */
 475         /* skip trailing space and commas */
 476         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 477             --l;
 478         request->x_forwarded_for_iterator.cut(l);
 479         /* look for start of last item in list */
 480         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 481             --l;
 482         asciiaddr = p+l;
 483         if ((addr = asciiaddr)) {
 484             request->indirect_client_addr = addr;
 485             request->x_forwarded_for_iterator.cut(l);
 486             calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 487             if (!Config.onoff.acl_uses_indirect_client) {
 488                 /* override the default src_addr tested if we have to go deeper than one level into XFF */
 489                 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
 490             }
 491             calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
 492             return;
 493         }
 494     }
 495
 496     /* clean up, and pass control to clientAccessCheck */
 497     if (Config.onoff.log_uses_indirect_client) {
 498         /*
 499         * Ensure that the access log shows the indirect client
 500         * instead of the direct client.
 501         */
 502         http->al->cache.caddr = request->indirect_client_addr;
 503         if (ConnStateData *conn = http->getConn())
 504             conn->log_addr = request->indirect_client_addr;
 505     }
 506     request->x_forwarded_for_iterator.clean();
 507     request->flags.done_follow_x_forwarded_for = true;
 508
 509     if (answer.conflicted()) {
 510         debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
 511     }
 512
 513     /* process actual access ACL as normal. */
 514     calloutContext->clientAccessCheck();
 515 }
 516 #endif /* FOLLOW_X_FORWARDED_FOR */
 517
 518 static void
 519 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const Dns::LookupDetails &dns, void *data)
 520 {
 521     ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
 522     c->hostHeaderIpVerify(ia, dns);
 523 }
 524
 525 void
 526 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const Dns::LookupDetails &dns)
 527 {
 528     Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
 529
 530     // note the DNS details for the transaction stats.
 531     http->request->recordLookup(dns);
 532
 533     // Is the NAT destination IP in DNS?
 534     if (ia && ia->have(clientConn->local)) {
 535         debugs(85, 3, "validate IP " << clientConn->local << " possible from Host:");
 536         http->request->flags.hostVerified = true;
 537         http->doCallouts();
 538         return;
 539     }
 540     debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
 541     hostHeaderVerifyFailed("local IP", "any domain IP");
 542 }
 543
 544 void
 545 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
 546 {
 547     // IP address validation for Host: failed. Admin wants to ignore them.
 548     // NP: we do not yet handle CONNECT tunnels well, so ignore for them
 549     if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
 550         debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
 551                " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
 552
 553         // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
 554         // The problems here are about WRITE for new cache content, which means flags.cachable
 555         http->request->flags.cachable = false; // MUST NOT cache (for now)
 556         // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
 557         http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
 558         // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
 559         http->doCallouts();
 560         return;
 561     }
 562
 563     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
 564            http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
 565     if (const char *ua = http->request->header.getStr(Http::HdrType::USER_AGENT))
 566         debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << ua);
 567     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
 568
 569     // IP address validation for Host: failed. reject the connection.
 570     clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 571     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 572     assert (repContext);
 573     repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
 574                                 http->request->method, NULL,
 575                                 http->getConn()->clientConnection->remote,
 576                                 http->request,
 577                                 NULL,
 578 #if USE_AUTH
 579                                 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 580                                 http->getConn()->getAuth() : http->request->auth_user_request);
 581 #else
 582                                 NULL);
 583 #endif
 584     node = (clientStreamNode *)http->client_stream.tail->data;
 585     clientStreamRead(node, http, node->readBuffer);
 586 }
 587
 588 void
 589 ClientRequestContext::hostHeaderVerify()
 590 {
 591     // Require a Host: header.
 592     const char *host = http->request->header.getStr(Http::HdrType::HOST);
 593
 594     if (!host) {
 595         // TODO: dump out the HTTP/1.1 error about missing host header.
 596         // otherwise this is fine, can't forge a header value when its not even set.
 597         debugs(85, 3, HERE << "validate skipped with no Host: header present.");
 598         http->doCallouts();
 599         return;
 600     }
 601
 602     if (http->request->flags.internal) {
 603         // TODO: kill this when URL handling allows partial URLs out of accel mode
 604         //       and we no longer screw with the URL just to add our internal host there
 605         debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
 606         http->doCallouts();
 607         return;
 608     }
 609
 610     // Locate if there is a port attached, strip ready for IP lookup
 611     char *portStr = NULL;
 612     char *hostB = xstrdup(host);
 613     host = hostB;
 614     if (host[0] == '[') {
 615         // IPv6 literal.
 616         portStr = strchr(hostB, ']');
 617         if (portStr && *(++portStr) != ':') {
 618             portStr = NULL;
 619         }
 620     } else {
 621         // Domain or IPv4 literal with port
 622         portStr = strrchr(hostB, ':');
 623     }
 624
 625     uint16_t port = 0;
 626     if (portStr) {
 627         *portStr = '\0'; // strip the ':'
 628         if (*(++portStr) != '\0') {
 629             char *end = NULL;
 630             int64_t ret = strtoll(portStr, &end, 10);
 631             if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
 632                 // invalid port details. Replace the ':'
 633                 *(--portStr) = ':';
 634                 portStr = NULL;
 635             } else
 636                 port = (ret & 0xFFFF);
 637         }
 638     }
 639
 640     debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
 641     if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
 642         // verify the Host: port (if any) matches the apparent destination
 643         if (portStr && port != http->getConn()->clientConnection->local.port()) {
 644             debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
 645                    " matches Host: port " << port << " (" << portStr << ")");
 646             hostHeaderVerifyFailed("intercepted port", portStr);
 647         } else {
 648             // XXX: match the scheme default port against the apparent destination
 649
 650             // verify the destination DNS is one of the Host: headers IPs
 651             ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
 652         }
 653     } else if (!Config.onoff.hostStrictVerify) {
 654         debugs(85, 3, "validate skipped.");
 655         http->doCallouts();
 656     } else if (strlen(host) != strlen(http->request->url.host())) {
 657         // Verify forward-proxy requested URL domain matches the Host: header
 658         debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
 659         hostHeaderVerifyFailed(host, http->request->url.host());
 660     } else if (matchDomainName(host, http->request->url.host()) != 0) {
 661         // Verify forward-proxy requested URL domain matches the Host: header
 662         debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
 663         hostHeaderVerifyFailed(host, http->request->url.host());
 664     } else if (portStr && port != http->request->url.port()) {
 665         // Verify forward-proxy requested URL domain matches the Host: header
 666         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
 667         hostHeaderVerifyFailed("URL port", portStr);
 668     } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
 669         // Verify forward-proxy requested URL domain matches the Host: header
 670         // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
 671         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
 672         hostHeaderVerifyFailed("URL port", "default port");
 673     } else {
 674         // Okay no problem.
 675         debugs(85, 3, "validate passed.");
 676         http->request->flags.hostVerified = true;
 677         http->doCallouts();
 678     }
 679     safe_free(hostB);
 680 }
 681
 682 /* This is the entry point for external users of the client_side routines */
 683 void
 684 ClientRequestContext::clientAccessCheck()
 685 {
 686 #if FOLLOW_X_FORWARDED_FOR
 687     if (!http->request->flags.doneFollowXff() &&
 688             Config.accessList.followXFF &&
 689             http->request->header.has(Http::HdrType::X_FORWARDED_FOR)) {
 690
 691         /* we always trust the direct client address for actual use */
 692         http->request->indirect_client_addr = http->request->client_addr;
 693         http->request->indirect_client_addr.port(0);
 694
 695         /* setup the XFF iterator for processing */
 696         http->request->x_forwarded_for_iterator = http->request->header.getList(Http::HdrType::X_FORWARDED_FOR);
 697
 698         /* begin by checking to see if we trust direct client enough to walk XFF */
 699         acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 700         acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
 701         return;
 702     }
 703 #endif
 704
 705     if (Config.accessList.http) {
 706         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 707         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 708     } else {
 709         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 710         clientAccessCheckDone(ACCESS_DENIED);
 711     }
 712 }
 713
 714 /**
 715  * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
 716  * The default here is to allow all. Since the earlier http_access should do a default deny all.
 717  * This check is just for a last-minute denial based on adapted request headers.
 718  */
 719 void
 720 ClientRequestContext::clientAccessCheck2()
 721 {
 722     if (Config.accessList.adapted_http) {
 723         acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
 724         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 725     } else {
 726         debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
 727         clientAccessCheckDone(ACCESS_ALLOWED);
 728     }
 729 }
 730
 731 void
 732 clientAccessCheckDoneWrapper(Acl::Answer answer, void *data)
 733 {
 734     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 735
 736     if (!calloutContext->httpStateIsValid())
 737         return;
 738
 739     calloutContext->clientAccessCheckDone(answer);
 740 }
 741
 742 void
 743 ClientRequestContext::clientAccessCheckDone(const Acl::Answer &answer)
 744 {
 745     acl_checklist = NULL;
 746     err_type page_id;
 747     Http::StatusCode status;
 748     debugs(85, 2, "The request " << http->request->method << ' ' <<
 749            http->uri << " is " << answer <<
 750            "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
 751
 752 #if USE_AUTH
 753     char const *proxy_auth_msg = "<null>";
 754     if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
 755         proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
 756     else if (http->request->auth_user_request != NULL)
 757         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 758 #endif
 759
 760     if (!answer.allowed()) {
 761         // auth has a grace period where credentials can be expired but okay not to challenge.
 762
 763         /* Send an auth challenge or error */
 764         // XXX: do we still need aclIsProxyAuth() ?
 765         bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
 766         debugs(85, 5, "Access Denied: " << http->uri);
 767         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 768 #if USE_AUTH
 769         if (auth_challenge)
 770             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 771 #endif
 772
 773         /*
 774          * NOTE: get page_id here, based on AclMatchedName because if
 775          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 776          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 777          * <pribeiro@isel.pt>
 778          */
 779         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
 780
 781         http->logType.update(LOG_TCP_DENIED);
 782
 783         if (auth_challenge) {
 784 #if USE_AUTH
 785             if (http->request->flags.sslBumped) {
 786                 /*SSL Bumped request, authentication is not possible*/
 787                 status = Http::scForbidden;
 788             } else if (!http->flags.accel) {
 789                 /* Proxy authorisation needed */
 790                 status = Http::scProxyAuthenticationRequired;
 791             } else {
 792                 /* WWW authorisation needed */
 793                 status = Http::scUnauthorized;
 794             }
 795 #else
 796             // need auth, but not possible to do.
 797             status = Http::scForbidden;
 798 #endif
 799             if (page_id == ERR_NONE)
 800                 page_id = ERR_CACHE_ACCESS_DENIED;
 801         } else {
 802             status = Http::scForbidden;
 803
 804             if (page_id == ERR_NONE)
 805                 page_id = ERR_ACCESS_DENIED;
 806         }
 807
 808         Ip::Address tmpnoaddr;
 809         tmpnoaddr.setNoAddr();
 810         error = clientBuildError(page_id, status,
 811                                  NULL,
 812                                  http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
 813                                  http->request, http->al
 814                                 );
 815
 816 #if USE_AUTH
 817         error->auth_user_request =
 818             http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 819             http->getConn()->getAuth() : http->request->auth_user_request;
 820 #endif
 821
 822         readNextRequest = true;
 823     }
 824
 825     /* ACCESS_ALLOWED continues here ... */
 826     xfree(http->uri);
 827     http->uri = SBufToCstring(http->request->effectiveRequestUri());
 828     http->doCallouts();
 829 }
 830
 831 #if USE_ADAPTATION
 832 void
 833 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 834 {
 835     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 836
 837 #if ICAP_CLIENT
 838     Adaptation::Icap::History::Pointer ih = request->icapHistory();
 839     if (ih != NULL) {
 840         if (getConn() != NULL && getConn()->clientConnection != NULL) {
 841             ih->rfc931 = getConn()->clientConnection->rfc931;
 842 #if USE_OPENSSL
 843             if (getConn()->clientConnection->isOpen()) {
 844                 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl.get());
 845             }
 846 #endif
 847         }
 848         ih->log_uri = log_uri;
 849         ih->req_sz = req_sz;
 850     }
 851 #endif
 852
 853     if (!g) {
 854         debugs(85,3, HERE << "no adaptation needed");
 855         doCallouts();
 856         return;
 857     }
 858
 859     startAdaptation(g);
 860 }
 861
 862 #endif
 863
 864 static void
 865 clientRedirectAccessCheckDone(Acl::Answer answer, void *data)
 866 {
 867     ClientRequestContext *context = (ClientRequestContext *)data;
 868     ClientHttpRequest *http = context->http;
 869     context->acl_checklist = NULL;
 870
 871     if (answer.allowed())
 872         redirectStart(http, clientRedirectDoneWrapper, context);
 873     else {
 874         Helper::Reply const nilReply(Helper::Error);
 875         context->clientRedirectDone(nilReply);
 876     }
 877 }
 878
 879 void
 880 ClientRequestContext::clientRedirectStart()
 881 {
 882     debugs(33, 5, HERE << "'" << http->uri << "'");
 883     http->al->syncNotes(http->request);
 884     if (Config.accessList.redirector) {
 885         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 886         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 887     } else
 888         redirectStart(http, clientRedirectDoneWrapper, this);
 889 }
 890
 891 /**
 892  * This methods handles Access checks result of StoreId access list.
 893  * Will handle as "ERR" (no change) in a case Access is not allowed.
 894  */
 895 static void
 896 clientStoreIdAccessCheckDone(Acl::Answer answer, void *data)
 897 {
 898     ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
 899     ClientHttpRequest *http = context->http;
 900     context->acl_checklist = NULL;
 901
 902     if (answer.allowed())
 903         storeIdStart(http, clientStoreIdDoneWrapper, context);
 904     else {
 905         debugs(85, 3, "access denied expected ERR reply handling: " << answer);
 906         Helper::Reply const nilReply(Helper::Error);
 907         context->clientStoreIdDone(nilReply);
 908     }
 909 }
 910
 911 /**
 912  * Start locating an alternative storeage ID string (if any) from admin
 913  * configured helper program. This is an asynchronous operation terminating in
 914  * ClientRequestContext::clientStoreIdDone() when completed.
 915  */
 916 void
 917 ClientRequestContext::clientStoreIdStart()
 918 {
 919     debugs(33, 5,"'" << http->uri << "'");
 920
 921     if (Config.accessList.store_id) {
 922         acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
 923         acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
 924     } else
 925         storeIdStart(http, clientStoreIdDoneWrapper, this);
 926 }
 927
 928 static int
 929 clientHierarchical(ClientHttpRequest * http)
 930 {
 931     HttpRequest *request = http->request;
 932     HttpRequestMethod method = request->method;
 933
 934     // intercepted requests MUST NOT (yet) be sent to peers unless verified
 935     if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
 936         return 0;
 937
 938     /*
 939      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 940      * neighbors support private keys
 941      */
 942
 943     if (request->flags.ims && !neighbors_do_private_keys)
 944         return 0;
 945
 946     /*
 947      * This is incorrect: authenticating requests can be sent via a hierarchy
 948      * (they can even be cached if the correct headers are set on the reply)
 949      */
 950     if (request->flags.auth)
 951         return 0;
 952
 953     if (method == Http::METHOD_TRACE)
 954         return 1;
 955
 956     if (method != Http::METHOD_GET)
 957         return 0;
 958
 959     if (request->flags.loopDetected)
 960         return 0;
 961
 962     if (request->url.getScheme() == AnyP::PROTO_HTTP)
 963         return method.respMaybeCacheable();
 964
 965     if (request->url.getScheme() == AnyP::PROTO_GOPHER)
 966         return gopherCachable(request);
 967
 968     if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
 969         return 0;
 970
 971     return 1;
 972 }
 973
 974 static void
 975 clientCheckPinning(ClientHttpRequest * http)
 976 {
 977     HttpRequest *request = http->request;
 978     HttpHeader *req_hdr = &request->header;
 979     ConnStateData *http_conn = http->getConn();
 980
 981     /* Internal requests such as those from ESI includes may be without
 982      * a client connection
 983      */
 984     if (!http_conn)
 985         return;
 986
 987     request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
 988     if (!request->flags.connectionAuthDisabled) {
 989         if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
 990             if (http_conn->pinning.auth) {
 991                 request->flags.connectionAuth = true;
 992                 request->flags.auth = true;
 993             } else {
 994                 request->flags.connectionProxyAuth = true;
 995             }
 996             // These should already be linked correctly.
 997             assert(request->clientConnectionManager == http_conn);
 998         }
 999     }
1000
1001     /* check if connection auth is used, and flag as candidate for pinning
1002      * in such case.
1003      * Note: we may need to set flags.connectionAuth even if the connection
1004      * is already pinned if it was pinned earlier due to proxy auth
1005      */
1006     if (!request->flags.connectionAuth) {
1007         if (req_hdr->has(Http::HdrType::AUTHORIZATION) || req_hdr->has(Http::HdrType::PROXY_AUTHORIZATION)) {
1008             HttpHeaderPos pos = HttpHeaderInitPos;
1009             HttpHeaderEntry *e;
1010             int may_pin = 0;
1011             while ((e = req_hdr->getEntry(&pos))) {
1012                 if (e->id == Http::HdrType::AUTHORIZATION || e->id == Http::HdrType::PROXY_AUTHORIZATION) {
1013                     const char *value = e->value.rawBuf();
1014                     if (strncasecmp(value, "NTLM ", 5) == 0
1015                             ||
1016                             strncasecmp(value, "Negotiate ", 10) == 0
1017                             ||
1018                             strncasecmp(value, "Kerberos ", 9) == 0) {
1019                         if (e->id == Http::HdrType::AUTHORIZATION) {
1020                             request->flags.connectionAuth = true;
1021                             may_pin = 1;
1022                         } else {
1023                             request->flags.connectionProxyAuth = true;
1024                             may_pin = 1;
1025                         }
1026                     }
1027                 }
1028             }
1029             if (may_pin && !request->pinnedConnection()) {
1030                 // These should already be linked correctly. Just need the ServerConnection to pinn.
1031                 assert(request->clientConnectionManager == http_conn);
1032             }
1033         }
1034     }
1035 }
1036
1037 static void
1038 clientInterpretRequestHeaders(ClientHttpRequest * http)
1039 {
1040     HttpRequest *request = http->request;
1041     HttpHeader *req_hdr = &request->header;
1042     bool no_cache = false;
1043
1044     request->imslen = -1;
1045     request->ims = req_hdr->getTime(Http::HdrType::IF_MODIFIED_SINCE);
1046
1047     if (request->ims > 0)
1048         request->flags.ims = true;
1049
1050     if (!request->flags.ignoreCc) {
1051         if (request->cache_control) {
1052             if (request->cache_control->hasNoCache())
1053                 no_cache=true;
1054
1055             // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1056         } else if (req_hdr->has(Http::HdrType::PRAGMA))
1057             no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1058     }
1059
1060     if (request->method == Http::METHOD_OTHER) {
1061         no_cache=true;
1062     }
1063
1064     if (no_cache) {
1065 #if USE_HTTP_VIOLATIONS
1066
1067         if (Config.onoff.reload_into_ims)
1068             request->flags.nocacheHack = true;
1069         else if (refresh_nocache_hack)
1070             request->flags.nocacheHack = true;
1071         else
1072 #endif
1073
1074             request->flags.noCache = true;
1075     }
1076
1077     /* ignore range header in non-GETs or non-HEADs */
1078     if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1079         // XXX: initialize if we got here without HttpRequest::parseHeader()
1080         if (!request->range)
1081             request->range = req_hdr->getRange();
1082
1083         if (request->range) {
1084             request->flags.isRanged = true;
1085             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1086             /* XXX: This is suboptimal. We should give the stream the range set,
1087              * and thereby let the top of the stream set the offset when the
1088              * size becomes known. As it is, we will end up requesting from 0
1089              * for evey -X range specification.
1090              * RBC - this may be somewhat wrong. We should probably set the range
1091              * iter up at this point.
1092              */
1093             node->readBuffer.offset = request->range->lowestOffset(0);
1094             http->range_iter.pos = request->range->begin();
1095             http->range_iter.end = request->range->end();
1096             http->range_iter.valid = true;
1097         }
1098     }
1099
1100     /* Only HEAD and GET requests permit a Range or Request-Range header.
1101      * If these headers appear on any other type of request, delete them now.
1102      */
1103     else {
1104         req_hdr->delById(Http::HdrType::RANGE);
1105         req_hdr->delById(Http::HdrType::REQUEST_RANGE);
1106         request->ignoreRange("neither HEAD nor GET");
1107     }
1108
1109     if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1110         request->flags.auth = true;
1111
1112     clientCheckPinning(http);
1113
1114     if (!request->url.userInfo().isEmpty())
1115         request->flags.auth = true;
1116
1117     if (req_hdr->has(Http::HdrType::VIA)) {
1118         String s = req_hdr->getList(Http::HdrType::VIA);
1119         /*
1120          * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1121          * Note ThisCache2 has a space prepended to the hostname so we don't
1122          * accidentally match super-domains.
1123          */
1124
1125         if (strListIsSubstr(&s, ThisCache2, ',')) {
1126             request->flags.loopDetected = true;
1127         }
1128
1129 #if USE_FORW_VIA_DB
1130         fvdbCountVia(s.termedBuf());
1131
1132 #endif
1133
1134         s.clean();
1135     }
1136
1137     // headers only relevant to reverse-proxy
1138     if (request->flags.accelerated) {
1139         // check for a cdn-info member with a cdn-id matching surrogate_id
1140         // XXX: HttpHeader::hasListMember() does not handle OWS around ";" yet
1141         if (req_hdr->hasListMember(Http::HdrType::CDN_LOOP, Config.Accel.surrogate_id, ','))
1142             request->flags.loopDetected = true;
1143     }
1144
1145     if (request->flags.loopDetected) {
1146         debugObj(33, DBG_IMPORTANT, "WARNING: Forwarding loop detected for:\n",
1147                  request, (ObjPackMethod) & httpRequestPack);
1148     }
1149
1150 #if USE_FORW_VIA_DB
1151
1152     if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1153         String s = req_hdr->getList(Http::HdrType::X_FORWARDED_FOR);
1154         fvdbCountForw(s.termedBuf());
1155         s.clean();
1156     }
1157
1158 #endif
1159
1160     request->flags.cachable = http->request->maybeCacheable();
1161
1162     if (clientHierarchical(http))
1163         request->flags.hierarchical = true;
1164
1165     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1166            (request->flags.noCache ? "SET" : "NOT SET"));
1167     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1168            (request->flags.cachable ? "SET" : "NOT SET"));
1169     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1170            (request->flags.hierarchical ? "SET" : "NOT SET"));
1171
1172 }
1173
1174 void
1175 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1176 {
1177     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1178
1179     if (!calloutContext->httpStateIsValid())
1180         return;
1181
1182     calloutContext->clientRedirectDone(result);
1183 }
1184
1185 void
1186 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1187 {
1188     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1189
1190     if (!calloutContext->httpStateIsValid())
1191         return;
1192
1193     calloutContext->clientStoreIdDone(result);
1194 }
1195
1196 void
1197 ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1198 {
1199     HttpRequest *old_request = http->request;
1200     debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1201     assert(redirect_state == REDIRECT_PENDING);
1202     redirect_state = REDIRECT_DONE;
1203
1204     // Put helper response Notes into the transaction state record (ALE) eventually
1205     // do it early to ensure that no matter what the outcome the notes are present.
1206     if (http->al)
1207         http->al->syncNotes(old_request);
1208
1209     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1210
1211     switch (reply.result) {
1212     case Helper::TimedOut:
1213         if (Config.onUrlRewriteTimeout.action != toutActBypass) {
1214             http->calloutsError(ERR_GATEWAY_FAILURE, ERR_DETAIL_REDIRECTOR_TIMEDOUT);
1215             debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1216         }
1217         break;
1218
1219     case Helper::Unknown:
1220     case Helper::TT:
1221         // Handler in redirect.cc should have already mapped Unknown
1222         // IF it contained valid entry for the old URL-rewrite helper protocol
1223         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1224         break;
1225
1226     case Helper::BrokenHelper:
1227         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1228         break;
1229
1230     case Helper::Error:
1231         // no change to be done.
1232         break;
1233
1234     case Helper::Okay: {
1235         // #1: redirect with a specific status code    OK status=NNN url="..."
1236         // #2: redirect with a default status code     OK url="..."
1237         // #3: re-write the URL                        OK rewrite-url="..."
1238
1239         const char *statusNote = reply.notes.findFirst("status");
1240         const char *urlNote = reply.notes.findFirst("url");
1241
1242         if (urlNote != NULL) {
1243             // HTTP protocol redirect to be done.
1244
1245             // TODO: change default redirect status for appropriate requests
1246             // Squid defaults to 302 status for now for better compatibility with old clients.
1247             // HTTP/1.0 client should get 302 (Http::scFound)
1248             // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1249             // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1250             Http::StatusCode status = Http::scFound;
1251             if (statusNote != NULL) {
1252                 const char * result = statusNote;
1253                 status = static_cast<Http::StatusCode>(atoi(result));
1254             }
1255
1256             if (status == Http::scMovedPermanently
1257                     || status == Http::scFound
1258                     || status == Http::scSeeOther
1259                     || status == Http::scPermanentRedirect
1260                     || status == Http::scTemporaryRedirect) {
1261                 http->redirect.status = status;
1262                 http->redirect.location = xstrdup(urlNote);
1263                 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1264             } else {
1265                 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1266             }
1267         } else {
1268             // URL-rewrite wanted. Ew.
1269             urlNote = reply.notes.findFirst("rewrite-url");
1270
1271             // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1272             if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1273                 AnyP::Uri tmpUrl;
1274                 if (tmpUrl.parse(old_request->method, SBuf(urlNote))) {
1275                     HttpRequest *new_request = old_request->clone();
1276                     new_request->url = tmpUrl;
1277                     debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1278
1279                     // update the new request to flag the re-writing was done on it
1280                     new_request->flags.redirected = true;
1281
1282                     // unlink bodypipe from the old request. Not needed there any longer.
1283                     if (old_request->body_pipe != NULL) {
1284                         old_request->body_pipe = NULL;
1285                         debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1286                                " from request " << old_request << " to " << new_request);
1287                     }
1288
1289                     http->resetRequest(new_request);
1290                     old_request = nullptr;
1291                 } else {
1292                     debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1293                            old_request->method << " " << urlNote << " " << old_request->http_ver);
1294                 }
1295             }
1296         }
1297     }
1298     break;
1299     }
1300
1301     /* FIXME PIPELINE: This is innacurate during pipelining */
1302
1303     if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1304         fd_note(http->getConn()->clientConnection->fd, http->uri);
1305
1306     assert(http->uri);
1307
1308     http->doCallouts();
1309 }
1310
1311 /**
1312  * This method handles the different replies from StoreID helper.
1313  */
1314 void
1315 ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1316 {
1317     HttpRequest *old_request = http->request;
1318     debugs(85, 5, "'" << http->uri << "' result=" << reply);
1319     assert(store_id_state == REDIRECT_PENDING);
1320     store_id_state = REDIRECT_DONE;
1321
1322     // Put helper response Notes into the transaction state record (ALE) eventually
1323     // do it early to ensure that no matter what the outcome the notes are present.
1324     if (http->al)
1325         http->al->syncNotes(old_request);
1326
1327     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1328
1329     switch (reply.result) {
1330     case Helper::Unknown:
1331     case Helper::TT:
1332         // Handler in redirect.cc should have already mapped Unknown
1333         // IF it contained valid entry for the old helper protocol
1334         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1335         break;
1336
1337     case Helper::TimedOut:
1338     // Timeouts for storeID are not implemented
1339     case Helper::BrokenHelper:
1340         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1341         break;
1342
1343     case Helper::Error:
1344         // no change to be done.
1345         break;
1346
1347     case Helper::Okay: {
1348         const char *urlNote = reply.notes.findFirst("store-id");
1349
1350         // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1351         if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1352             // Debug section required for some very specific cases.
1353             debugs(85, 9, "Setting storeID with: " << urlNote );
1354             http->request->store_id = urlNote;
1355             http->store_id = urlNote;
1356         }
1357     }
1358     break;
1359     }
1360
1361     http->doCallouts();
1362 }
1363
1364 /** Test cache allow/deny configuration
1365  *  Sets flags.cachable=1 if caching is not denied.
1366  */
1367 void
1368 ClientRequestContext::checkNoCache()
1369 {
1370     if (Config.accessList.noCache) {
1371         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1372         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1373     } else {
1374         /* unless otherwise specified, we try to cache. */
1375         checkNoCacheDone(ACCESS_ALLOWED);
1376     }
1377 }
1378
1379 static void
1380 checkNoCacheDoneWrapper(Acl::Answer answer, void *data)
1381 {
1382     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1383
1384     if (!calloutContext->httpStateIsValid())
1385         return;
1386
1387     calloutContext->checkNoCacheDone(answer);
1388 }
1389
1390 void
1391 ClientRequestContext::checkNoCacheDone(const Acl::Answer &answer)
1392 {
1393     acl_checklist = NULL;
1394     if (answer.denied()) {
1395         http->request->flags.noCache = true; // do not read reply from cache
1396         http->request->flags.cachable = false; // do not store reply into cache
1397     }
1398     http->doCallouts();
1399 }
1400
1401 #if USE_OPENSSL
1402 bool
1403 ClientRequestContext::sslBumpAccessCheck()
1404 {
1405     if (!http->getConn()) {
1406         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1407         return false;
1408     }
1409
1410     const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1411     if (http->request->flags.forceTunnel) {
1412         debugs(85, 5, "not needed; already decided to tunnel " << http->getConn());
1413         if (bumpMode != Ssl::bumpEnd)
1414             http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1415         return false;
1416     }
1417
1418     // If SSL connection tunneling or bumping decision has been made, obey it.
1419     if (bumpMode != Ssl::bumpEnd) {
1420         debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1421                "), " << "ignoring ssl_bump for " << http->getConn());
1422
1423         // We need the following "if" for transparently bumped TLS connection,
1424         // because in this case we are running ssl_bump access list before
1425         // the doCallouts runs. It can be removed after the bug #4340 fixed.
1426         // We do not want to proceed to bumping steps:
1427         //  - if the TLS connection with the client is already established
1428         //    because we are accepting normal HTTP requests on TLS port,
1429         //    or because of the client-first bumping mode
1430         //  - When the bumping is already started
1431         if (!http->getConn()->switchedToHttps() &&
1432                 !http->getConn()->serverBump())
1433             http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1434         http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1435         return false;
1436     }
1437
1438     // If we have not decided yet, decide whether to bump now.
1439
1440     // Bumping here can only start with a CONNECT request on a bumping port
1441     // (bumping of intercepted SSL conns is decided before we get 1st request).
1442     // We also do not bump redirected CONNECT requests.
1443     if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1444             !Config.accessList.ssl_bump ||
1445             !http->getConn()->port->flags.tunnelSslBumping) {
1446         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1447         debugs(85, 5, HERE << "cannot SslBump this request");
1448         return false;
1449     }
1450
1451     // Do not bump during authentication: clients would not proxy-authenticate
1452     // if we delay a 407 response and respond with 200 OK to CONNECT.
1453     if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1454         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1455         debugs(85, 5, HERE << "no SslBump during proxy authentication");
1456         return false;
1457     }
1458
1459     if (error) {
1460         debugs(85, 5, "SslBump applies. Force bump action on error " << errorTypeName(error->type));
1461         http->sslBumpNeed(Ssl::bumpBump);
1462         http->al->ssl.bumpMode = Ssl::bumpBump;
1463         return false;
1464     }
1465
1466     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1467
1468     ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1469     aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1470     return true;
1471 }
1472
1473 /**
1474  * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1475  * as ACLFilledChecklist callback
1476  */
1477 static void
1478 sslBumpAccessCheckDoneWrapper(Acl::Answer answer, void *data)
1479 {
1480     ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1481
1482     if (!calloutContext->httpStateIsValid())
1483         return;
1484     calloutContext->sslBumpAccessCheckDone(answer);
1485 }
1486
1487 void
1488 ClientRequestContext::sslBumpAccessCheckDone(const Acl::Answer &answer)
1489 {
1490     if (!httpStateIsValid())
1491         return;
1492
1493     const Ssl::BumpMode bumpMode = answer.allowed() ?
1494                                    static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpSplice;
1495     http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1496     http->al->ssl.bumpMode = bumpMode; // for logging
1497
1498     if (bumpMode == Ssl::bumpTerminate) {
1499         const Comm::ConnectionPointer clientConn = http->getConn() ? http->getConn()->clientConnection : nullptr;
1500         if (Comm::IsConnOpen(clientConn)) {
1501             debugs(85, 3, "closing after Ssl::bumpTerminate ");
1502             clientConn->close();
1503         }
1504         return;
1505     }
1506
1507     http->doCallouts();
1508 }
1509 #endif
1510
1511 /*
1512  * Identify requests that do not go through the store and client side stream
1513  * and forward them to the appropriate location. All other requests, request
1514  * them.
1515  */
1516 void
1517 ClientHttpRequest::processRequest()
1518 {
1519     debugs(85, 4, request->method << ' ' << uri);
1520
1521     const bool untouchedConnect = request->method == Http::METHOD_CONNECT && !redirect.status;
1522
1523 #if USE_OPENSSL
1524     if (untouchedConnect && sslBumpNeeded()) {
1525         assert(!request->flags.forceTunnel);
1526         sslBumpStart();
1527         return;
1528     }
1529 #endif
1530
1531     if (untouchedConnect || request->flags.forceTunnel) {
1532         getConn()->stopReading(); // tunnels read for themselves
1533         tunnelStart(this);
1534         return;
1535     }
1536
1537     httpStart();
1538 }
1539
1540 void
1541 ClientHttpRequest::httpStart()
1542 {
1543     PROF_start(httpStart);
1544     // XXX: Re-initializes rather than updates. Should not be needed at all.
1545     logType.update(LOG_TAG_NONE);
1546     debugs(85, 4, logType.c_str() << " for '" << uri << "'");
1547
1548     /* no one should have touched this */
1549     assert(out.offset == 0);
1550     /* Use the Stream Luke */
1551     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1552     clientStreamRead(node, this, node->readBuffer);
1553     PROF_stop(httpStart);
1554 }
1555
1556 #if USE_OPENSSL
1557
1558 void
1559 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1560 {
1561     debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1562     sslBumpNeed_ = mode;
1563 }
1564
1565 // called when comm_write has completed
1566 static void
1567 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1568 {
1569     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1570     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1571
1572     assert(r && cbdataReferenceValid(r));
1573     r->sslBumpEstablish(errflag);
1574 }
1575
1576 void
1577 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1578 {
1579     // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1580     if (errflag == Comm::ERR_CLOSING)
1581         return;
1582
1583     if (errflag) {
1584         debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1585         getConn()->clientConnection->close();
1586         return;
1587     }
1588
1589 #if USE_AUTH
1590     // Preserve authentication info for the ssl-bumped request
1591     if (request->auth_user_request != NULL)
1592         getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1593 #endif
1594
1595     assert(sslBumpNeeded());
1596     getConn()->switchToHttps(this, sslBumpNeed_);
1597 }
1598
1599 void
1600 ClientHttpRequest::sslBumpStart()
1601 {
1602     debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1603            "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1604     getConn()->sslBumpMode = sslBumpNeed_;
1605
1606     AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1607                                   CommIoCbPtrFun(&SslBumpEstablish, this));
1608
1609     if (request->flags.interceptTproxy || request->flags.intercepted) {
1610         CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1611         params.flag = Comm::OK;
1612         params.conn = getConn()->clientConnection;
1613         ScheduleCallHere(bumpCall);
1614         return;
1615     }
1616
1617     al->reply = HttpReply::MakeConnectionEstablished();
1618
1619     const auto mb = al->reply->pack();
1620     // send an HTTP 200 response to kick client SSL negotiation
1621     // TODO: Unify with tunnel.cc and add a Server(?) header
1622     Comm::Write(getConn()->clientConnection, mb, bumpCall);
1623     delete mb;
1624 }
1625
1626 #endif
1627
1628 bool
1629 ClientHttpRequest::gotEnough() const
1630 {
1631     // TODO: See also (and unify with) clientReplyContext::storeNotOKTransferDone()
1632     int64_t contentLength =
1633         memObject()->baseReply().bodySize(request->method);
1634     assert(contentLength >= 0);
1635
1636     if (out.offset < contentLength)
1637         return false;
1638
1639     return true;
1640 }
1641
1642 void
1643 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1644 {
1645     entry_ = newEntry;
1646 }
1647
1648 void
1649 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1650 {
1651     if (loggingEntry_)
1652         loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1653
1654     loggingEntry_ = newEntry;
1655
1656     if (loggingEntry_)
1657         loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1658 }
1659
1660 void
1661 ClientHttpRequest::initRequest(HttpRequest *aRequest)
1662 {
1663     assignRequest(aRequest);
1664     if (const auto csd = getConn()) {
1665         if (!csd->notes()->empty())
1666             request->notes()->appendNewOnly(csd->notes().getRaw());
1667     }
1668     // al is created in the constructor
1669     assert(al);
1670     if (!al->request) {
1671         al->request = request;
1672         HTTPMSGLOCK(al->request);
1673         al->syncNotes(request);
1674     }
1675 }
1676
1677 void
1678 ClientHttpRequest::resetRequest(HttpRequest *newRequest)
1679 {
1680     assert(request != newRequest);
1681     clearRequest();
1682     assignRequest(newRequest);
1683     xfree(uri);
1684     uri = SBufToCstring(request->effectiveRequestUri());
1685 }
1686
1687 void
1688 ClientHttpRequest::assignRequest(HttpRequest *newRequest)
1689 {
1690     assert(newRequest);
1691     assert(!request);
1692     const_cast<HttpRequest *&>(request) = newRequest;
1693     HTTPMSGLOCK(request);
1694     setLogUriToRequestUri();
1695 }
1696
1697 void
1698 ClientHttpRequest::clearRequest()
1699 {
1700     HttpRequest *oldRequest = request;
1701     HTTPMSGUNLOCK(oldRequest);
1702     const_cast<HttpRequest *&>(request) = nullptr;
1703     absorbLogUri(nullptr);
1704 }
1705
1706 /*
1707  * doCallouts() - This function controls the order of "callout"
1708  * executions, including non-blocking access control checks, the
1709  * redirector, and ICAP.  Previously, these callouts were chained
1710  * together such that "clientAccessCheckDone()" would call
1711  * "clientRedirectStart()" and so on.
1712  *
1713  * The ClientRequestContext (aka calloutContext) class holds certain
1714  * state data for the callout/callback operations.  Previously
1715  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1716  * for a short time.  ClientRequestContext would then delete itself
1717  * and pass control back to ClientHttpRequest when all callouts
1718  * were finished.
1719  *
1720  * This caused some problems for ICAP because we want to make the
1721  * ICAP callout after checking ACLs, but before checking the no_cache
1722  * list.  We can't stuff the ICAP state into the ClientRequestContext
1723  * class because we still need the ICAP state after ClientRequestContext
1724  * goes away.
1725  *
1726  * Note that ClientRequestContext is created before the first call
1727  * to doCallouts().
1728  *
1729  * If one of the callouts notices that ClientHttpRequest is no
1730  * longer valid, it should call cbdataReferenceDone() so that
1731  * ClientHttpRequest's reference count goes to zero and it will get
1732  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1733  *
1734  * Note that we set the _done flags here before actually starting
1735  * the callout.  This is strictly for convenience.
1736  */
1737
1738 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1739 Ip::NfMarkConfig aclFindNfMarkConfig (acl_nfmark * head, ACLChecklist * ch);
1740
1741 void
1742 ClientHttpRequest::doCallouts()
1743 {
1744     assert(calloutContext);
1745
1746     if (!calloutContext->error) {
1747         // CVE-2009-0801: verify the Host: header is consistent with other known details.
1748         if (!calloutContext->host_header_verify_done) {
1749             debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1750             calloutContext->host_header_verify_done = true;
1751             calloutContext->hostHeaderVerify();
1752             return;
1753         }
1754
1755         if (!calloutContext->http_access_done) {
1756             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1757             calloutContext->http_access_done = true;
1758             calloutContext->clientAccessCheck();
1759             return;
1760         }
1761
1762 #if USE_ADAPTATION
1763         if (!calloutContext->adaptation_acl_check_done) {
1764             calloutContext->adaptation_acl_check_done = true;
1765             if (Adaptation::AccessCheck::Start(
1766                         Adaptation::methodReqmod, Adaptation::pointPreCache,
1767                         request, NULL, calloutContext->http->al, this))
1768                 return; // will call callback
1769         }
1770 #endif
1771
1772         if (!calloutContext->redirect_done) {
1773             calloutContext->redirect_done = true;
1774
1775             if (Config.Program.redirect) {
1776                 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1777                 calloutContext->redirect_state = REDIRECT_PENDING;
1778                 calloutContext->clientRedirectStart();
1779                 return;
1780             }
1781         }
1782
1783         if (!calloutContext->adapted_http_access_done) {
1784             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1785             calloutContext->adapted_http_access_done = true;
1786             calloutContext->clientAccessCheck2();
1787             return;
1788         }
1789
1790         if (!calloutContext->store_id_done) {
1791             calloutContext->store_id_done = true;
1792
1793             if (Config.Program.store_id) {
1794                 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1795                 calloutContext->store_id_state = REDIRECT_PENDING;
1796                 calloutContext->clientStoreIdStart();
1797                 return;
1798             }
1799         }
1800
1801         if (!calloutContext->interpreted_req_hdrs) {
1802             debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1803             calloutContext->interpreted_req_hdrs = 1;
1804             clientInterpretRequestHeaders(this);
1805         }
1806
1807         if (!calloutContext->no_cache_done) {
1808             calloutContext->no_cache_done = true;
1809
1810             if (Config.accessList.noCache && request->flags.cachable) {
1811                 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1812                 calloutContext->checkNoCache();
1813                 return;
1814             }
1815         }
1816     } //  if !calloutContext->error
1817
1818     // Set appropriate MARKs and CONNMARKs if needed.
1819     if (getConn() && Comm::IsConnOpen(getConn()->clientConnection)) {
1820         ACLFilledChecklist ch(nullptr, request, nullptr);
1821         ch.al = calloutContext->http->al;
1822         ch.src_addr = request->client_addr;
1823         ch.my_addr = request->my_addr;
1824         ch.syncAle(request, log_uri);
1825
1826         if (!calloutContext->toClientMarkingDone) {
1827             calloutContext->toClientMarkingDone = true;
1828             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1829             if (tos)
1830                 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1831
1832             const auto packetMark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1833             if (!packetMark.isEmpty())
1834                 Ip::Qos::setSockNfmark(getConn()->clientConnection, packetMark.mark);
1835
1836             const auto connmark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfConnmarkToClient, &ch);
1837             if (!connmark.isEmpty())
1838                 Ip::Qos::setNfConnmark(getConn()->clientConnection, Ip::Qos::dirAccepted, connmark);
1839         }
1840     }
1841
1842 #if USE_OPENSSL
1843     // Even with calloutContext->error, we call sslBumpAccessCheck() to decide
1844     // whether SslBump applies to this transaction. If it applies, we will
1845     // attempt to bump the client to serve the error.
1846     if (!calloutContext->sslBumpCheckDone) {
1847         calloutContext->sslBumpCheckDone = true;
1848         if (calloutContext->sslBumpAccessCheck())
1849             return;
1850         /* else no ssl bump required*/
1851     }
1852 #endif
1853
1854     if (calloutContext->error) {
1855         // XXX: prformance regression. c_str() reallocates
1856         SBuf storeUriBuf(request->storeId());
1857         const char *storeUri = storeUriBuf.c_str();
1858         StoreEntry *e = storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1859 #if USE_OPENSSL
1860         if (sslBumpNeeded()) {
1861             // We have to serve an error, so bump the client first.
1862             sslBumpNeed(Ssl::bumpClientFirst);
1863             // set final error but delay sending until we bump
1864             Ssl::ServerBump *srvBump = new Ssl::ServerBump(this, e, Ssl::bumpClientFirst);
1865             errorAppendEntry(e, calloutContext->error);
1866             calloutContext->error = NULL;
1867             getConn()->setServerBump(srvBump);
1868             e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1869         } else
1870 #endif
1871         {
1872             // send the error to the client now
1873             clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1874             clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1875             assert (repContext);
1876             repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1877             errorAppendEntry(e, calloutContext->error);
1878             calloutContext->error = NULL;
1879             if (calloutContext->readNextRequest && getConn())
1880                 getConn()->flags.readMore = true; // resume any pipeline reads.
1881             node = (clientStreamNode *)client_stream.tail->data;
1882             clientStreamRead(node, this, node->readBuffer);
1883             e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1884             return;
1885         }
1886     }
1887
1888     cbdataReferenceDone(calloutContext->http);
1889     delete calloutContext;
1890     calloutContext = NULL;
1891 #if HEADERS_LOG
1892
1893     headersLog(0, 1, request->method, request);
1894 #endif
1895
1896     debugs(83, 3, HERE << "calling processRequest()");
1897     processRequest();
1898
1899 #if ICAP_CLIENT
1900     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1901     if (ih != NULL)
1902         ih->logType = logType;
1903 #endif
1904 }
1905
1906 void
1907 ClientHttpRequest::setLogUriToRequestUri()
1908 {
1909     assert(request);
1910     const auto canonicalUri = request->canonicalCleanUrl();
1911     absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1912 }
1913
1914 void
1915 ClientHttpRequest::setLogUriToRawUri(const char *rawUri, const HttpRequestMethod &method)
1916 {
1917     assert(rawUri);
1918     // Should(!request);
1919
1920     // TODO: SBuf() performance regression, fix by converting rawUri to SBuf
1921     char *canonicalUri = urlCanonicalCleanWithoutRequest(SBuf(rawUri), method, AnyP::UriScheme());
1922
1923     absorbLogUri(AnyP::Uri::cleanup(canonicalUri));
1924
1925     char *cleanedRawUri = AnyP::Uri::cleanup(rawUri);
1926     al->setVirginUrlForMissingRequest(SBuf(cleanedRawUri));
1927     xfree(cleanedRawUri);
1928 }
1929
1930 void
1931 ClientHttpRequest::absorbLogUri(char *aUri)
1932 {
1933     xfree(log_uri);
1934     const_cast<char *&>(log_uri) = aUri;
1935 }
1936
1937 void
1938 ClientHttpRequest::setErrorUri(const char *aUri)
1939 {
1940     assert(!uri);
1941     assert(aUri);
1942     // Should(!request);
1943
1944     uri = xstrdup(aUri);
1945     // TODO: SBuf() performance regression, fix by converting setErrorUri() parameter to SBuf
1946     const SBuf errorUri(aUri);
1947     const auto canonicalUri = urlCanonicalCleanWithoutRequest(errorUri, HttpRequestMethod(), AnyP::UriScheme());
1948     absorbLogUri(xstrndup(canonicalUri, MAX_URL));
1949
1950     al->setVirginUrlForMissingRequest(errorUri);
1951 }
1952
1953 #if USE_ADAPTATION
1954 /// Initiate an asynchronous adaptation transaction which will call us back.
1955 void
1956 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1957 {
1958     debugs(85, 3, HERE << "adaptation needed for " << this);
1959     assert(!virginHeadSource);
1960     assert(!adaptedBodySource);
1961     virginHeadSource = initiateAdaptation(
1962                            new Adaptation::Iterator(request, NULL, al, g));
1963
1964     // we could try to guess whether we can bypass this adaptation
1965     // initiation failure, but it should not really happen
1966     Must(initiated(virginHeadSource));
1967 }
1968
1969 void
1970 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1971 {
1972     assert(cbdataReferenceValid(this));     // indicates bug
1973     clearAdaptation(virginHeadSource);
1974     assert(!adaptedBodySource);
1975
1976     switch (answer.kind) {
1977     case Adaptation::Answer::akForward:
1978         handleAdaptedHeader(const_cast<Http::Message*>(answer.message.getRaw()));
1979         break;
1980
1981     case Adaptation::Answer::akBlock:
1982         handleAdaptationBlock(answer);
1983         break;
1984
1985     case Adaptation::Answer::akError:
1986         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1987         break;
1988     }
1989 }
1990
1991 void
1992 ClientHttpRequest::handleAdaptedHeader(Http::Message *msg)
1993 {
1994     assert(msg);
1995
1996     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1997         // update the new message to flag whether URL re-writing was done on it
1998         if (request->effectiveRequestUri() != new_req->effectiveRequestUri())
1999             new_req->flags.redirected = true;
2000         resetRequest(new_req);
2001         assert(request->method.id());
2002     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
2003         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
2004
2005         // subscribe to receive reply body
2006         if (new_rep->body_pipe != NULL) {
2007             adaptedBodySource = new_rep->body_pipe;
2008             int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
2009             assert(consumer_ok);
2010         }
2011
2012         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2013         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2014         assert(repContext);
2015         repContext->createStoreEntry(request->method, request->flags);
2016
2017         request_satisfaction_mode = true;
2018         request_satisfaction_offset = 0;
2019         storeEntry()->replaceHttpReply(new_rep);
2020         storeEntry()->timestampsSet();
2021
2022         al->reply = new_rep;
2023
2024         if (!adaptedBodySource) // no body
2025             storeEntry()->complete();
2026         clientGetMoreData(node, this);
2027     }
2028
2029     // we are done with getting headers (but may be receiving body)
2030     clearAdaptation(virginHeadSource);
2031
2032     if (!request_satisfaction_mode)
2033         doCallouts();
2034 }
2035
2036 void
2037 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
2038 {
2039     request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
2040     AclMatchedName = answer.ruleId.termedBuf();
2041     assert(calloutContext);
2042     calloutContext->clientAccessCheckDone(ACCESS_DENIED);
2043     AclMatchedName = NULL;
2044 }
2045
2046 void
2047 ClientHttpRequest::resumeBodyStorage()
2048 {
2049     if (!adaptedBodySource)
2050         return;
2051
2052     noteMoreBodyDataAvailable(adaptedBodySource);
2053 }
2054
2055 void
2056 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
2057 {
2058     assert(request_satisfaction_mode);
2059     assert(adaptedBodySource != NULL);
2060
2061     if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
2062         const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
2063
2064         if (spaceAvailable < contentSize ) {
2065             // No or partial body data consuming
2066             typedef NullaryMemFunT<ClientHttpRequest> Dialer;
2067             AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
2068                                                 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
2069             storeEntry()->deferProducer(call);
2070         }
2071
2072         if (!spaceAvailable)
2073             return;
2074
2075         if (spaceAvailable < contentSize )
2076             contentSize = spaceAvailable;
2077
2078         BodyPipeCheckout bpc(*adaptedBodySource);
2079         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
2080         storeEntry()->write(ioBuf);
2081         // assume StoreEntry::write() writes the entire ioBuf
2082         request_satisfaction_offset += ioBuf.length;
2083         bpc.buf.consume(contentSize);
2084         bpc.checkIn();
2085     }
2086
2087     if (adaptedBodySource->exhausted())
2088         endRequestSatisfaction();
2089     // else wait for more body data
2090 }
2091
2092 void
2093 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
2094 {
2095     assert(!virginHeadSource);
2096     // should we end request satisfaction now?
2097     if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
2098         endRequestSatisfaction();
2099 }
2100
2101 void
2102 ClientHttpRequest::endRequestSatisfaction()
2103 {
2104     debugs(85,4, HERE << this << " ends request satisfaction");
2105     assert(request_satisfaction_mode);
2106     stopConsumingFrom(adaptedBodySource);
2107
2108     // TODO: anything else needed to end store entry formation correctly?
2109     storeEntry()->complete();
2110 }
2111
2112 void
2113 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2114 {
2115     assert(!virginHeadSource);
2116     stopConsumingFrom(adaptedBodySource);
2117
2118     debugs(85,3, HERE << "REQMOD body production failed");
2119     if (request_satisfaction_mode) { // too late to recover or serve an error
2120         request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2121         const Comm::ConnectionPointer c = getConn()->clientConnection;
2122         Must(Comm::IsConnOpen(c));
2123         c->close(); // drastic, but we may be writing a response already
2124     } else {
2125         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2126     }
2127 }
2128
2129 void
2130 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2131 {
2132     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2133
2134     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2135     const bool usedPipe = request->body_pipe != NULL &&
2136                           request->body_pipe->consumedSize() > 0;
2137
2138     if (bypassable && !usedStore && !usedPipe) {
2139         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2140         if (calloutContext)
2141             doCallouts();
2142         return;
2143     }
2144
2145     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2146
2147     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2148     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2149     assert(repContext);
2150
2151     calloutsError(ERR_ICAP_FAILURE, errDetail);
2152
2153     if (calloutContext)
2154         doCallouts();
2155 }
2156
2157 void
2158 ClientHttpRequest::callException(const std::exception &ex)
2159 {
2160     if (const auto clientConn = getConn() ? getConn()->clientConnection : nullptr) {
2161         if (Comm::IsConnOpen(clientConn)) {
2162             debugs(85, 3, "closing after exception: " << ex.what());
2163             clientConn->close(); // initiate orderly top-to-bottom cleanup
2164             return;
2165         }
2166     }
2167     debugs(85, DBG_IMPORTANT, "ClientHttpRequest exception without connection. Ignoring " << ex.what());
2168     // XXX: Normally, we mustStop() but we cannot do that here because it is
2169     // likely to leave Http::Stream and ConnStateData with a dangling http
2170     // pointer. See r13480 or XXX in Http::Stream class description.
2171 }
2172 #endif
2173
2174 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2175 void
2176 ClientHttpRequest::calloutsError(const err_type error, const int errDetail)
2177 {
2178     // The original author of the code also wanted to pass an errno to
2179     // setReplyToError, but it seems unlikely that the errno reflects the
2180     // true cause of the error at this point, so I did not pass it.
2181     if (calloutContext) {
2182         Ip::Address noAddr;
2183         noAddr.setNoAddr();
2184         ConnStateData * c = getConn();
2185         calloutContext->error = clientBuildError(error, Http::scInternalServerError,
2186                                 NULL,
2187                                 c != NULL ? c->clientConnection->remote : noAddr,
2188                                 request,
2189                                 al
2190                                                 );
2191 #if USE_AUTH
2192         calloutContext->error->auth_user_request =
2193             c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2194 #endif
2195         calloutContext->error->detailError(errDetail);
2196         calloutContext->readNextRequest = true;
2197         if (c != NULL)
2198             c->expectNoForwarding();
2199     }
2200     //else if(calloutContext == NULL) is it possible?
2201 }
2202