src/client_side_request.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 85    Client-side Request Routines */
  10
  11 /*
  12  * General logic of request processing:
  13  *
  14  * We run a series of tests to determine if access will be permitted, and to do
  15  * any redirection. Then we call into the result clientStream to retrieve data.
  16  * From that point on it's up to reply management.
  17  */
  18
  19 #include "squid.h"
  20 #include "acl/FilledChecklist.h"
  21 #include "acl/Gadgets.h"
  22 #include "anyp/PortCfg.h"
  23 #include "client_side.h"
  24 #include "client_side_reply.h"
  25 #include "client_side_request.h"
  26 #include "ClientRequestContext.h"
  27 #include "clientStream.h"
  28 #include "comm/Connection.h"
  29 #include "comm/Write.h"
  30 #include "compat/inet_pton.h"
  31 #include "err_detail_type.h"
  32 #include "errorpage.h"
  33 #include "fd.h"
  34 #include "fde.h"
  35 #include "format/Token.h"
  36 #include "gopher.h"
  37 #include "helper.h"
  38 #include "http.h"
  39 #include "HttpHdrCc.h"
  40 #include "HttpReply.h"
  41 #include "HttpRequest.h"
  42 #include "ip/QosConfig.h"
  43 #include "ipcache.h"
  44 #include "log/access_log.h"
  45 #include "MemObject.h"
  46 #include "Parsing.h"
  47 #include "profiler/Profiler.h"
  48 #include "redirect.h"
  49 #include "SquidConfig.h"
  50 #include "SquidTime.h"
  51 #include "Store.h"
  52 #include "StrList.h"
  53 #include "tools.h"
  54 #include "URL.h"
  55 #include "wordlist.h"
  56 #if USE_AUTH
  57 #include "auth/UserRequest.h"
  58 #endif
  59 #if USE_ADAPTATION
  60 #include "adaptation/AccessCheck.h"
  61 #include "adaptation/Answer.h"
  62 #include "adaptation/Iterator.h"
  63 #include "adaptation/Service.h"
  64 #if ICAP_CLIENT
  65 #include "adaptation/icap/History.h"
  66 #endif
  67 #endif
  68 #if USE_OPENSSL
  69 #include "ssl/ServerBump.h"
  70 #include "ssl/support.h"
  71 #endif
  72
  73 #if LINGERING_CLOSE
  74 #define comm_close comm_lingering_close
  75 #endif
  76
  77 static const char *const crlf = "\r\n";
  78
  79 #if FOLLOW_X_FORWARDED_FOR
  80 static void clientFollowXForwardedForCheck(allow_t answer, void *data);
  81 #endif /* FOLLOW_X_FORWARDED_FOR */
  82
  83 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
  84
  85 CBDATA_CLASS_INIT(ClientRequestContext);
  86
  87 /* Local functions */
  88 /* other */
  89 static void clientAccessCheckDoneWrapper(allow_t, void *);
  90 #if USE_OPENSSL
  91 static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
  92 #endif
  93 static int clientHierarchical(ClientHttpRequest * http);
  94 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  95 static HLPCB clientRedirectDoneWrapper;
  96 static HLPCB clientStoreIdDoneWrapper;
  97 static void checkNoCacheDoneWrapper(allow_t, void *);
  98 SQUIDCEXTERN CSR clientGetMoreData;
  99 SQUIDCEXTERN CSS clientReplyStatus;
 100 SQUIDCEXTERN CSD clientReplyDetach;
 101 static void checkFailureRatio(err_type, hier_code);
 102
 103 ClientRequestContext::~ClientRequestContext()
 104 {
 105     /*
 106      * Release our "lock" on our parent, ClientHttpRequest, if we
 107      * still have one
 108      */
 109
 110     if (http)
 111         cbdataReferenceDone(http);
 112
 113     delete error;
 114     debugs(85,3, HERE << this << " ClientRequestContext destructed");
 115 }
 116
 117 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_id_state(REDIRECT_NONE),error(NULL), readNextRequest(false)
 118 {
 119     http_access_done = false;
 120     redirect_done = false;
 121     redirect_fail_count = 0;
 122     store_id_done = false;
 123     store_id_fail_count = 0;
 124     no_cache_done = false;
 125     interpreted_req_hdrs = false;
 126 #if USE_OPENSSL
 127     sslBumpCheckDone = false;
 128 #endif
 129     debugs(85,3, HERE << this << " ClientRequestContext constructed");
 130 }
 131
 132 CBDATA_CLASS_INIT(ClientHttpRequest);
 133
 134 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 135 #if USE_ADAPTATION
 136         AsyncJob("ClientHttpRequest"),
 137 #endif
 138         loggingEntry_(NULL)
 139 {
 140     setConn(aConn);
 141     al = new AccessLogEntry;
 142     al->cache.start_time = current_time;
 143     al->tcpClient = clientConnection = aConn->clientConnection;
 144     al->cache.port = aConn->port;
 145     al->cache.caddr = aConn->log_addr;
 146
 147 #if USE_OPENSSL
 148     if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
 149         if (SSL *ssl = fd_table[aConn->clientConnection->fd].ssl)
 150             al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
 151     }
 152 #endif
 153     dlinkAdd(this, &active, &ClientActiveRequests);
 154 #if USE_ADAPTATION
 155     request_satisfaction_mode = false;
 156 #endif
 157 #if USE_OPENSSL
 158     sslBumpNeed_ = Ssl::bumpEnd;
 159 #endif
 160 }
 161
 162 /*
 163  * returns true if client specified that the object must come from the cache
 164  * without contacting origin server
 165  */
 166 bool
 167 ClientHttpRequest::onlyIfCached()const
 168 {
 169     assert(request);
 170     return request->cache_control &&
 171            request->cache_control->onlyIfCached();
 172 }
 173
 174 /**
 175  * This function is designed to serve a fairly specific purpose.
 176  * Occasionally our vBNS-connected caches can talk to each other, but not
 177  * the rest of the world.  Here we try to detect frequent failures which
 178  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 179  * the failure:success ratio goes above 1.0 then we go into "hit only"
 180  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 181  * will only fetch HITs from us if they are using the ICP protocol.  We
 182  * stay in this mode for 5 minutes.
 183  *
 184  * Duane W., Sept 16, 1996
 185  */
 186 static void
 187 checkFailureRatio(err_type etype, hier_code hcode)
 188 {
 189     // Can be set at compile time with -D compiler flag
 190 #ifndef FAILURE_MODE_TIME
 191 #define FAILURE_MODE_TIME 300
 192 #endif
 193
 194     if (hcode == HIER_NONE)
 195         return;
 196
 197     // don't bother when ICP is disabled.
 198     if (Config.Port.icp <= 0)
 199         return;
 200
 201     static double magic_factor = 100.0;
 202     double n_good;
 203     double n_bad;
 204
 205     n_good = magic_factor / (1.0 + request_failure_ratio);
 206
 207     n_bad = magic_factor - n_good;
 208
 209     switch (etype) {
 210
 211     case ERR_DNS_FAIL:
 212
 213     case ERR_CONNECT_FAIL:
 214     case ERR_SECURE_CONNECT_FAIL:
 215
 216     case ERR_READ_ERROR:
 217         ++n_bad;
 218         break;
 219
 220     default:
 221         ++n_good;
 222     }
 223
 224     request_failure_ratio = n_bad / n_good;
 225
 226     if (hit_only_mode_until > squid_curtime)
 227         return;
 228
 229     if (request_failure_ratio < 1.0)
 230         return;
 231
 232     debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
 233            std::setprecision(3) << request_failure_ratio);
 234
 235     debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
 236            FAILURE_MODE_TIME / 60 << " minutes...");
 237
 238     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 239
 240     request_failure_ratio = 0.8;        /* reset to something less than 1.0 */
 241 }
 242
 243 ClientHttpRequest::~ClientHttpRequest()
 244 {
 245     debugs(33, 3, "httpRequestFree: " << uri);
 246     PROF_start(httpRequestFree);
 247
 248     // Even though freeResources() below may destroy the request,
 249     // we no longer set request->body_pipe to NULL here
 250     // because we did not initiate that pipe (ConnStateData did)
 251
 252     /* the ICP check here was erroneous
 253      * - StoreEntry::releaseRequest was always called if entry was valid
 254      */
 255     assert(logType < LOG_TYPE_MAX);
 256
 257     logRequest();
 258
 259     loggingEntry(NULL);
 260
 261     if (request)
 262         checkFailureRatio(request->errType, al->hier.code);
 263
 264     freeResources();
 265
 266 #if USE_ADAPTATION
 267     announceInitiatorAbort(virginHeadSource);
 268
 269     if (adaptedBodySource != NULL)
 270         stopConsumingFrom(adaptedBodySource);
 271 #endif
 272
 273     if (calloutContext)
 274         delete calloutContext;
 275
 276     clientConnection = NULL;
 277
 278     if (conn_)
 279         cbdataReferenceDone(conn_);
 280
 281     /* moving to the next connection is handled by the context free */
 282     dlinkDelete(&active, &ClientActiveRequests);
 283
 284     PROF_stop(httpRequestFree);
 285 }
 286
 287 /**
 288  * Create a request and kick it off
 289  *
 290  * \retval 0     success
 291  * \retval -1    failure
 292  *
 293  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 294  * determined by the user
 295  */
 296 int
 297 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 298                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 299                    char *tailbuf, size_t taillen)
 300 {
 301     size_t url_sz;
 302     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 303     HttpRequest *request;
 304     StoreIOBuffer tempBuffer;
 305     if (http->al != NULL)
 306         http->al->cache.start_time = current_time;
 307     /* this is only used to adjust the connection offset in client_side.c */
 308     http->req_sz = 0;
 309     tempBuffer.length = taillen;
 310     tempBuffer.data = tailbuf;
 311     /* client stream setup */
 312     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 313                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 314                      streamdetach, streamdata, tempBuffer);
 315     /* make it visible in the 'current acctive requests list' */
 316     /* Set flags */
 317     /* internal requests only makes sense in an
 318      * accelerator today. TODO: accept flags ? */
 319     http->flags.accel = true;
 320     /* allow size for url rewriting */
 321     url_sz = strlen(url) + Config.appendDomainLen + 5;
 322     http->uri = (char *)xcalloc(url_sz, 1);
 323     strcpy(http->uri, url);
 324
 325     if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
 326         debugs(85, 5, "Invalid URL: " << http->uri);
 327         return -1;
 328     }
 329
 330     /*
 331      * now update the headers in request with our supplied headers. urlParse
 332      * should return a blank header set, but we use Update to be sure of
 333      * correctness.
 334      */
 335     if (header)
 336         request->header.update(header, NULL);
 337
 338     http->log_uri = xstrdup(urlCanonicalClean(request));
 339
 340     /* http struct now ready */
 341
 342     /*
 343      * build new header list *? TODO
 344      */
 345     request->flags.accelerated = http->flags.accel;
 346
 347     request->flags.internalClient = true;
 348
 349     /* this is an internally created
 350      * request, not subject to acceleration
 351      * target overrides */
 352     /*
 353      * FIXME? Do we want to detect and handle internal requests of internal
 354      * objects ?
 355      */
 356
 357     /* Internally created requests cannot have bodies today */
 358     request->content_length = 0;
 359
 360     request->client_addr.setNoAddr();
 361
 362 #if FOLLOW_X_FORWARDED_FOR
 363     request->indirect_client_addr.setNoAddr();
 364 #endif /* FOLLOW_X_FORWARDED_FOR */
 365
 366     request->my_addr.setNoAddr();       /* undefined for internal requests */
 367
 368     request->my_addr.port(0);
 369
 370     /* Our version is HTTP/1.1 */
 371     request->http_ver = Http::ProtocolVersion(1,1);
 372
 373     http->request = request;
 374     HTTPMSGLOCK(http->request);
 375
 376     /* optional - skip the access check ? */
 377     http->calloutContext = new ClientRequestContext(http);
 378
 379     http->calloutContext->http_access_done = false;
 380
 381     http->calloutContext->redirect_done = true;
 382
 383     http->calloutContext->no_cache_done = true;
 384
 385     http->doCallouts();
 386
 387     return 0;
 388 }
 389
 390 bool
 391 ClientRequestContext::httpStateIsValid()
 392 {
 393     ClientHttpRequest *http_ = http;
 394
 395     if (cbdataReferenceValid(http_))
 396         return true;
 397
 398     http = NULL;
 399
 400     cbdataReferenceDone(http_);
 401
 402     return false;
 403 }
 404
 405 #if FOLLOW_X_FORWARDED_FOR
 406 /**
 407  * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
 408  * against the followXFF ACL, or cleans up and passes control to
 409  * clientAccessCheck().
 410  *
 411  * The trust model here is a little ambiguous. So to clarify the logic:
 412  * - we may always use the direct client address as the client IP.
 413  * - these trust tests merey tell whether we trust given IP enough to believe the
 414  *   IP string which it appended to the X-Forwarded-For: header.
 415  * - if at any point we don't trust what an IP adds we stop looking.
 416  * - at that point the current contents of indirect_client_addr are the value set
 417  *   by the last previously trusted IP.
 418  * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
 419  */
 420 static void
 421 clientFollowXForwardedForCheck(allow_t answer, void *data)
 422 {
 423     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 424
 425     if (!calloutContext->httpStateIsValid())
 426         return;
 427
 428     ClientHttpRequest *http = calloutContext->http;
 429     HttpRequest *request = http->request;
 430
 431     /*
 432      * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
 433      * called as a result of ACL checks, or -1 if we are called when
 434      * there's nothing left to do.
 435      */
 436     if (answer == ACCESS_ALLOWED &&
 437             request->x_forwarded_for_iterator.size () != 0) {
 438
 439         /*
 440          * Remove the last comma-delimited element from the
 441          * x_forwarded_for_iterator and use it to repeat the cycle.
 442          */
 443         const char *p;
 444         const char *asciiaddr;
 445         int l;
 446         Ip::Address addr;
 447         p = request->x_forwarded_for_iterator.termedBuf();
 448         l = request->x_forwarded_for_iterator.size();
 449
 450         /*
 451         * XXX x_forwarded_for_iterator should really be a list of
 452         * IP addresses, but it's a String instead.  We have to
 453         * walk backwards through the String, biting off the last
 454         * comma-delimited part each time.  As long as the data is in
 455         * a String, we should probably implement and use a variant of
 456         * strListGetItem() that walks backwards instead of forwards
 457         * through a comma-separated list.  But we don't even do that;
 458         * we just do the work in-line here.
 459         */
 460         /* skip trailing space and commas */
 461         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 462             --l;
 463         request->x_forwarded_for_iterator.cut(l);
 464         /* look for start of last item in list */
 465         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 466             --l;
 467         asciiaddr = p+l;
 468         if ((addr = asciiaddr)) {
 469             request->indirect_client_addr = addr;
 470             request->x_forwarded_for_iterator.cut(l);
 471             calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 472             if (!Config.onoff.acl_uses_indirect_client) {
 473                 /* override the default src_addr tested if we have to go deeper than one level into XFF */
 474                 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
 475             }
 476             calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
 477             return;
 478         }
 479     } /*if (answer == ACCESS_ALLOWED &&
 480         request->x_forwarded_for_iterator.size () != 0)*/
 481
 482     /* clean up, and pass control to clientAccessCheck */
 483     if (Config.onoff.log_uses_indirect_client) {
 484         /*
 485         * Ensure that the access log shows the indirect client
 486         * instead of the direct client.
 487         */
 488         ConnStateData *conn = http->getConn();
 489         conn->log_addr = request->indirect_client_addr;
 490         http->al->cache.caddr = conn->log_addr;
 491     }
 492     request->x_forwarded_for_iterator.clean();
 493     request->flags.done_follow_x_forwarded_for = true;
 494
 495     if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
 496         debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
 497     }
 498
 499     /* process actual access ACL as normal. */
 500     calloutContext->clientAccessCheck();
 501 }
 502 #endif /* FOLLOW_X_FORWARDED_FOR */
 503
 504 static void
 505 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const DnsLookupDetails &dns, void *data)
 506 {
 507     ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
 508     c->hostHeaderIpVerify(ia, dns);
 509 }
 510
 511 void
 512 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns)
 513 {
 514     Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
 515
 516     // note the DNS details for the transaction stats.
 517     http->request->recordLookup(dns);
 518
 519     if (ia != NULL && ia->count > 0) {
 520         // Is the NAT destination IP in DNS?
 521         for (int i = 0; i < ia->count; ++i) {
 522             if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
 523                 debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
 524                 http->request->flags.hostVerified = true;
 525                 http->doCallouts();
 526                 return;
 527             }
 528             debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
 529         }
 530     }
 531     debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
 532     hostHeaderVerifyFailed("local IP", "any domain IP");
 533 }
 534
 535 void
 536 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
 537 {
 538     // IP address validation for Host: failed. Admin wants to ignore them.
 539     // NP: we do not yet handle CONNECT tunnels well, so ignore for them
 540     if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
 541         debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
 542                " (" << A << " does not match " << B << ") on URL: " << urlCanonical(http->request));
 543
 544         // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
 545         // The problems here are about WRITE for new cache content, which means flags.cachable
 546         http->request->flags.cachable = false; // MUST NOT cache (for now)
 547         // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
 548         http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
 549         // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
 550         http->doCallouts();
 551         return;
 552     }
 553
 554     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
 555            http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
 556     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << http->request->header.getStr(HDR_USER_AGENT));
 557     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << urlCanonical(http->request));
 558
 559     // IP address validation for Host: failed. reject the connection.
 560     clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 561     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 562     assert (repContext);
 563     repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
 564                                 http->request->method, NULL,
 565                                 http->getConn()->clientConnection->remote,
 566                                 http->request,
 567                                 NULL,
 568 #if USE_AUTH
 569                                 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 570                                 http->getConn()->getAuth() : http->request->auth_user_request);
 571 #else
 572                                 NULL);
 573 #endif
 574     node = (clientStreamNode *)http->client_stream.tail->data;
 575     clientStreamRead(node, http, node->readBuffer);
 576 }
 577
 578 void
 579 ClientRequestContext::hostHeaderVerify()
 580 {
 581     // Require a Host: header.
 582     const char *host = http->request->header.getStr(HDR_HOST);
 583
 584     if (!host) {
 585         // TODO: dump out the HTTP/1.1 error about missing host header.
 586         // otherwise this is fine, can't forge a header value when its not even set.
 587         debugs(85, 3, HERE << "validate skipped with no Host: header present.");
 588         http->doCallouts();
 589         return;
 590     }
 591
 592     if (http->request->flags.internal) {
 593         // TODO: kill this when URL handling allows partial URLs out of accel mode
 594         //       and we no longer screw with the URL just to add our internal host there
 595         debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
 596         http->doCallouts();
 597         return;
 598     }
 599
 600     // Locate if there is a port attached, strip ready for IP lookup
 601     char *portStr = NULL;
 602     char *hostB = xstrdup(host);
 603     host = hostB;
 604     if (host[0] == '[') {
 605         // IPv6 literal.
 606         portStr = strchr(hostB, ']');
 607         if (portStr && *(++portStr) != ':') {
 608             portStr = NULL;
 609         }
 610     } else {
 611         // Domain or IPv4 literal with port
 612         portStr = strrchr(hostB, ':');
 613     }
 614
 615     uint16_t port = 0;
 616     if (portStr) {
 617         *portStr = '\0'; // strip the ':'
 618         if (*(++portStr) != '\0') {
 619             char *end = NULL;
 620             int64_t ret = strtoll(portStr, &end, 10);
 621             if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
 622                 // invalid port details. Replace the ':'
 623                 *(--portStr) = ':';
 624                 portStr = NULL;
 625             } else
 626                 port = (ret & 0xFFFF);
 627         }
 628     }
 629
 630     debugs(85, 3, HERE << "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
 631     if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
 632         // verify the Host: port (if any) matches the apparent destination
 633         if (portStr && port != http->getConn()->clientConnection->local.port()) {
 634             debugs(85, 3, HERE << "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
 635                    " matches Host: port " << port << " (" << portStr << ")");
 636             hostHeaderVerifyFailed("intercepted port", portStr);
 637         } else {
 638             // XXX: match the scheme default port against the apparent destination
 639
 640             // verify the destination DNS is one of the Host: headers IPs
 641             ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
 642         }
 643     } else if (!Config.onoff.hostStrictVerify) {
 644         debugs(85, 3, HERE << "validate skipped.");
 645         http->doCallouts();
 646     } else if (strlen(host) != strlen(http->request->GetHost())) {
 647         // Verify forward-proxy requested URL domain matches the Host: header
 648         debugs(85, 3, HERE << "FAIL on validate URL domain length " << http->request->GetHost() << " matches Host: " << host);
 649         hostHeaderVerifyFailed(host, http->request->GetHost());
 650     } else if (matchDomainName(host, http->request->GetHost()) != 0) {
 651         // Verify forward-proxy requested URL domain matches the Host: header
 652         debugs(85, 3, HERE << "FAIL on validate URL domain " << http->request->GetHost() << " matches Host: " << host);
 653         hostHeaderVerifyFailed(host, http->request->GetHost());
 654     } else if (portStr && port != http->request->port) {
 655         // Verify forward-proxy requested URL domain matches the Host: header
 656         debugs(85, 3, HERE << "FAIL on validate URL port " << http->request->port << " matches Host: port " << portStr);
 657         hostHeaderVerifyFailed("URL port", portStr);
 658     } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->port != urlDefaultPort(http->request->url.getScheme())) {
 659         // Verify forward-proxy requested URL domain matches the Host: header
 660         // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
 661         debugs(85, 3, "FAIL on validate URL port " << http->request->port << " matches Host: default port " << urlDefaultPort(http->request->url.getScheme()));
 662         hostHeaderVerifyFailed("URL port", "default port");
 663     } else {
 664         // Okay no problem.
 665         debugs(85, 3, HERE << "validate passed.");
 666         http->request->flags.hostVerified = true;
 667         http->doCallouts();
 668     }
 669     safe_free(hostB);
 670 }
 671
 672 /* This is the entry point for external users of the client_side routines */
 673 void
 674 ClientRequestContext::clientAccessCheck()
 675 {
 676 #if FOLLOW_X_FORWARDED_FOR
 677     if (!http->request->flags.doneFollowXff() &&
 678             Config.accessList.followXFF &&
 679             http->request->header.has(HDR_X_FORWARDED_FOR)) {
 680
 681         /* we always trust the direct client address for actual use */
 682         http->request->indirect_client_addr = http->request->client_addr;
 683         http->request->indirect_client_addr.port(0);
 684
 685         /* setup the XFF iterator for processing */
 686         http->request->x_forwarded_for_iterator = http->request->header.getList(HDR_X_FORWARDED_FOR);
 687
 688         /* begin by checking to see if we trust direct client enough to walk XFF */
 689         acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 690         acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
 691         return;
 692     }
 693 #endif
 694
 695     if (Config.accessList.http) {
 696         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 697         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 698     } else {
 699         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 700         clientAccessCheckDone(ACCESS_DENIED);
 701     }
 702 }
 703
 704 /**
 705  * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
 706  * The default here is to allow all. Since the earlier http_access should do a default deny all.
 707  * This check is just for a last-minute denial based on adapted request headers.
 708  */
 709 void
 710 ClientRequestContext::clientAccessCheck2()
 711 {
 712     if (Config.accessList.adapted_http) {
 713         acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
 714         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 715     } else {
 716         debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
 717         clientAccessCheckDone(ACCESS_ALLOWED);
 718     }
 719 }
 720
 721 void
 722 clientAccessCheckDoneWrapper(allow_t answer, void *data)
 723 {
 724     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 725
 726     if (!calloutContext->httpStateIsValid())
 727         return;
 728
 729     calloutContext->clientAccessCheckDone(answer);
 730 }
 731
 732 void
 733 ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
 734 {
 735     acl_checklist = NULL;
 736     err_type page_id;
 737     Http::StatusCode status;
 738     debugs(85, 2, "The request " << http->request->method << ' ' <<
 739            http->uri << " is " << answer <<
 740            "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
 741
 742 #if USE_AUTH
 743     char const *proxy_auth_msg = "<null>";
 744     if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
 745         proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
 746     else if (http->request->auth_user_request != NULL)
 747         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 748 #endif
 749
 750     if (answer != ACCESS_ALLOWED) {
 751         // auth has a grace period where credentials can be expired but okay not to challenge.
 752
 753         /* Send an auth challenge or error */
 754         // XXX: do we still need aclIsProxyAuth() ?
 755         bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
 756         debugs(85, 5, "Access Denied: " << http->uri);
 757         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 758 #if USE_AUTH
 759         if (auth_challenge)
 760             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 761 #endif
 762
 763         /*
 764          * NOTE: get page_id here, based on AclMatchedName because if
 765          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 766          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 767          * <pribeiro@isel.pt>
 768          */
 769         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
 770
 771         http->logType = LOG_TCP_DENIED;
 772
 773         if (auth_challenge) {
 774 #if USE_AUTH
 775             if (http->request->flags.sslBumped) {
 776                 /*SSL Bumped request, authentication is not possible*/
 777                 status = Http::scForbidden;
 778             } else if (!http->flags.accel) {
 779                 /* Proxy authorisation needed */
 780                 status = Http::scProxyAuthenticationRequired;
 781             } else {
 782                 /* WWW authorisation needed */
 783                 status = Http::scUnauthorized;
 784             }
 785 #else
 786             // need auth, but not possible to do.
 787             status = Http::scForbidden;
 788 #endif
 789             if (page_id == ERR_NONE)
 790                 page_id = ERR_CACHE_ACCESS_DENIED;
 791         } else {
 792             status = Http::scForbidden;
 793
 794             if (page_id == ERR_NONE)
 795                 page_id = ERR_ACCESS_DENIED;
 796         }
 797
 798         Ip::Address tmpnoaddr;
 799         tmpnoaddr.setNoAddr();
 800         error = clientBuildError(page_id, status,
 801                                  NULL,
 802                                  http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
 803                                  http->request
 804                                 );
 805
 806 #if USE_AUTH
 807         error->auth_user_request =
 808             http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 809             http->getConn()->getAuth() : http->request->auth_user_request;
 810 #endif
 811
 812         readNextRequest = true;
 813     }
 814
 815     /* ACCESS_ALLOWED continues here ... */
 816     safe_free(http->uri);
 817
 818     http->uri = xstrdup(urlCanonical(http->request));
 819
 820     http->doCallouts();
 821 }
 822
 823 #if USE_ADAPTATION
 824 void
 825 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 826 {
 827     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 828
 829 #if ICAP_CLIENT
 830     Adaptation::Icap::History::Pointer ih = request->icapHistory();
 831     if (ih != NULL) {
 832         if (getConn() != NULL && getConn()->clientConnection != NULL) {
 833             ih->rfc931 = getConn()->clientConnection->rfc931;
 834 #if USE_OPENSSL
 835             if (getConn()->clientConnection->isOpen()) {
 836                 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl);
 837             }
 838 #endif
 839         }
 840         ih->log_uri = log_uri;
 841         ih->req_sz = req_sz;
 842     }
 843 #endif
 844
 845     if (!g) {
 846         debugs(85,3, HERE << "no adaptation needed");
 847         doCallouts();
 848         return;
 849     }
 850
 851     startAdaptation(g);
 852 }
 853
 854 #endif
 855
 856 static void
 857 clientRedirectAccessCheckDone(allow_t answer, void *data)
 858 {
 859     ClientRequestContext *context = (ClientRequestContext *)data;
 860     ClientHttpRequest *http = context->http;
 861     context->acl_checklist = NULL;
 862
 863     if (answer == ACCESS_ALLOWED)
 864         redirectStart(http, clientRedirectDoneWrapper, context);
 865     else {
 866         HelperReply nilReply;
 867         nilReply.result = HelperReply::Error;
 868         context->clientRedirectDone(nilReply);
 869     }
 870 }
 871
 872 void
 873 ClientRequestContext::clientRedirectStart()
 874 {
 875     debugs(33, 5, HERE << "'" << http->uri << "'");
 876     (void)SyncNotes(*http->al, *http->request);
 877     if (Config.accessList.redirector) {
 878         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 879         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 880     } else
 881         redirectStart(http, clientRedirectDoneWrapper, this);
 882 }
 883
 884 /**
 885  * This methods handles Access checks result of StoreId access list.
 886  * Will handle as "ERR" (no change) in a case Access is not allowed.
 887  */
 888 static void
 889 clientStoreIdAccessCheckDone(allow_t answer, void *data)
 890 {
 891     ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
 892     ClientHttpRequest *http = context->http;
 893     context->acl_checklist = NULL;
 894
 895     if (answer == ACCESS_ALLOWED)
 896         storeIdStart(http, clientStoreIdDoneWrapper, context);
 897     else {
 898         debugs(85, 3, "access denied expected ERR reply handling: " << answer);
 899         HelperReply nilReply;
 900         nilReply.result = HelperReply::Error;
 901         context->clientStoreIdDone(nilReply);
 902     }
 903 }
 904
 905 /**
 906  * Start locating an alternative storeage ID string (if any) from admin
 907  * configured helper program. This is an asynchronous operation terminating in
 908  * ClientRequestContext::clientStoreIdDone() when completed.
 909  */
 910 void
 911 ClientRequestContext::clientStoreIdStart()
 912 {
 913     debugs(33, 5,"'" << http->uri << "'");
 914
 915     if (Config.accessList.store_id) {
 916         acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
 917         acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
 918     } else
 919         storeIdStart(http, clientStoreIdDoneWrapper, this);
 920 }
 921
 922 static int
 923 clientHierarchical(ClientHttpRequest * http)
 924 {
 925     HttpRequest *request = http->request;
 926     HttpRequestMethod method = request->method;
 927
 928     // intercepted requests MUST NOT (yet) be sent to peers unless verified
 929     if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
 930         return 0;
 931
 932     /*
 933      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 934      * neighbors support private keys
 935      */
 936
 937     if (request->flags.ims && !neighbors_do_private_keys)
 938         return 0;
 939
 940     /*
 941      * This is incorrect: authenticating requests can be sent via a hierarchy
 942      * (they can even be cached if the correct headers are set on the reply)
 943      */
 944     if (request->flags.auth)
 945         return 0;
 946
 947     if (method == Http::METHOD_TRACE)
 948         return 1;
 949
 950     if (method != Http::METHOD_GET)
 951         return 0;
 952
 953     if (request->flags.loopDetected)
 954         return 0;
 955
 956     if (request->url.getScheme() == AnyP::PROTO_HTTP)
 957         return method.respMaybeCacheable();
 958
 959     if (request->url.getScheme() == AnyP::PROTO_GOPHER)
 960         return gopherCachable(request);
 961
 962     if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
 963         return 0;
 964
 965     return 1;
 966 }
 967
 968 static void
 969 clientCheckPinning(ClientHttpRequest * http)
 970 {
 971     HttpRequest *request = http->request;
 972     HttpHeader *req_hdr = &request->header;
 973     ConnStateData *http_conn = http->getConn();
 974
 975     /* Internal requests such as those from ESI includes may be without
 976      * a client connection
 977      */
 978     if (!http_conn)
 979         return;
 980
 981     request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
 982     if (!request->flags.connectionAuthDisabled) {
 983         if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
 984             if (http_conn->pinning.auth) {
 985                 request->flags.connectionAuth = true;
 986                 request->flags.auth = true;
 987             } else {
 988                 request->flags.connectionProxyAuth = true;
 989             }
 990             // These should already be linked correctly.
 991             assert(request->clientConnectionManager == http_conn);
 992         }
 993     }
 994
 995     /* check if connection auth is used, and flag as candidate for pinning
 996      * in such case.
 997      * Note: we may need to set flags.connectionAuth even if the connection
 998      * is already pinned if it was pinned earlier due to proxy auth
 999      */
1000     if (!request->flags.connectionAuth) {
1001         if (req_hdr->has(HDR_AUTHORIZATION) || req_hdr->has(HDR_PROXY_AUTHORIZATION)) {
1002             HttpHeaderPos pos = HttpHeaderInitPos;
1003             HttpHeaderEntry *e;
1004             int may_pin = 0;
1005             while ((e = req_hdr->getEntry(&pos))) {
1006                 if (e->id == HDR_AUTHORIZATION || e->id == HDR_PROXY_AUTHORIZATION) {
1007                     const char *value = e->value.rawBuf();
1008                     if (strncasecmp(value, "NTLM ", 5) == 0
1009                             ||
1010                             strncasecmp(value, "Negotiate ", 10) == 0
1011                             ||
1012                             strncasecmp(value, "Kerberos ", 9) == 0) {
1013                         if (e->id == HDR_AUTHORIZATION) {
1014                             request->flags.connectionAuth = true;
1015                             may_pin = 1;
1016                         } else {
1017                             request->flags.connectionProxyAuth = true;
1018                             may_pin = 1;
1019                         }
1020                     }
1021                 }
1022             }
1023             if (may_pin && !request->pinnedConnection()) {
1024                 // These should already be linked correctly. Just need the ServerConnection to pinn.
1025                 assert(request->clientConnectionManager == http_conn);
1026             }
1027         }
1028     }
1029 }
1030
1031 static void
1032 clientInterpretRequestHeaders(ClientHttpRequest * http)
1033 {
1034     HttpRequest *request = http->request;
1035     HttpHeader *req_hdr = &request->header;
1036     bool no_cache = false;
1037     const char *str;
1038
1039     request->imslen = -1;
1040     request->ims = req_hdr->getTime(HDR_IF_MODIFIED_SINCE);
1041
1042     if (request->ims > 0)
1043         request->flags.ims = true;
1044
1045     if (!request->flags.ignoreCc) {
1046         if (request->cache_control) {
1047             if (request->cache_control->hasNoCache())
1048                 no_cache=true;
1049
1050             // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1051         } else if (req_hdr->has(HDR_PRAGMA))
1052             no_cache = req_hdr->hasListMember(HDR_PRAGMA,"no-cache",',');
1053
1054         /*
1055         * Work around for supporting the Reload button in IE browsers when Squid
1056         * is used as an accelerator or transparent proxy, by turning accelerated
1057         * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1058         * actually only fixed in SP1, but we can't tell whether we are talking to
1059         * SP1 or not so all 5.5 versions are treated 'normally').
1060         */
1061         if (Config.onoff.ie_refresh) {
1062             if (http->flags.accel && request->flags.ims) {
1063                 if ((str = req_hdr->getStr(HDR_USER_AGENT))) {
1064                     if (strstr(str, "MSIE 5.01") != NULL)
1065                         no_cache=true;
1066                     else if (strstr(str, "MSIE 5.0") != NULL)
1067                         no_cache=true;
1068                     else if (strstr(str, "MSIE 4.") != NULL)
1069                         no_cache=true;
1070                     else if (strstr(str, "MSIE 3.") != NULL)
1071                         no_cache=true;
1072                 }
1073             }
1074         }
1075     }
1076
1077     if (request->method == Http::METHOD_OTHER) {
1078         no_cache=true;
1079     }
1080
1081     if (no_cache) {
1082 #if USE_HTTP_VIOLATIONS
1083
1084         if (Config.onoff.reload_into_ims)
1085             request->flags.nocacheHack = true;
1086         else if (refresh_nocache_hack)
1087             request->flags.nocacheHack = true;
1088         else
1089 #endif
1090
1091             request->flags.noCache = true;
1092     }
1093
1094     /* ignore range header in non-GETs or non-HEADs */
1095     if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1096         // XXX: initialize if we got here without HttpRequest::parseHeader()
1097         if (!request->range)
1098             request->range = req_hdr->getRange();
1099
1100         if (request->range) {
1101             request->flags.isRanged = true;
1102             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1103             /* XXX: This is suboptimal. We should give the stream the range set,
1104              * and thereby let the top of the stream set the offset when the
1105              * size becomes known. As it is, we will end up requesting from 0
1106              * for evey -X range specification.
1107              * RBC - this may be somewhat wrong. We should probably set the range
1108              * iter up at this point.
1109              */
1110             node->readBuffer.offset = request->range->lowestOffset(0);
1111             http->range_iter.pos = request->range->begin();
1112             http->range_iter.end = request->range->end();
1113             http->range_iter.valid = true;
1114         }
1115     }
1116
1117     /* Only HEAD and GET requests permit a Range or Request-Range header.
1118      * If these headers appear on any other type of request, delete them now.
1119      */
1120     else {
1121         req_hdr->delById(HDR_RANGE);
1122         req_hdr->delById(HDR_REQUEST_RANGE);
1123         request->ignoreRange("neither HEAD nor GET");
1124     }
1125
1126     if (req_hdr->has(HDR_AUTHORIZATION))
1127         request->flags.auth = true;
1128
1129     clientCheckPinning(http);
1130
1131     if (request->login[0] != '\0')
1132         request->flags.auth = true;
1133
1134     if (req_hdr->has(HDR_VIA)) {
1135         String s = req_hdr->getList(HDR_VIA);
1136         /*
1137          * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1138          * Note ThisCache2 has a space prepended to the hostname so we don't
1139          * accidentally match super-domains.
1140          */
1141
1142         if (strListIsSubstr(&s, ThisCache2, ',')) {
1143             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1144                      request, (ObjPackMethod) & httpRequestPack);
1145             request->flags.loopDetected = true;
1146         }
1147
1148 #if USE_FORW_VIA_DB
1149         fvdbCountVia(s.termedBuf());
1150
1151 #endif
1152
1153         s.clean();
1154     }
1155
1156 #if USE_FORW_VIA_DB
1157
1158     if (req_hdr->has(HDR_X_FORWARDED_FOR)) {
1159         String s = req_hdr->getList(HDR_X_FORWARDED_FOR);
1160         fvdbCountForw(s.termedBuf());
1161         s.clean();
1162     }
1163
1164 #endif
1165
1166     request->flags.cachable = http->request->maybeCacheable();
1167
1168     if (clientHierarchical(http))
1169         request->flags.hierarchical = true;
1170
1171     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1172            (request->flags.noCache ? "SET" : "NOT SET"));
1173     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1174            (request->flags.cachable ? "SET" : "NOT SET"));
1175     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1176            (request->flags.hierarchical ? "SET" : "NOT SET"));
1177
1178 }
1179
1180 void
1181 clientRedirectDoneWrapper(void *data, const HelperReply &result)
1182 {
1183     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1184
1185     if (!calloutContext->httpStateIsValid())
1186         return;
1187
1188     calloutContext->clientRedirectDone(result);
1189 }
1190
1191 void
1192 clientStoreIdDoneWrapper(void *data, const HelperReply &result)
1193 {
1194     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1195
1196     if (!calloutContext->httpStateIsValid())
1197         return;
1198
1199     calloutContext->clientStoreIdDone(result);
1200 }
1201
1202 void
1203 ClientRequestContext::clientRedirectDone(const HelperReply &reply)
1204 {
1205     HttpRequest *old_request = http->request;
1206     debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1207     assert(redirect_state == REDIRECT_PENDING);
1208     redirect_state = REDIRECT_DONE;
1209
1210     // Put helper response Notes into the transaction state record (ALE) eventually
1211     // do it early to ensure that no matter what the outcome the notes are present.
1212     if (http->al != NULL)
1213         (void)SyncNotes(*http->al, *old_request);
1214
1215     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1216
1217     switch (reply.result) {
1218     case HelperReply::Unknown:
1219     case HelperReply::TT:
1220         // Handler in redirect.cc should have already mapped Unknown
1221         // IF it contained valid entry for the old URL-rewrite helper protocol
1222         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1223         break;
1224
1225     case HelperReply::BrokenHelper:
1226         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply << ", attempt #" << (redirect_fail_count+1) << " of 2");
1227         if (redirect_fail_count < 2) { // XXX: make this configurable ?
1228             ++redirect_fail_count;
1229             // reset state flag to try redirector again from scratch.
1230             redirect_done = false;
1231         }
1232         break;
1233
1234     case HelperReply::Error:
1235         // no change to be done.
1236         break;
1237
1238     case HelperReply::Okay: {
1239         // #1: redirect with a specific status code    OK status=NNN url="..."
1240         // #2: redirect with a default status code     OK url="..."
1241         // #3: re-write the URL                        OK rewrite-url="..."
1242
1243         const char *statusNote = reply.notes.findFirst("status");
1244         const char *urlNote = reply.notes.findFirst("url");
1245
1246         if (urlNote != NULL) {
1247             // HTTP protocol redirect to be done.
1248
1249             // TODO: change default redirect status for appropriate requests
1250             // Squid defaults to 302 status for now for better compatibility with old clients.
1251             // HTTP/1.0 client should get 302 (Http::scFound)
1252             // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1253             // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1254             Http::StatusCode status = Http::scFound;
1255             if (statusNote != NULL) {
1256                 const char * result = statusNote;
1257                 status = static_cast<Http::StatusCode>(atoi(result));
1258             }
1259
1260             if (status == Http::scMovedPermanently
1261                     || status == Http::scFound
1262                     || status == Http::scSeeOther
1263                     || status == Http::scPermanentRedirect
1264                     || status == Http::scTemporaryRedirect) {
1265                 http->redirect.status = status;
1266                 http->redirect.location = xstrdup(urlNote);
1267                 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1268             } else {
1269                 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1270             }
1271         } else {
1272             // URL-rewrite wanted. Ew.
1273             urlNote = reply.notes.findFirst("rewrite-url");
1274
1275             // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1276             if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1277                 // XXX: validate the URL properly *without* generating a whole new request object right here.
1278                 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1279                 HttpRequest *new_request = old_request->clone();
1280                 if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1281                     debugs(61,2, HERE << "URL-rewriter diverts URL from " << urlCanonical(old_request) << " to " << urlCanonical(new_request));
1282
1283                     // update the new request to flag the re-writing was done on it
1284                     new_request->flags.redirected = true;
1285
1286                     // unlink bodypipe from the old request. Not needed there any longer.
1287                     if (old_request->body_pipe != NULL) {
1288                         old_request->body_pipe = NULL;
1289                         debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1290                                " from request " << old_request << " to " << new_request);
1291                     }
1292
1293                     // update the current working ClientHttpRequest fields
1294                     safe_free(http->uri);
1295                     http->uri = xstrdup(urlCanonical(new_request));
1296                     HTTPMSGUNLOCK(old_request);
1297                     http->request = new_request;
1298                     HTTPMSGLOCK(http->request);
1299                 } else {
1300                     debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1301                            old_request->method << " " << urlNote << " " << old_request->http_ver);
1302                     delete new_request;
1303                 }
1304             }
1305         }
1306     }
1307     break;
1308     }
1309
1310     /* FIXME PIPELINE: This is innacurate during pipelining */
1311
1312     if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1313         fd_note(http->getConn()->clientConnection->fd, http->uri);
1314
1315     assert(http->uri);
1316
1317     http->doCallouts();
1318 }
1319
1320 /**
1321  * This method handles the different replies from StoreID helper.
1322  */
1323 void
1324 ClientRequestContext::clientStoreIdDone(const HelperReply &reply)
1325 {
1326     HttpRequest *old_request = http->request;
1327     debugs(85, 5, "'" << http->uri << "' result=" << reply);
1328     assert(store_id_state == REDIRECT_PENDING);
1329     store_id_state = REDIRECT_DONE;
1330
1331     // Put helper response Notes into the transaction state record (ALE) eventually
1332     // do it early to ensure that no matter what the outcome the notes are present.
1333     if (http->al != NULL)
1334         (void)SyncNotes(*http->al, *old_request);
1335
1336     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1337
1338     switch (reply.result) {
1339     case HelperReply::Unknown:
1340     case HelperReply::TT:
1341         // Handler in redirect.cc should have already mapped Unknown
1342         // IF it contained valid entry for the old helper protocol
1343         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1344         break;
1345
1346     case HelperReply::BrokenHelper:
1347         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2");
1348         if (store_id_fail_count < 2) { // XXX: make this configurable ?
1349             ++store_id_fail_count;
1350             // reset state flag to try StoreID again from scratch.
1351             store_id_done = false;
1352         }
1353         break;
1354
1355     case HelperReply::Error:
1356         // no change to be done.
1357         break;
1358
1359     case HelperReply::Okay: {
1360         const char *urlNote = reply.notes.findFirst("store-id");
1361
1362         // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1363         if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1364             // Debug section required for some very specific cases.
1365             debugs(85, 9, "Setting storeID with: " << urlNote );
1366             http->request->store_id = urlNote;
1367             http->store_id = urlNote;
1368         }
1369     }
1370     break;
1371     }
1372
1373     http->doCallouts();
1374 }
1375
1376 /** Test cache allow/deny configuration
1377  *  Sets flags.cachable=1 if caching is not denied.
1378  */
1379 void
1380 ClientRequestContext::checkNoCache()
1381 {
1382     if (Config.accessList.noCache) {
1383         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1384         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1385     } else {
1386         /* unless otherwise specified, we try to cache. */
1387         checkNoCacheDone(ACCESS_ALLOWED);
1388     }
1389 }
1390
1391 static void
1392 checkNoCacheDoneWrapper(allow_t answer, void *data)
1393 {
1394     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1395
1396     if (!calloutContext->httpStateIsValid())
1397         return;
1398
1399     calloutContext->checkNoCacheDone(answer);
1400 }
1401
1402 void
1403 ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1404 {
1405     acl_checklist = NULL;
1406     http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1407     http->doCallouts();
1408 }
1409
1410 #if USE_OPENSSL
1411 bool
1412 ClientRequestContext::sslBumpAccessCheck()
1413 {
1414     // If SSL connection tunneling or bumping decision has been made, obey it.
1415     const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1416     if (bumpMode != Ssl::bumpEnd) {
1417         debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1418                "), " << "ignoring ssl_bump for " << http->getConn());
1419         http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1420         return false;
1421     }
1422
1423     // If we have not decided yet, decide whether to bump now.
1424
1425     // Bumping here can only start with a CONNECT request on a bumping port
1426     // (bumping of intercepted SSL conns is decided before we get 1st request).
1427     // We also do not bump redirected CONNECT requests.
1428     if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1429             !Config.accessList.ssl_bump ||
1430             !http->getConn()->port->flags.tunnelSslBumping) {
1431         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1432         debugs(85, 5, HERE << "cannot SslBump this request");
1433         return false;
1434     }
1435
1436     // Do not bump during authentication: clients would not proxy-authenticate
1437     // if we delay a 407 response and respond with 200 OK to CONNECT.
1438     if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1439         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1440         debugs(85, 5, HERE << "no SslBump during proxy authentication");
1441         return false;
1442     }
1443
1444     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1445
1446     ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1447     aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1448     return true;
1449 }
1450
1451 /**
1452  * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1453  * as ACLFilledChecklist callback
1454  */
1455 static void
1456 sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1457 {
1458     ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1459
1460     if (!calloutContext->httpStateIsValid())
1461         return;
1462     calloutContext->sslBumpAccessCheckDone(answer);
1463 }
1464
1465 void
1466 ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1467 {
1468     if (!httpStateIsValid())
1469         return;
1470
1471     const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1472                                    static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1473     http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1474     http->al->ssl.bumpMode = bumpMode; // for logging
1475
1476     http->doCallouts();
1477 }
1478 #endif
1479
1480 /*
1481  * Identify requests that do not go through the store and client side stream
1482  * and forward them to the appropriate location. All other requests, request
1483  * them.
1484  */
1485 void
1486 ClientHttpRequest::processRequest()
1487 {
1488     debugs(85, 4, request->method << ' ' << uri);
1489
1490     if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1491 #if USE_OPENSSL
1492         if (sslBumpNeeded()) {
1493             sslBumpStart();
1494             return;
1495         }
1496 #endif
1497         getConn()->stopReading(); // tunnels read for themselves
1498         tunnelStart(this, &out.size, &al->http.code, al);
1499         return;
1500     }
1501
1502     httpStart();
1503 }
1504
1505 void
1506 ClientHttpRequest::httpStart()
1507 {
1508     PROF_start(httpStart);
1509     logType = LOG_TAG_NONE;
1510     debugs(85, 4, LogTags_str[logType] << " for '" << uri << "'");
1511
1512     /* no one should have touched this */
1513     assert(out.offset == 0);
1514     /* Use the Stream Luke */
1515     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1516     clientStreamRead(node, this, node->readBuffer);
1517     PROF_stop(httpStart);
1518 }
1519
1520 #if USE_OPENSSL
1521
1522 void
1523 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1524 {
1525     debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1526     sslBumpNeed_ = mode;
1527 }
1528
1529 // called when comm_write has completed
1530 static void
1531 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1532 {
1533     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1534     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1535
1536     assert(r && cbdataReferenceValid(r));
1537     r->sslBumpEstablish(errflag);
1538 }
1539
1540 void
1541 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1542 {
1543     // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1544     if (errflag == Comm::ERR_CLOSING)
1545         return;
1546
1547     if (errflag) {
1548         debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1549         getConn()->clientConnection->close();
1550         return;
1551     }
1552
1553     // We lack HttpReply which logRequest() uses to log the status code.
1554     // TODO: Use HttpReply instead of the "200 Connection established" string.
1555     al->http.code = 200;
1556
1557 #if USE_AUTH
1558     // Preserve authentication info for the ssl-bumped request
1559     if (request->auth_user_request != NULL)
1560         getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1561 #endif
1562
1563     assert(sslBumpNeeded());
1564     getConn()->switchToHttps(request, sslBumpNeed_);
1565 }
1566
1567 void
1568 ClientHttpRequest::sslBumpStart()
1569 {
1570     debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1571            "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1572     getConn()->sslBumpMode = sslBumpNeed_;
1573
1574     // send an HTTP 200 response to kick client SSL negotiation
1575     // TODO: Unify with tunnel.cc and add a Server(?) header
1576     static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1577     AsyncCall::Pointer call = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1578                                          CommIoCbPtrFun(&SslBumpEstablish, this));
1579     Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), call, NULL);
1580 }
1581
1582 #endif
1583
1584 bool
1585 ClientHttpRequest::gotEnough() const
1586 {
1587     /** TODO: should be querying the stream. */
1588     int64_t contentLength =
1589         memObject()->getReply()->bodySize(request->method);
1590     assert(contentLength >= 0);
1591
1592     if (out.offset < contentLength)
1593         return false;
1594
1595     return true;
1596 }
1597
1598 void
1599 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1600 {
1601     entry_ = newEntry;
1602 }
1603
1604 void
1605 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1606 {
1607     if (loggingEntry_)
1608         loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1609
1610     loggingEntry_ = newEntry;
1611
1612     if (loggingEntry_)
1613         loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1614 }
1615
1616 /*
1617  * doCallouts() - This function controls the order of "callout"
1618  * executions, including non-blocking access control checks, the
1619  * redirector, and ICAP.  Previously, these callouts were chained
1620  * together such that "clientAccessCheckDone()" would call
1621  * "clientRedirectStart()" and so on.
1622  *
1623  * The ClientRequestContext (aka calloutContext) class holds certain
1624  * state data for the callout/callback operations.  Previously
1625  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1626  * for a short time.  ClientRequestContext would then delete itself
1627  * and pass control back to ClientHttpRequest when all callouts
1628  * were finished.
1629  *
1630  * This caused some problems for ICAP because we want to make the
1631  * ICAP callout after checking ACLs, but before checking the no_cache
1632  * list.  We can't stuff the ICAP state into the ClientRequestContext
1633  * class because we still need the ICAP state after ClientRequestContext
1634  * goes away.
1635  *
1636  * Note that ClientRequestContext is created before the first call
1637  * to doCallouts().
1638  *
1639  * If one of the callouts notices that ClientHttpRequest is no
1640  * longer valid, it should call cbdataReferenceDone() so that
1641  * ClientHttpRequest's reference count goes to zero and it will get
1642  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1643  *
1644  * Note that we set the _done flags here before actually starting
1645  * the callout.  This is strictly for convenience.
1646  */
1647
1648 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1649 nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1650
1651 void
1652 ClientHttpRequest::doCallouts()
1653 {
1654     assert(calloutContext);
1655
1656     /*Save the original request for logging purposes*/
1657     if (!calloutContext->http->al->request) {
1658         calloutContext->http->al->request = request;
1659         HTTPMSGLOCK(calloutContext->http->al->request);
1660
1661         NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1662         // Make the previously set client connection ID available as annotation.
1663         if (ConnStateData *csd = calloutContext->http->getConn()) {
1664             if (!csd->connectionTag().isEmpty())
1665                 notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1666         }
1667     }
1668
1669     if (!calloutContext->error) {
1670         // CVE-2009-0801: verify the Host: header is consistent with other known details.
1671         if (!calloutContext->host_header_verify_done) {
1672             debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1673             calloutContext->host_header_verify_done = true;
1674             calloutContext->hostHeaderVerify();
1675             return;
1676         }
1677
1678         if (!calloutContext->http_access_done) {
1679             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1680             calloutContext->http_access_done = true;
1681             calloutContext->clientAccessCheck();
1682             return;
1683         }
1684
1685 #if USE_ADAPTATION
1686         if (!calloutContext->adaptation_acl_check_done) {
1687             calloutContext->adaptation_acl_check_done = true;
1688             if (Adaptation::AccessCheck::Start(
1689                         Adaptation::methodReqmod, Adaptation::pointPreCache,
1690                         request, NULL, calloutContext->http->al, this))
1691                 return; // will call callback
1692         }
1693 #endif
1694
1695         if (!calloutContext->redirect_done) {
1696             calloutContext->redirect_done = true;
1697             assert(calloutContext->redirect_state == REDIRECT_NONE);
1698
1699             if (Config.Program.redirect) {
1700                 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1701                 calloutContext->redirect_state = REDIRECT_PENDING;
1702                 calloutContext->clientRedirectStart();
1703                 return;
1704             }
1705         }
1706
1707         if (!calloutContext->adapted_http_access_done) {
1708             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1709             calloutContext->adapted_http_access_done = true;
1710             calloutContext->clientAccessCheck2();
1711             return;
1712         }
1713
1714         if (!calloutContext->store_id_done) {
1715             calloutContext->store_id_done = true;
1716             assert(calloutContext->store_id_state == REDIRECT_NONE);
1717
1718             if (Config.Program.store_id) {
1719                 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1720                 calloutContext->store_id_state = REDIRECT_PENDING;
1721                 calloutContext->clientStoreIdStart();
1722                 return;
1723             }
1724         }
1725
1726         if (!calloutContext->interpreted_req_hdrs) {
1727             debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1728             calloutContext->interpreted_req_hdrs = 1;
1729             clientInterpretRequestHeaders(this);
1730         }
1731
1732         if (!calloutContext->no_cache_done) {
1733             calloutContext->no_cache_done = true;
1734
1735             if (Config.accessList.noCache && request->flags.cachable) {
1736                 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1737                 calloutContext->checkNoCache();
1738                 return;
1739             }
1740         }
1741     } //  if !calloutContext->error
1742
1743     if (!calloutContext->tosToClientDone) {
1744         calloutContext->tosToClientDone = true;
1745         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1746             ACLFilledChecklist ch(NULL, request, NULL);
1747             ch.src_addr = request->client_addr;
1748             ch.my_addr = request->my_addr;
1749             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1750             if (tos)
1751                 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1752         }
1753     }
1754
1755     if (!calloutContext->nfmarkToClientDone) {
1756         calloutContext->nfmarkToClientDone = true;
1757         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1758             ACLFilledChecklist ch(NULL, request, NULL);
1759             ch.src_addr = request->client_addr;
1760             ch.my_addr = request->my_addr;
1761             nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1762             if (mark)
1763                 Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1764         }
1765     }
1766
1767 #if USE_OPENSSL
1768     // We need to check for SslBump even if the calloutContext->error is set
1769     // because bumping may require delaying the error until after CONNECT.
1770     if (!calloutContext->sslBumpCheckDone) {
1771         calloutContext->sslBumpCheckDone = true;
1772         if (calloutContext->sslBumpAccessCheck())
1773             return;
1774         /* else no ssl bump required*/
1775     }
1776 #endif
1777
1778     if (calloutContext->error) {
1779         const char *storeUri = request->storeId();
1780         StoreEntry *e= storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1781 #if USE_OPENSSL
1782         if (sslBumpNeeded()) {
1783             // set final error but delay sending until we bump
1784             Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1785             errorAppendEntry(e, calloutContext->error);
1786             calloutContext->error = NULL;
1787             getConn()->setServerBump(srvBump);
1788             e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1789         } else
1790 #endif
1791         {
1792             // send the error to the client now
1793             clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1794             clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1795             assert (repContext);
1796             repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1797             errorAppendEntry(e, calloutContext->error);
1798             calloutContext->error = NULL;
1799             if (calloutContext->readNextRequest)
1800                 getConn()->flags.readMore = true; // resume any pipeline reads.
1801             node = (clientStreamNode *)client_stream.tail->data;
1802             clientStreamRead(node, this, node->readBuffer);
1803             e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1804             return;
1805         }
1806     }
1807
1808     cbdataReferenceDone(calloutContext->http);
1809     delete calloutContext;
1810     calloutContext = NULL;
1811 #if HEADERS_LOG
1812
1813     headersLog(0, 1, request->method, request);
1814 #endif
1815
1816     debugs(83, 3, HERE << "calling processRequest()");
1817     processRequest();
1818
1819 #if ICAP_CLIENT
1820     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1821     if (ih != NULL)
1822         ih->logType = logType;
1823 #endif
1824 }
1825
1826 #if !_USE_INLINE_
1827 #include "client_side_request.cci"
1828 #endif
1829
1830 #if USE_ADAPTATION
1831 /// Initiate an asynchronous adaptation transaction which will call us back.
1832 void
1833 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1834 {
1835     debugs(85, 3, HERE << "adaptation needed for " << this);
1836     assert(!virginHeadSource);
1837     assert(!adaptedBodySource);
1838     virginHeadSource = initiateAdaptation(
1839                            new Adaptation::Iterator(request, NULL, al, g));
1840
1841     // we could try to guess whether we can bypass this adaptation
1842     // initiation failure, but it should not really happen
1843     Must(initiated(virginHeadSource));
1844 }
1845
1846 void
1847 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1848 {
1849     assert(cbdataReferenceValid(this));         // indicates bug
1850     clearAdaptation(virginHeadSource);
1851     assert(!adaptedBodySource);
1852
1853     switch (answer.kind) {
1854     case Adaptation::Answer::akForward:
1855         handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1856         break;
1857
1858     case Adaptation::Answer::akBlock:
1859         handleAdaptationBlock(answer);
1860         break;
1861
1862     case Adaptation::Answer::akError:
1863         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1864         break;
1865     }
1866 }
1867
1868 void
1869 ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1870 {
1871     assert(msg);
1872
1873     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1874         /*
1875          * Replace the old request with the new request.
1876          */
1877         HTTPMSGUNLOCK(request);
1878         request = new_req;
1879         HTTPMSGLOCK(request);
1880
1881         // update the new message to flag whether URL re-writing was done on it
1882         if (strcmp(urlCanonical(request),uri) != 0)
1883             request->flags.redirected = 1;
1884
1885         /*
1886          * Store the new URI for logging
1887          */
1888         xfree(uri);
1889         uri = xstrdup(urlCanonical(request));
1890         setLogUri(this, urlCanonicalClean(request));
1891         assert(request->method.id());
1892     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1893         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1894
1895         // subscribe to receive reply body
1896         if (new_rep->body_pipe != NULL) {
1897             adaptedBodySource = new_rep->body_pipe;
1898             int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1899             assert(consumer_ok);
1900         }
1901
1902         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1903         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1904         assert(repContext);
1905         repContext->createStoreEntry(request->method, request->flags);
1906
1907         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1908         request_satisfaction_mode = true;
1909         request_satisfaction_offset = 0;
1910         storeEntry()->replaceHttpReply(new_rep);
1911         storeEntry()->timestampsSet();
1912
1913         if (!adaptedBodySource) // no body
1914             storeEntry()->complete();
1915         clientGetMoreData(node, this);
1916     }
1917
1918     // we are done with getting headers (but may be receiving body)
1919     clearAdaptation(virginHeadSource);
1920
1921     if (!request_satisfaction_mode)
1922         doCallouts();
1923 }
1924
1925 void
1926 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1927 {
1928     request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1929     AclMatchedName = answer.ruleId.termedBuf();
1930     assert(calloutContext);
1931     calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1932     AclMatchedName = NULL;
1933 }
1934
1935 void
1936 ClientHttpRequest::resumeBodyStorage()
1937 {
1938     if (!adaptedBodySource)
1939         return;
1940
1941     noteMoreBodyDataAvailable(adaptedBodySource);
1942 }
1943
1944 void
1945 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1946 {
1947     assert(request_satisfaction_mode);
1948     assert(adaptedBodySource != NULL);
1949
1950     if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1951         const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1952
1953         if (spaceAvailable < contentSize ) {
1954             // No or partial body data consuming
1955             typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1956             AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1957                                                 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1958             storeEntry()->deferProducer(call);
1959         }
1960
1961         if (!spaceAvailable)
1962             return;
1963
1964         if (spaceAvailable < contentSize )
1965             contentSize = spaceAvailable;
1966
1967         BodyPipeCheckout bpc(*adaptedBodySource);
1968         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
1969         storeEntry()->write(ioBuf);
1970         // assume StoreEntry::write() writes the entire ioBuf
1971         request_satisfaction_offset += ioBuf.length;
1972         bpc.buf.consume(contentSize);
1973         bpc.checkIn();
1974     }
1975
1976     if (adaptedBodySource->exhausted())
1977         endRequestSatisfaction();
1978     // else wait for more body data
1979 }
1980
1981 void
1982 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
1983 {
1984     assert(!virginHeadSource);
1985     // should we end request satisfaction now?
1986     if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
1987         endRequestSatisfaction();
1988 }
1989
1990 void
1991 ClientHttpRequest::endRequestSatisfaction()
1992 {
1993     debugs(85,4, HERE << this << " ends request satisfaction");
1994     assert(request_satisfaction_mode);
1995     stopConsumingFrom(adaptedBodySource);
1996
1997     // TODO: anything else needed to end store entry formation correctly?
1998     storeEntry()->complete();
1999 }
2000
2001 void
2002 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2003 {
2004     assert(!virginHeadSource);
2005     stopConsumingFrom(adaptedBodySource);
2006
2007     debugs(85,3, HERE << "REQMOD body production failed");
2008     if (request_satisfaction_mode) { // too late to recover or serve an error
2009         request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2010         const Comm::ConnectionPointer c = getConn()->clientConnection;
2011         Must(Comm::IsConnOpen(c));
2012         c->close(); // drastic, but we may be writing a response already
2013     } else {
2014         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2015     }
2016 }
2017
2018 void
2019 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2020 {
2021     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2022
2023     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2024     const bool usedPipe = request->body_pipe != NULL &&
2025                           request->body_pipe->consumedSize() > 0;
2026
2027     if (bypassable && !usedStore && !usedPipe) {
2028         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2029         if (calloutContext)
2030             doCallouts();
2031         return;
2032     }
2033
2034     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2035
2036     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2037     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2038     assert(repContext);
2039
2040     // The original author of the code also wanted to pass an errno to
2041     // setReplyToError, but it seems unlikely that the errno reflects the
2042     // true cause of the error at this point, so I did not pass it.
2043     if (calloutContext) {
2044         Ip::Address noAddr;
2045         noAddr.setNoAddr();
2046         ConnStateData * c = getConn();
2047         calloutContext->error = clientBuildError(ERR_ICAP_FAILURE, Http::scInternalServerError,
2048                                 NULL,
2049                                 c != NULL ? c->clientConnection->remote : noAddr,
2050                                 request
2051                                                 );
2052 #if USE_AUTH
2053         calloutContext->error->auth_user_request =
2054             c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2055 #endif
2056         calloutContext->error->detailError(errDetail);
2057         calloutContext->readNextRequest = true;
2058         if (c != NULL)
2059             c->expectNoForwarding();
2060         doCallouts();
2061     }
2062     //else if(calloutContext == NULL) is it possible?
2063 }
2064
2065 #endif