src/client_side_request.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 85    Client-side Request Routines */
  10
  11 /*
  12  * General logic of request processing:
  13  *
  14  * We run a series of tests to determine if access will be permitted, and to do
  15  * any redirection. Then we call into the result clientStream to retrieve data.
  16  * From that point on it's up to reply management.
  17  */
  18
  19 #include "squid.h"
  20 #include "acl/FilledChecklist.h"
  21 #include "acl/Gadgets.h"
  22 #include "anyp/PortCfg.h"
  23 #include "base/AsyncJobCalls.h"
  24 #include "client_side.h"
  25 #include "client_side_reply.h"
  26 #include "client_side_request.h"
  27 #include "ClientRequestContext.h"
  28 #include "clientStream.h"
  29 #include "comm/Connection.h"
  30 #include "comm/Write.h"
  31 #include "err_detail_type.h"
  32 #include "errorpage.h"
  33 #include "fd.h"
  34 #include "fde.h"
  35 #include "format/Token.h"
  36 #include "gopher.h"
  37 #include "helper.h"
  38 #include "helper/Reply.h"
  39 #include "http.h"
  40 #include "HttpHdrCc.h"
  41 #include "HttpReply.h"
  42 #include "HttpRequest.h"
  43 #include "ip/QosConfig.h"
  44 #include "ipcache.h"
  45 #include "log/access_log.h"
  46 #include "MemObject.h"
  47 #include "Parsing.h"
  48 #include "profiler/Profiler.h"
  49 #include "redirect.h"
  50 #include "SquidConfig.h"
  51 #include "SquidTime.h"
  52 #include "Store.h"
  53 #include "StrList.h"
  54 #include "tools.h"
  55 #include "URL.h"
  56 #include "wordlist.h"
  57 #if USE_AUTH
  58 #include "auth/UserRequest.h"
  59 #endif
  60 #if USE_ADAPTATION
  61 #include "adaptation/AccessCheck.h"
  62 #include "adaptation/Answer.h"
  63 #include "adaptation/Iterator.h"
  64 #include "adaptation/Service.h"
  65 #if ICAP_CLIENT
  66 #include "adaptation/icap/History.h"
  67 #endif
  68 #endif
  69 #if USE_OPENSSL
  70 #include "ssl/ServerBump.h"
  71 #include "ssl/support.h"
  72 #endif
  73
  74 #if LINGERING_CLOSE
  75 #define comm_close comm_lingering_close
  76 #endif
  77
  78 static const char *const crlf = "\r\n";
  79
  80 #if FOLLOW_X_FORWARDED_FOR
  81 static void clientFollowXForwardedForCheck(allow_t answer, void *data);
  82 #endif /* FOLLOW_X_FORWARDED_FOR */
  83
  84 ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
  85
  86 CBDATA_CLASS_INIT(ClientRequestContext);
  87
  88 /* Local functions */
  89 /* other */
  90 static void clientAccessCheckDoneWrapper(allow_t, void *);
  91 #if USE_OPENSSL
  92 static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
  93 #endif
  94 static int clientHierarchical(ClientHttpRequest * http);
  95 static void clientInterpretRequestHeaders(ClientHttpRequest * http);
  96 static HLPCB clientRedirectDoneWrapper;
  97 static HLPCB clientStoreIdDoneWrapper;
  98 static void checkNoCacheDoneWrapper(allow_t, void *);
  99 SQUIDCEXTERN CSR clientGetMoreData;
 100 SQUIDCEXTERN CSS clientReplyStatus;
 101 SQUIDCEXTERN CSD clientReplyDetach;
 102 static void checkFailureRatio(err_type, hier_code);
 103
 104 ClientRequestContext::~ClientRequestContext()
 105 {
 106     /*
 107      * Release our "lock" on our parent, ClientHttpRequest, if we
 108      * still have one
 109      */
 110
 111     if (http)
 112         cbdataReferenceDone(http);
 113
 114     delete error;
 115     debugs(85,3, "ClientRequestContext destructed, this=" << this);
 116 }
 117
 118 ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) :
 119     http(cbdataReference(anHttp)),
 120     acl_checklist(NULL),
 121     redirect_state(REDIRECT_NONE),
 122     store_id_state(REDIRECT_NONE),
 123     host_header_verify_done(false),
 124     http_access_done(false),
 125     adapted_http_access_done(false),
 126 #if USE_ADAPTATION
 127     adaptation_acl_check_done(false),
 128 #endif
 129     redirect_done(false),
 130     store_id_done(false),
 131     no_cache_done(false),
 132     interpreted_req_hdrs(false),
 133     tosToClientDone(false),
 134     nfmarkToClientDone(false),
 135 #if USE_OPENSSL
 136     sslBumpCheckDone(false),
 137 #endif
 138     error(NULL),
 139     readNextRequest(false)
 140 {
 141     debugs(85, 3, "ClientRequestContext constructed, this=" << this);
 142 }
 143
 144 CBDATA_CLASS_INIT(ClientHttpRequest);
 145
 146 ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
 147 #if USE_ADAPTATION
 148     AsyncJob("ClientHttpRequest"),
 149 #endif
 150     request(NULL),
 151     uri(NULL),
 152     log_uri(NULL),
 153     req_sz(0),
 154     logType(LOG_TAG_NONE),
 155     calloutContext(NULL),
 156     maxReplyBodySize_(0),
 157     entry_(NULL),
 158     loggingEntry_(NULL),
 159     conn_(NULL)
 160 #if USE_OPENSSL
 161     , sslBumpNeed_(Ssl::bumpEnd)
 162 #endif
 163 #if USE_ADAPTATION
 164     , request_satisfaction_mode(false)
 165     , request_satisfaction_offset(0)
 166 #endif
 167 {
 168     setConn(aConn);
 169     al = new AccessLogEntry;
 170     al->cache.start_time = current_time;
 171     al->tcpClient = clientConnection = aConn->clientConnection;
 172     al->cache.port = aConn->port;
 173     al->cache.caddr = aConn->log_addr;
 174
 175 #if USE_OPENSSL
 176     if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
 177         if (auto ssl = fd_table[aConn->clientConnection->fd].ssl)
 178             al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
 179     }
 180 #endif
 181     dlinkAdd(this, &active, &ClientActiveRequests);
 182 }
 183
 184 /*
 185  * returns true if client specified that the object must come from the cache
 186  * without contacting origin server
 187  */
 188 bool
 189 ClientHttpRequest::onlyIfCached()const
 190 {
 191     assert(request);
 192     return request->cache_control &&
 193            request->cache_control->onlyIfCached();
 194 }
 195
 196 /**
 197  * This function is designed to serve a fairly specific purpose.
 198  * Occasionally our vBNS-connected caches can talk to each other, but not
 199  * the rest of the world.  Here we try to detect frequent failures which
 200  * make the cache unusable (e.g. DNS lookup and connect() failures).  If
 201  * the failure:success ratio goes above 1.0 then we go into "hit only"
 202  * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
 203  * will only fetch HITs from us if they are using the ICP protocol.  We
 204  * stay in this mode for 5 minutes.
 205  *
 206  * Duane W., Sept 16, 1996
 207  */
 208 static void
 209 checkFailureRatio(err_type etype, hier_code hcode)
 210 {
 211     // Can be set at compile time with -D compiler flag
 212 #ifndef FAILURE_MODE_TIME
 213 #define FAILURE_MODE_TIME 300
 214 #endif
 215
 216     if (hcode == HIER_NONE)
 217         return;
 218
 219     // don't bother when ICP is disabled.
 220     if (Config.Port.icp <= 0)
 221         return;
 222
 223     static double magic_factor = 100.0;
 224     double n_good;
 225     double n_bad;
 226
 227     n_good = magic_factor / (1.0 + request_failure_ratio);
 228
 229     n_bad = magic_factor - n_good;
 230
 231     switch (etype) {
 232
 233     case ERR_DNS_FAIL:
 234
 235     case ERR_CONNECT_FAIL:
 236     case ERR_SECURE_CONNECT_FAIL:
 237
 238     case ERR_READ_ERROR:
 239         ++n_bad;
 240         break;
 241
 242     default:
 243         ++n_good;
 244     }
 245
 246     request_failure_ratio = n_bad / n_good;
 247
 248     if (hit_only_mode_until > squid_curtime)
 249         return;
 250
 251     if (request_failure_ratio < 1.0)
 252         return;
 253
 254     debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
 255            std::setprecision(3) << request_failure_ratio);
 256
 257     debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
 258            FAILURE_MODE_TIME / 60 << " minutes...");
 259
 260     hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
 261
 262     request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
 263 }
 264
 265 ClientHttpRequest::~ClientHttpRequest()
 266 {
 267     debugs(33, 3, "httpRequestFree: " << uri);
 268     PROF_start(httpRequestFree);
 269
 270     // Even though freeResources() below may destroy the request,
 271     // we no longer set request->body_pipe to NULL here
 272     // because we did not initiate that pipe (ConnStateData did)
 273
 274     /* the ICP check here was erroneous
 275      * - StoreEntry::releaseRequest was always called if entry was valid
 276      */
 277
 278     logRequest();
 279
 280     loggingEntry(NULL);
 281
 282     if (request)
 283         checkFailureRatio(request->errType, al->hier.code);
 284
 285     freeResources();
 286
 287 #if USE_ADAPTATION
 288     announceInitiatorAbort(virginHeadSource);
 289
 290     if (adaptedBodySource != NULL)
 291         stopConsumingFrom(adaptedBodySource);
 292 #endif
 293
 294     if (calloutContext)
 295         delete calloutContext;
 296
 297     clientConnection = NULL;
 298
 299     if (conn_)
 300         cbdataReferenceDone(conn_);
 301
 302     /* moving to the next connection is handled by the context free */
 303     dlinkDelete(&active, &ClientActiveRequests);
 304
 305     PROF_stop(httpRequestFree);
 306 }
 307
 308 /**
 309  * Create a request and kick it off
 310  *
 311  * \retval 0     success
 312  * \retval -1    failure
 313  *
 314  * TODO: Pass in the buffers to be used in the inital Read request, as they are
 315  * determined by the user
 316  */
 317 int
 318 clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
 319                    CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
 320                    char *tailbuf, size_t taillen)
 321 {
 322     size_t url_sz;
 323     ClientHttpRequest *http = new ClientHttpRequest(NULL);
 324     HttpRequest *request;
 325     StoreIOBuffer tempBuffer;
 326     if (http->al != NULL)
 327         http->al->cache.start_time = current_time;
 328     /* this is only used to adjust the connection offset in client_side.c */
 329     http->req_sz = 0;
 330     tempBuffer.length = taillen;
 331     tempBuffer.data = tailbuf;
 332     /* client stream setup */
 333     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 334                      clientReplyStatus, new clientReplyContext(http), streamcallback,
 335                      streamdetach, streamdata, tempBuffer);
 336     /* make it visible in the 'current acctive requests list' */
 337     /* Set flags */
 338     /* internal requests only makes sense in an
 339      * accelerator today. TODO: accept flags ? */
 340     http->flags.accel = true;
 341     /* allow size for url rewriting */
 342     url_sz = strlen(url) + Config.appendDomainLen + 5;
 343     http->uri = (char *)xcalloc(url_sz, 1);
 344     strcpy(http->uri, url);
 345
 346     if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
 347         debugs(85, 5, "Invalid URL: " << http->uri);
 348         return -1;
 349     }
 350
 351     /*
 352      * now update the headers in request with our supplied headers. urlParse
 353      * should return a blank header set, but we use Update to be sure of
 354      * correctness.
 355      */
 356     if (header)
 357         request->header.update(header, NULL);
 358
 359     http->log_uri = xstrdup(urlCanonicalClean(request));
 360
 361     /* http struct now ready */
 362
 363     /*
 364      * build new header list *? TODO
 365      */
 366     request->flags.accelerated = http->flags.accel;
 367
 368     request->flags.internalClient = true;
 369
 370     /* this is an internally created
 371      * request, not subject to acceleration
 372      * target overrides */
 373     /*
 374      * FIXME? Do we want to detect and handle internal requests of internal
 375      * objects ?
 376      */
 377
 378     /* Internally created requests cannot have bodies today */
 379     request->content_length = 0;
 380
 381     request->client_addr.setNoAddr();
 382
 383 #if FOLLOW_X_FORWARDED_FOR
 384     request->indirect_client_addr.setNoAddr();
 385 #endif /* FOLLOW_X_FORWARDED_FOR */
 386
 387     request->my_addr.setNoAddr();   /* undefined for internal requests */
 388
 389     request->my_addr.port(0);
 390
 391     request->http_ver = Http::ProtocolVersion();
 392
 393     http->request = request;
 394     HTTPMSGLOCK(http->request);
 395
 396     /* optional - skip the access check ? */
 397     http->calloutContext = new ClientRequestContext(http);
 398
 399     http->calloutContext->http_access_done = false;
 400
 401     http->calloutContext->redirect_done = true;
 402
 403     http->calloutContext->no_cache_done = true;
 404
 405     http->doCallouts();
 406
 407     return 0;
 408 }
 409
 410 bool
 411 ClientRequestContext::httpStateIsValid()
 412 {
 413     ClientHttpRequest *http_ = http;
 414
 415     if (cbdataReferenceValid(http_))
 416         return true;
 417
 418     http = NULL;
 419
 420     cbdataReferenceDone(http_);
 421
 422     return false;
 423 }
 424
 425 #if FOLLOW_X_FORWARDED_FOR
 426 /**
 427  * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
 428  * against the followXFF ACL, or cleans up and passes control to
 429  * clientAccessCheck().
 430  *
 431  * The trust model here is a little ambiguous. So to clarify the logic:
 432  * - we may always use the direct client address as the client IP.
 433  * - these trust tests merey tell whether we trust given IP enough to believe the
 434  *   IP string which it appended to the X-Forwarded-For: header.
 435  * - if at any point we don't trust what an IP adds we stop looking.
 436  * - at that point the current contents of indirect_client_addr are the value set
 437  *   by the last previously trusted IP.
 438  * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
 439  */
 440 static void
 441 clientFollowXForwardedForCheck(allow_t answer, void *data)
 442 {
 443     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 444
 445     if (!calloutContext->httpStateIsValid())
 446         return;
 447
 448     ClientHttpRequest *http = calloutContext->http;
 449     HttpRequest *request = http->request;
 450
 451     /*
 452      * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
 453      * called as a result of ACL checks, or -1 if we are called when
 454      * there's nothing left to do.
 455      */
 456     if (answer == ACCESS_ALLOWED &&
 457             request->x_forwarded_for_iterator.size () != 0) {
 458
 459         /*
 460          * Remove the last comma-delimited element from the
 461          * x_forwarded_for_iterator and use it to repeat the cycle.
 462          */
 463         const char *p;
 464         const char *asciiaddr;
 465         int l;
 466         Ip::Address addr;
 467         p = request->x_forwarded_for_iterator.termedBuf();
 468         l = request->x_forwarded_for_iterator.size();
 469
 470         /*
 471         * XXX x_forwarded_for_iterator should really be a list of
 472         * IP addresses, but it's a String instead.  We have to
 473         * walk backwards through the String, biting off the last
 474         * comma-delimited part each time.  As long as the data is in
 475         * a String, we should probably implement and use a variant of
 476         * strListGetItem() that walks backwards instead of forwards
 477         * through a comma-separated list.  But we don't even do that;
 478         * we just do the work in-line here.
 479         */
 480         /* skip trailing space and commas */
 481         while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
 482             --l;
 483         request->x_forwarded_for_iterator.cut(l);
 484         /* look for start of last item in list */
 485         while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
 486             --l;
 487         asciiaddr = p+l;
 488         if ((addr = asciiaddr)) {
 489             request->indirect_client_addr = addr;
 490             request->x_forwarded_for_iterator.cut(l);
 491             calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 492             if (!Config.onoff.acl_uses_indirect_client) {
 493                 /* override the default src_addr tested if we have to go deeper than one level into XFF */
 494                 Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
 495             }
 496             calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
 497             return;
 498         }
 499     } /*if (answer == ACCESS_ALLOWED &&
 500         request->x_forwarded_for_iterator.size () != 0)*/
 501
 502     /* clean up, and pass control to clientAccessCheck */
 503     if (Config.onoff.log_uses_indirect_client) {
 504         /*
 505         * Ensure that the access log shows the indirect client
 506         * instead of the direct client.
 507         */
 508         ConnStateData *conn = http->getConn();
 509         conn->log_addr = request->indirect_client_addr;
 510         http->al->cache.caddr = conn->log_addr;
 511     }
 512     request->x_forwarded_for_iterator.clean();
 513     request->flags.done_follow_x_forwarded_for = true;
 514
 515     if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
 516         debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
 517     }
 518
 519     /* process actual access ACL as normal. */
 520     calloutContext->clientAccessCheck();
 521 }
 522 #endif /* FOLLOW_X_FORWARDED_FOR */
 523
 524 static void
 525 hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const Dns::LookupDetails &dns, void *data)
 526 {
 527     ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
 528     c->hostHeaderIpVerify(ia, dns);
 529 }
 530
 531 void
 532 ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const Dns::LookupDetails &dns)
 533 {
 534     Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
 535
 536     // note the DNS details for the transaction stats.
 537     http->request->recordLookup(dns);
 538
 539     if (ia != NULL && ia->count > 0) {
 540         // Is the NAT destination IP in DNS?
 541         for (int i = 0; i < ia->count; ++i) {
 542             if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
 543                 debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
 544                 http->request->flags.hostVerified = true;
 545                 http->doCallouts();
 546                 return;
 547             }
 548             debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
 549         }
 550     }
 551     debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
 552     hostHeaderVerifyFailed("local IP", "any domain IP");
 553 }
 554
 555 void
 556 ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
 557 {
 558     // IP address validation for Host: failed. Admin wants to ignore them.
 559     // NP: we do not yet handle CONNECT tunnels well, so ignore for them
 560     if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
 561         debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
 562                " (" << A << " does not match " << B << ") on URL: " << http->request->effectiveRequestUri());
 563
 564         // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
 565         // The problems here are about WRITE for new cache content, which means flags.cachable
 566         http->request->flags.cachable = false; // MUST NOT cache (for now)
 567         // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
 568         http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
 569         // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
 570         http->doCallouts();
 571         return;
 572     }
 573
 574     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
 575            http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
 576     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << http->request->header.getStr(Http::HdrType::USER_AGENT));
 577     debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << http->request->effectiveRequestUri());
 578
 579     // IP address validation for Host: failed. reject the connection.
 580     clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
 581     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 582     assert (repContext);
 583     repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
 584                                 http->request->method, NULL,
 585                                 http->getConn()->clientConnection->remote,
 586                                 http->request,
 587                                 NULL,
 588 #if USE_AUTH
 589                                 http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 590                                 http->getConn()->getAuth() : http->request->auth_user_request);
 591 #else
 592                                 NULL);
 593 #endif
 594     node = (clientStreamNode *)http->client_stream.tail->data;
 595     clientStreamRead(node, http, node->readBuffer);
 596 }
 597
 598 void
 599 ClientRequestContext::hostHeaderVerify()
 600 {
 601     // Require a Host: header.
 602     const char *host = http->request->header.getStr(Http::HdrType::HOST);
 603
 604     if (!host) {
 605         // TODO: dump out the HTTP/1.1 error about missing host header.
 606         // otherwise this is fine, can't forge a header value when its not even set.
 607         debugs(85, 3, HERE << "validate skipped with no Host: header present.");
 608         http->doCallouts();
 609         return;
 610     }
 611
 612     if (http->request->flags.internal) {
 613         // TODO: kill this when URL handling allows partial URLs out of accel mode
 614         //       and we no longer screw with the URL just to add our internal host there
 615         debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
 616         http->doCallouts();
 617         return;
 618     }
 619
 620     // Locate if there is a port attached, strip ready for IP lookup
 621     char *portStr = NULL;
 622     char *hostB = xstrdup(host);
 623     host = hostB;
 624     if (host[0] == '[') {
 625         // IPv6 literal.
 626         portStr = strchr(hostB, ']');
 627         if (portStr && *(++portStr) != ':') {
 628             portStr = NULL;
 629         }
 630     } else {
 631         // Domain or IPv4 literal with port
 632         portStr = strrchr(hostB, ':');
 633     }
 634
 635     uint16_t port = 0;
 636     if (portStr) {
 637         *portStr = '\0'; // strip the ':'
 638         if (*(++portStr) != '\0') {
 639             char *end = NULL;
 640             int64_t ret = strtoll(portStr, &end, 10);
 641             if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
 642                 // invalid port details. Replace the ':'
 643                 *(--portStr) = ':';
 644                 portStr = NULL;
 645             } else
 646                 port = (ret & 0xFFFF);
 647         }
 648     }
 649
 650     debugs(85, 3, "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
 651     if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
 652         // verify the Host: port (if any) matches the apparent destination
 653         if (portStr && port != http->getConn()->clientConnection->local.port()) {
 654             debugs(85, 3, "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
 655                    " matches Host: port " << port << " (" << portStr << ")");
 656             hostHeaderVerifyFailed("intercepted port", portStr);
 657         } else {
 658             // XXX: match the scheme default port against the apparent destination
 659
 660             // verify the destination DNS is one of the Host: headers IPs
 661             ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
 662         }
 663     } else if (!Config.onoff.hostStrictVerify) {
 664         debugs(85, 3, "validate skipped.");
 665         http->doCallouts();
 666     } else if (strlen(host) != strlen(http->request->url.host())) {
 667         // Verify forward-proxy requested URL domain matches the Host: header
 668         debugs(85, 3, "FAIL on validate URL domain length " << http->request->url.host() << " matches Host: " << host);
 669         hostHeaderVerifyFailed(host, http->request->url.host());
 670     } else if (matchDomainName(host, http->request->url.host()) != 0) {
 671         // Verify forward-proxy requested URL domain matches the Host: header
 672         debugs(85, 3, "FAIL on validate URL domain " << http->request->url.host() << " matches Host: " << host);
 673         hostHeaderVerifyFailed(host, http->request->url.host());
 674     } else if (portStr && port != http->request->url.port()) {
 675         // Verify forward-proxy requested URL domain matches the Host: header
 676         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: port " << portStr);
 677         hostHeaderVerifyFailed("URL port", portStr);
 678     } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->url.port() != http->request->url.getScheme().defaultPort()) {
 679         // Verify forward-proxy requested URL domain matches the Host: header
 680         // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
 681         debugs(85, 3, "FAIL on validate URL port " << http->request->url.port() << " matches Host: default port " << http->request->url.getScheme().defaultPort());
 682         hostHeaderVerifyFailed("URL port", "default port");
 683     } else {
 684         // Okay no problem.
 685         debugs(85, 3, "validate passed.");
 686         http->request->flags.hostVerified = true;
 687         http->doCallouts();
 688     }
 689     safe_free(hostB);
 690 }
 691
 692 /* This is the entry point for external users of the client_side routines */
 693 void
 694 ClientRequestContext::clientAccessCheck()
 695 {
 696 #if FOLLOW_X_FORWARDED_FOR
 697     if (!http->request->flags.doneFollowXff() &&
 698             Config.accessList.followXFF &&
 699             http->request->header.has(Http::HdrType::X_FORWARDED_FOR)) {
 700
 701         /* we always trust the direct client address for actual use */
 702         http->request->indirect_client_addr = http->request->client_addr;
 703         http->request->indirect_client_addr.port(0);
 704
 705         /* setup the XFF iterator for processing */
 706         http->request->x_forwarded_for_iterator = http->request->header.getList(Http::HdrType::X_FORWARDED_FOR);
 707
 708         /* begin by checking to see if we trust direct client enough to walk XFF */
 709         acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
 710         acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
 711         return;
 712     }
 713 #endif
 714
 715     if (Config.accessList.http) {
 716         acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
 717         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 718     } else {
 719         debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
 720         clientAccessCheckDone(ACCESS_DENIED);
 721     }
 722 }
 723
 724 /**
 725  * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
 726  * The default here is to allow all. Since the earlier http_access should do a default deny all.
 727  * This check is just for a last-minute denial based on adapted request headers.
 728  */
 729 void
 730 ClientRequestContext::clientAccessCheck2()
 731 {
 732     if (Config.accessList.adapted_http) {
 733         acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
 734         acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
 735     } else {
 736         debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
 737         clientAccessCheckDone(ACCESS_ALLOWED);
 738     }
 739 }
 740
 741 void
 742 clientAccessCheckDoneWrapper(allow_t answer, void *data)
 743 {
 744     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
 745
 746     if (!calloutContext->httpStateIsValid())
 747         return;
 748
 749     calloutContext->clientAccessCheckDone(answer);
 750 }
 751
 752 void
 753 ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
 754 {
 755     acl_checklist = NULL;
 756     err_type page_id;
 757     Http::StatusCode status;
 758     debugs(85, 2, "The request " << http->request->method << ' ' <<
 759            http->uri << " is " << answer <<
 760            "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
 761
 762 #if USE_AUTH
 763     char const *proxy_auth_msg = "<null>";
 764     if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
 765         proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
 766     else if (http->request->auth_user_request != NULL)
 767         proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
 768 #endif
 769
 770     if (answer != ACCESS_ALLOWED) {
 771         // auth has a grace period where credentials can be expired but okay not to challenge.
 772
 773         /* Send an auth challenge or error */
 774         // XXX: do we still need aclIsProxyAuth() ?
 775         bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
 776         debugs(85, 5, "Access Denied: " << http->uri);
 777         debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
 778 #if USE_AUTH
 779         if (auth_challenge)
 780             debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
 781 #endif
 782
 783         /*
 784          * NOTE: get page_id here, based on AclMatchedName because if
 785          * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
 786          * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
 787          * <pribeiro@isel.pt>
 788          */
 789         page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
 790
 791         http->logType = LOG_TCP_DENIED;
 792
 793         if (auth_challenge) {
 794 #if USE_AUTH
 795             if (http->request->flags.sslBumped) {
 796                 /*SSL Bumped request, authentication is not possible*/
 797                 status = Http::scForbidden;
 798             } else if (!http->flags.accel) {
 799                 /* Proxy authorisation needed */
 800                 status = Http::scProxyAuthenticationRequired;
 801             } else {
 802                 /* WWW authorisation needed */
 803                 status = Http::scUnauthorized;
 804             }
 805 #else
 806             // need auth, but not possible to do.
 807             status = Http::scForbidden;
 808 #endif
 809             if (page_id == ERR_NONE)
 810                 page_id = ERR_CACHE_ACCESS_DENIED;
 811         } else {
 812             status = Http::scForbidden;
 813
 814             if (page_id == ERR_NONE)
 815                 page_id = ERR_ACCESS_DENIED;
 816         }
 817
 818         Ip::Address tmpnoaddr;
 819         tmpnoaddr.setNoAddr();
 820         error = clientBuildError(page_id, status,
 821                                  NULL,
 822                                  http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
 823                                  http->request
 824                                 );
 825
 826 #if USE_AUTH
 827         error->auth_user_request =
 828             http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
 829             http->getConn()->getAuth() : http->request->auth_user_request;
 830 #endif
 831
 832         readNextRequest = true;
 833     }
 834
 835     /* ACCESS_ALLOWED continues here ... */
 836     xfree(http->uri);
 837     http->uri = SBufToCstring(http->request->effectiveRequestUri());
 838     http->doCallouts();
 839 }
 840
 841 #if USE_ADAPTATION
 842 void
 843 ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
 844 {
 845     debugs(93,3,HERE << this << " adaptationAclCheckDone called");
 846
 847 #if ICAP_CLIENT
 848     Adaptation::Icap::History::Pointer ih = request->icapHistory();
 849     if (ih != NULL) {
 850         if (getConn() != NULL && getConn()->clientConnection != NULL) {
 851             ih->rfc931 = getConn()->clientConnection->rfc931;
 852 #if USE_OPENSSL
 853             if (getConn()->clientConnection->isOpen()) {
 854                 ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl);
 855             }
 856 #endif
 857         }
 858         ih->log_uri = log_uri;
 859         ih->req_sz = req_sz;
 860     }
 861 #endif
 862
 863     if (!g) {
 864         debugs(85,3, HERE << "no adaptation needed");
 865         doCallouts();
 866         return;
 867     }
 868
 869     startAdaptation(g);
 870 }
 871
 872 #endif
 873
 874 static void
 875 clientRedirectAccessCheckDone(allow_t answer, void *data)
 876 {
 877     ClientRequestContext *context = (ClientRequestContext *)data;
 878     ClientHttpRequest *http = context->http;
 879     context->acl_checklist = NULL;
 880
 881     if (answer == ACCESS_ALLOWED)
 882         redirectStart(http, clientRedirectDoneWrapper, context);
 883     else {
 884         Helper::Reply nilReply;
 885         nilReply.result = Helper::Error;
 886         context->clientRedirectDone(nilReply);
 887     }
 888 }
 889
 890 void
 891 ClientRequestContext::clientRedirectStart()
 892 {
 893     debugs(33, 5, HERE << "'" << http->uri << "'");
 894     (void)SyncNotes(*http->al, *http->request);
 895     if (Config.accessList.redirector) {
 896         acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
 897         acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
 898     } else
 899         redirectStart(http, clientRedirectDoneWrapper, this);
 900 }
 901
 902 /**
 903  * This methods handles Access checks result of StoreId access list.
 904  * Will handle as "ERR" (no change) in a case Access is not allowed.
 905  */
 906 static void
 907 clientStoreIdAccessCheckDone(allow_t answer, void *data)
 908 {
 909     ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
 910     ClientHttpRequest *http = context->http;
 911     context->acl_checklist = NULL;
 912
 913     if (answer == ACCESS_ALLOWED)
 914         storeIdStart(http, clientStoreIdDoneWrapper, context);
 915     else {
 916         debugs(85, 3, "access denied expected ERR reply handling: " << answer);
 917         Helper::Reply nilReply;
 918         nilReply.result = Helper::Error;
 919         context->clientStoreIdDone(nilReply);
 920     }
 921 }
 922
 923 /**
 924  * Start locating an alternative storeage ID string (if any) from admin
 925  * configured helper program. This is an asynchronous operation terminating in
 926  * ClientRequestContext::clientStoreIdDone() when completed.
 927  */
 928 void
 929 ClientRequestContext::clientStoreIdStart()
 930 {
 931     debugs(33, 5,"'" << http->uri << "'");
 932
 933     if (Config.accessList.store_id) {
 934         acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
 935         acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
 936     } else
 937         storeIdStart(http, clientStoreIdDoneWrapper, this);
 938 }
 939
 940 static int
 941 clientHierarchical(ClientHttpRequest * http)
 942 {
 943     HttpRequest *request = http->request;
 944     HttpRequestMethod method = request->method;
 945
 946     // intercepted requests MUST NOT (yet) be sent to peers unless verified
 947     if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
 948         return 0;
 949
 950     /*
 951      * IMS needs a private key, so we can use the hierarchy for IMS only if our
 952      * neighbors support private keys
 953      */
 954
 955     if (request->flags.ims && !neighbors_do_private_keys)
 956         return 0;
 957
 958     /*
 959      * This is incorrect: authenticating requests can be sent via a hierarchy
 960      * (they can even be cached if the correct headers are set on the reply)
 961      */
 962     if (request->flags.auth)
 963         return 0;
 964
 965     if (method == Http::METHOD_TRACE)
 966         return 1;
 967
 968     if (method != Http::METHOD_GET)
 969         return 0;
 970
 971     if (request->flags.loopDetected)
 972         return 0;
 973
 974     if (request->url.getScheme() == AnyP::PROTO_HTTP)
 975         return method.respMaybeCacheable();
 976
 977     if (request->url.getScheme() == AnyP::PROTO_GOPHER)
 978         return gopherCachable(request);
 979
 980     if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
 981         return 0;
 982
 983     return 1;
 984 }
 985
 986 static void
 987 clientCheckPinning(ClientHttpRequest * http)
 988 {
 989     HttpRequest *request = http->request;
 990     HttpHeader *req_hdr = &request->header;
 991     ConnStateData *http_conn = http->getConn();
 992
 993     /* Internal requests such as those from ESI includes may be without
 994      * a client connection
 995      */
 996     if (!http_conn)
 997         return;
 998
 999     // Internal requests such as those from Downloader does not have
1000     // local port
1001     if (http_conn->port == NULL)
1002         return;
1003
1004     request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
1005     if (!request->flags.connectionAuthDisabled) {
1006         if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
1007             if (http_conn->pinning.auth) {
1008                 request->flags.connectionAuth = true;
1009                 request->flags.auth = true;
1010             } else {
1011                 request->flags.connectionProxyAuth = true;
1012             }
1013             // These should already be linked correctly.
1014             assert(request->clientConnectionManager == http_conn);
1015         }
1016     }
1017
1018     /* check if connection auth is used, and flag as candidate for pinning
1019      * in such case.
1020      * Note: we may need to set flags.connectionAuth even if the connection
1021      * is already pinned if it was pinned earlier due to proxy auth
1022      */
1023     if (!request->flags.connectionAuth) {
1024         if (req_hdr->has(Http::HdrType::AUTHORIZATION) || req_hdr->has(Http::HdrType::PROXY_AUTHORIZATION)) {
1025             HttpHeaderPos pos = HttpHeaderInitPos;
1026             HttpHeaderEntry *e;
1027             int may_pin = 0;
1028             while ((e = req_hdr->getEntry(&pos))) {
1029                 if (e->id == Http::HdrType::AUTHORIZATION || e->id == Http::HdrType::PROXY_AUTHORIZATION) {
1030                     const char *value = e->value.rawBuf();
1031                     if (strncasecmp(value, "NTLM ", 5) == 0
1032                             ||
1033                             strncasecmp(value, "Negotiate ", 10) == 0
1034                             ||
1035                             strncasecmp(value, "Kerberos ", 9) == 0) {
1036                         if (e->id == Http::HdrType::AUTHORIZATION) {
1037                             request->flags.connectionAuth = true;
1038                             may_pin = 1;
1039                         } else {
1040                             request->flags.connectionProxyAuth = true;
1041                             may_pin = 1;
1042                         }
1043                     }
1044                 }
1045             }
1046             if (may_pin && !request->pinnedConnection()) {
1047                 // These should already be linked correctly. Just need the ServerConnection to pinn.
1048                 assert(request->clientConnectionManager == http_conn);
1049             }
1050         }
1051     }
1052 }
1053
1054 static void
1055 clientInterpretRequestHeaders(ClientHttpRequest * http)
1056 {
1057     HttpRequest *request = http->request;
1058     HttpHeader *req_hdr = &request->header;
1059     bool no_cache = false;
1060     const char *str;
1061
1062     request->imslen = -1;
1063     request->ims = req_hdr->getTime(Http::HdrType::IF_MODIFIED_SINCE);
1064
1065     if (request->ims > 0)
1066         request->flags.ims = true;
1067
1068     if (!request->flags.ignoreCc) {
1069         if (request->cache_control) {
1070             if (request->cache_control->hasNoCache())
1071                 no_cache=true;
1072
1073             // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1074         } else if (req_hdr->has(Http::HdrType::PRAGMA))
1075             no_cache = req_hdr->hasListMember(Http::HdrType::PRAGMA,"no-cache",',');
1076
1077         /*
1078         * Work around for supporting the Reload button in IE browsers when Squid
1079         * is used as an accelerator or transparent proxy, by turning accelerated
1080         * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1081         * actually only fixed in SP1, but we can't tell whether we are talking to
1082         * SP1 or not so all 5.5 versions are treated 'normally').
1083         */
1084         if (Config.onoff.ie_refresh) {
1085             if (http->flags.accel && request->flags.ims) {
1086                 if ((str = req_hdr->getStr(Http::HdrType::USER_AGENT))) {
1087                     if (strstr(str, "MSIE 5.01") != NULL)
1088                         no_cache=true;
1089                     else if (strstr(str, "MSIE 5.0") != NULL)
1090                         no_cache=true;
1091                     else if (strstr(str, "MSIE 4.") != NULL)
1092                         no_cache=true;
1093                     else if (strstr(str, "MSIE 3.") != NULL)
1094                         no_cache=true;
1095                 }
1096             }
1097         }
1098     }
1099
1100     if (request->method == Http::METHOD_OTHER) {
1101         no_cache=true;
1102     }
1103
1104     if (no_cache) {
1105 #if USE_HTTP_VIOLATIONS
1106
1107         if (Config.onoff.reload_into_ims)
1108             request->flags.nocacheHack = true;
1109         else if (refresh_nocache_hack)
1110             request->flags.nocacheHack = true;
1111         else
1112 #endif
1113
1114             request->flags.noCache = true;
1115     }
1116
1117     /* ignore range header in non-GETs or non-HEADs */
1118     if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1119         // XXX: initialize if we got here without HttpRequest::parseHeader()
1120         if (!request->range)
1121             request->range = req_hdr->getRange();
1122
1123         if (request->range) {
1124             request->flags.isRanged = true;
1125             clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1126             /* XXX: This is suboptimal. We should give the stream the range set,
1127              * and thereby let the top of the stream set the offset when the
1128              * size becomes known. As it is, we will end up requesting from 0
1129              * for evey -X range specification.
1130              * RBC - this may be somewhat wrong. We should probably set the range
1131              * iter up at this point.
1132              */
1133             node->readBuffer.offset = request->range->lowestOffset(0);
1134             http->range_iter.pos = request->range->begin();
1135             http->range_iter.end = request->range->end();
1136             http->range_iter.valid = true;
1137         }
1138     }
1139
1140     /* Only HEAD and GET requests permit a Range or Request-Range header.
1141      * If these headers appear on any other type of request, delete them now.
1142      */
1143     else {
1144         req_hdr->delById(Http::HdrType::RANGE);
1145         req_hdr->delById(Http::HdrType::REQUEST_RANGE);
1146         request->ignoreRange("neither HEAD nor GET");
1147     }
1148
1149     if (req_hdr->has(Http::HdrType::AUTHORIZATION))
1150         request->flags.auth = true;
1151
1152     clientCheckPinning(http);
1153
1154     if (!request->url.userInfo().isEmpty())
1155         request->flags.auth = true;
1156
1157     if (req_hdr->has(Http::HdrType::VIA)) {
1158         String s = req_hdr->getList(Http::HdrType::VIA);
1159         /*
1160          * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1161          * Note ThisCache2 has a space prepended to the hostname so we don't
1162          * accidentally match super-domains.
1163          */
1164
1165         if (strListIsSubstr(&s, ThisCache2, ',')) {
1166             debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1167                      request, (ObjPackMethod) & httpRequestPack);
1168             request->flags.loopDetected = true;
1169         }
1170
1171 #if USE_FORW_VIA_DB
1172         fvdbCountVia(s.termedBuf());
1173
1174 #endif
1175
1176         s.clean();
1177     }
1178
1179 #if USE_FORW_VIA_DB
1180
1181     if (req_hdr->has(Http::HdrType::X_FORWARDED_FOR)) {
1182         String s = req_hdr->getList(Http::HdrType::X_FORWARDED_FOR);
1183         fvdbCountForw(s.termedBuf());
1184         s.clean();
1185     }
1186
1187 #endif
1188
1189     request->flags.cachable = http->request->maybeCacheable();
1190
1191     if (clientHierarchical(http))
1192         request->flags.hierarchical = true;
1193
1194     debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1195            (request->flags.noCache ? "SET" : "NOT SET"));
1196     debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1197            (request->flags.cachable ? "SET" : "NOT SET"));
1198     debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1199            (request->flags.hierarchical ? "SET" : "NOT SET"));
1200
1201 }
1202
1203 void
1204 clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1205 {
1206     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1207
1208     if (!calloutContext->httpStateIsValid())
1209         return;
1210
1211     calloutContext->clientRedirectDone(result);
1212 }
1213
1214 void
1215 clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1216 {
1217     ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1218
1219     if (!calloutContext->httpStateIsValid())
1220         return;
1221
1222     calloutContext->clientStoreIdDone(result);
1223 }
1224
1225 void
1226 ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1227 {
1228     HttpRequest *old_request = http->request;
1229     debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1230     assert(redirect_state == REDIRECT_PENDING);
1231     redirect_state = REDIRECT_DONE;
1232
1233     // Put helper response Notes into the transaction state record (ALE) eventually
1234     // do it early to ensure that no matter what the outcome the notes are present.
1235     if (http->al != NULL)
1236         (void)SyncNotes(*http->al, *old_request);
1237
1238     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1239
1240     switch (reply.result) {
1241     case Helper::TimedOut:
1242         if (Config.onUrlRewriteTimeout.action != toutActBypass) {
1243             http->calloutsError(ERR_GATEWAY_FAILURE, ERR_DETAIL_REDIRECTOR_TIMEDOUT);
1244             debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: Timedout");
1245         }
1246         break;
1247
1248     case Helper::Unknown:
1249     case Helper::TT:
1250         // Handler in redirect.cc should have already mapped Unknown
1251         // IF it contained valid entry for the old URL-rewrite helper protocol
1252         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1253         break;
1254
1255     case Helper::BrokenHelper:
1256         debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply);
1257         break;
1258
1259     case Helper::Error:
1260         // no change to be done.
1261         break;
1262
1263     case Helper::Okay: {
1264         // #1: redirect with a specific status code    OK status=NNN url="..."
1265         // #2: redirect with a default status code     OK url="..."
1266         // #3: re-write the URL                        OK rewrite-url="..."
1267
1268         const char *statusNote = reply.notes.findFirst("status");
1269         const char *urlNote = reply.notes.findFirst("url");
1270
1271         if (urlNote != NULL) {
1272             // HTTP protocol redirect to be done.
1273
1274             // TODO: change default redirect status for appropriate requests
1275             // Squid defaults to 302 status for now for better compatibility with old clients.
1276             // HTTP/1.0 client should get 302 (Http::scFound)
1277             // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1278             // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1279             Http::StatusCode status = Http::scFound;
1280             if (statusNote != NULL) {
1281                 const char * result = statusNote;
1282                 status = static_cast<Http::StatusCode>(atoi(result));
1283             }
1284
1285             if (status == Http::scMovedPermanently
1286                     || status == Http::scFound
1287                     || status == Http::scSeeOther
1288                     || status == Http::scPermanentRedirect
1289                     || status == Http::scTemporaryRedirect) {
1290                 http->redirect.status = status;
1291                 http->redirect.location = xstrdup(urlNote);
1292                 // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1293             } else {
1294                 debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1295             }
1296         } else {
1297             // URL-rewrite wanted. Ew.
1298             urlNote = reply.notes.findFirst("rewrite-url");
1299
1300             // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1301             if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1302                 // XXX: validate the URL properly *without* generating a whole new request object right here.
1303                 // XXX: the clone() should be done only AFTER we know the new URL is valid.
1304                 HttpRequest *new_request = old_request->clone();
1305                 if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1306                     debugs(61, 2, "URL-rewriter diverts URL from " << old_request->effectiveRequestUri() << " to " << new_request->effectiveRequestUri());
1307
1308                     // update the new request to flag the re-writing was done on it
1309                     new_request->flags.redirected = true;
1310
1311                     // unlink bodypipe from the old request. Not needed there any longer.
1312                     if (old_request->body_pipe != NULL) {
1313                         old_request->body_pipe = NULL;
1314                         debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1315                                " from request " << old_request << " to " << new_request);
1316                     }
1317
1318                     // update the current working ClientHttpRequest fields
1319                     xfree(http->uri);
1320                     http->uri = SBufToCstring(new_request->effectiveRequestUri());
1321                     HTTPMSGUNLOCK(old_request);
1322                     http->request = new_request;
1323                     HTTPMSGLOCK(http->request);
1324                 } else {
1325                     debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1326                            old_request->method << " " << urlNote << " " << old_request->http_ver);
1327                     delete new_request;
1328                 }
1329             }
1330         }
1331     }
1332     break;
1333     }
1334
1335     /* FIXME PIPELINE: This is innacurate during pipelining */
1336
1337     if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1338         fd_note(http->getConn()->clientConnection->fd, http->uri);
1339
1340     assert(http->uri);
1341
1342     http->doCallouts();
1343 }
1344
1345 /**
1346  * This method handles the different replies from StoreID helper.
1347  */
1348 void
1349 ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1350 {
1351     HttpRequest *old_request = http->request;
1352     debugs(85, 5, "'" << http->uri << "' result=" << reply);
1353     assert(store_id_state == REDIRECT_PENDING);
1354     store_id_state = REDIRECT_DONE;
1355
1356     // Put helper response Notes into the transaction state record (ALE) eventually
1357     // do it early to ensure that no matter what the outcome the notes are present.
1358     if (http->al != NULL)
1359         (void)SyncNotes(*http->al, *old_request);
1360
1361     UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1362
1363     switch (reply.result) {
1364     case Helper::Unknown:
1365     case Helper::TT:
1366         // Handler in redirect.cc should have already mapped Unknown
1367         // IF it contained valid entry for the old helper protocol
1368         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1369         break;
1370
1371     case Helper::TimedOut:
1372     // Timeouts for storeID are not implemented
1373     case Helper::BrokenHelper:
1374         debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply);
1375         break;
1376
1377     case Helper::Error:
1378         // no change to be done.
1379         break;
1380
1381     case Helper::Okay: {
1382         const char *urlNote = reply.notes.findFirst("store-id");
1383
1384         // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1385         if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1386             // Debug section required for some very specific cases.
1387             debugs(85, 9, "Setting storeID with: " << urlNote );
1388             http->request->store_id = urlNote;
1389             http->store_id = urlNote;
1390         }
1391     }
1392     break;
1393     }
1394
1395     http->doCallouts();
1396 }
1397
1398 /** Test cache allow/deny configuration
1399  *  Sets flags.cachable=1 if caching is not denied.
1400  */
1401 void
1402 ClientRequestContext::checkNoCache()
1403 {
1404     if (Config.accessList.noCache) {
1405         acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1406         acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1407     } else {
1408         /* unless otherwise specified, we try to cache. */
1409         checkNoCacheDone(ACCESS_ALLOWED);
1410     }
1411 }
1412
1413 static void
1414 checkNoCacheDoneWrapper(allow_t answer, void *data)
1415 {
1416     ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1417
1418     if (!calloutContext->httpStateIsValid())
1419         return;
1420
1421     calloutContext->checkNoCacheDone(answer);
1422 }
1423
1424 void
1425 ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1426 {
1427     acl_checklist = NULL;
1428     http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1429     http->doCallouts();
1430 }
1431
1432 #if USE_OPENSSL
1433 bool
1434 ClientRequestContext::sslBumpAccessCheck()
1435 {
1436     // If SSL connection tunneling or bumping decision has been made, obey it.
1437     const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1438     if (bumpMode != Ssl::bumpEnd) {
1439         debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1440                "), " << "ignoring ssl_bump for " << http->getConn());
1441         if (!http->getConn()->serverBump())
1442             http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1443         http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1444         return false;
1445     }
1446
1447     // If we have not decided yet, decide whether to bump now.
1448
1449     // Bumping here can only start with a CONNECT request on a bumping port
1450     // (bumping of intercepted SSL conns is decided before we get 1st request).
1451     // We also do not bump redirected CONNECT requests.
1452     if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1453             !Config.accessList.ssl_bump ||
1454             !http->getConn()->port->flags.tunnelSslBumping) {
1455         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1456         debugs(85, 5, HERE << "cannot SslBump this request");
1457         return false;
1458     }
1459
1460     // Do not bump during authentication: clients would not proxy-authenticate
1461     // if we delay a 407 response and respond with 200 OK to CONNECT.
1462     if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1463         http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1464         debugs(85, 5, HERE << "no SslBump during proxy authentication");
1465         return false;
1466     }
1467
1468     debugs(85, 5, HERE << "SslBump possible, checking ACL");
1469
1470     ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1471     aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1472     return true;
1473 }
1474
1475 /**
1476  * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1477  * as ACLFilledChecklist callback
1478  */
1479 static void
1480 sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1481 {
1482     ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1483
1484     if (!calloutContext->httpStateIsValid())
1485         return;
1486     calloutContext->sslBumpAccessCheckDone(answer);
1487 }
1488
1489 void
1490 ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1491 {
1492     if (!httpStateIsValid())
1493         return;
1494
1495     const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1496                                    static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1497     http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1498     http->al->ssl.bumpMode = bumpMode; // for logging
1499
1500     http->doCallouts();
1501 }
1502 #endif
1503
1504 /*
1505  * Identify requests that do not go through the store and client side stream
1506  * and forward them to the appropriate location. All other requests, request
1507  * them.
1508  */
1509 void
1510 ClientHttpRequest::processRequest()
1511 {
1512     debugs(85, 4, request->method << ' ' << uri);
1513
1514     if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1515 #if USE_OPENSSL
1516         if (sslBumpNeeded()) {
1517             sslBumpStart();
1518             return;
1519         }
1520 #endif
1521         getConn()->stopReading(); // tunnels read for themselves
1522         tunnelStart(this);
1523         return;
1524     }
1525
1526     httpStart();
1527 }
1528
1529 void
1530 ClientHttpRequest::httpStart()
1531 {
1532     PROF_start(httpStart);
1533     logType = LOG_TAG_NONE;
1534     debugs(85, 4, logType.c_str() << " for '" << uri << "'");
1535
1536     /* no one should have touched this */
1537     assert(out.offset == 0);
1538     /* Use the Stream Luke */
1539     clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1540     clientStreamRead(node, this, node->readBuffer);
1541     PROF_stop(httpStart);
1542 }
1543
1544 #if USE_OPENSSL
1545
1546 void
1547 ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1548 {
1549     debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1550     sslBumpNeed_ = mode;
1551 }
1552
1553 // called when comm_write has completed
1554 static void
1555 SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1556 {
1557     ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1558     debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1559
1560     assert(r && cbdataReferenceValid(r));
1561     r->sslBumpEstablish(errflag);
1562 }
1563
1564 void
1565 ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1566 {
1567     // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1568     if (errflag == Comm::ERR_CLOSING)
1569         return;
1570
1571     if (errflag) {
1572         debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1573         getConn()->clientConnection->close();
1574         return;
1575     }
1576
1577     // We lack HttpReply which logRequest() uses to log the status code.
1578     // TODO: Use HttpReply instead of the "200 Connection established" string.
1579     al->http.code = 200;
1580
1581 #if USE_AUTH
1582     // Preserve authentication info for the ssl-bumped request
1583     if (request->auth_user_request != NULL)
1584         getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1585 #endif
1586
1587     assert(sslBumpNeeded());
1588     getConn()->switchToHttps(request, sslBumpNeed_);
1589 }
1590
1591 void
1592 ClientHttpRequest::sslBumpStart()
1593 {
1594     debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1595            "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1596     getConn()->sslBumpMode = sslBumpNeed_;
1597
1598     AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1599                                   CommIoCbPtrFun(&SslBumpEstablish, this));
1600
1601     if (request->flags.interceptTproxy || request->flags.intercepted) {
1602         CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1603         params.flag = Comm::OK;
1604         params.conn = getConn()->clientConnection;
1605         ScheduleCallHere(bumpCall);
1606         return;
1607     }
1608
1609     // send an HTTP 200 response to kick client SSL negotiation
1610     // TODO: Unify with tunnel.cc and add a Server(?) header
1611     static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1612     Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1613 }
1614
1615 #endif
1616
1617 bool
1618 ClientHttpRequest::gotEnough() const
1619 {
1620     /** TODO: should be querying the stream. */
1621     int64_t contentLength =
1622         memObject()->getReply()->bodySize(request->method);
1623     assert(contentLength >= 0);
1624
1625     if (out.offset < contentLength)
1626         return false;
1627
1628     return true;
1629 }
1630
1631 void
1632 ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1633 {
1634     entry_ = newEntry;
1635 }
1636
1637 void
1638 ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1639 {
1640     if (loggingEntry_)
1641         loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1642
1643     loggingEntry_ = newEntry;
1644
1645     if (loggingEntry_)
1646         loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1647 }
1648
1649 /*
1650  * doCallouts() - This function controls the order of "callout"
1651  * executions, including non-blocking access control checks, the
1652  * redirector, and ICAP.  Previously, these callouts were chained
1653  * together such that "clientAccessCheckDone()" would call
1654  * "clientRedirectStart()" and so on.
1655  *
1656  * The ClientRequestContext (aka calloutContext) class holds certain
1657  * state data for the callout/callback operations.  Previously
1658  * ClientHttpRequest would sort of hand off control to ClientRequestContext
1659  * for a short time.  ClientRequestContext would then delete itself
1660  * and pass control back to ClientHttpRequest when all callouts
1661  * were finished.
1662  *
1663  * This caused some problems for ICAP because we want to make the
1664  * ICAP callout after checking ACLs, but before checking the no_cache
1665  * list.  We can't stuff the ICAP state into the ClientRequestContext
1666  * class because we still need the ICAP state after ClientRequestContext
1667  * goes away.
1668  *
1669  * Note that ClientRequestContext is created before the first call
1670  * to doCallouts().
1671  *
1672  * If one of the callouts notices that ClientHttpRequest is no
1673  * longer valid, it should call cbdataReferenceDone() so that
1674  * ClientHttpRequest's reference count goes to zero and it will get
1675  * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1676  *
1677  * Note that we set the _done flags here before actually starting
1678  * the callout.  This is strictly for convenience.
1679  */
1680
1681 tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1682 nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1683
1684 void
1685 ClientHttpRequest::doCallouts()
1686 {
1687     assert(calloutContext);
1688
1689     /*Save the original request for logging purposes*/
1690     if (!calloutContext->http->al->request) {
1691         calloutContext->http->al->request = request;
1692         HTTPMSGLOCK(calloutContext->http->al->request);
1693
1694         NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1695         // Make the previously set client connection ID available as annotation.
1696         if (ConnStateData *csd = calloutContext->http->getConn()) {
1697             if (!csd->connectionTag().isEmpty())
1698                 notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1699         }
1700     }
1701
1702     if (!calloutContext->error) {
1703         // CVE-2009-0801: verify the Host: header is consistent with other known details.
1704         if (!calloutContext->host_header_verify_done) {
1705             debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1706             calloutContext->host_header_verify_done = true;
1707             calloutContext->hostHeaderVerify();
1708             return;
1709         }
1710
1711         if (!calloutContext->http_access_done) {
1712             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1713             calloutContext->http_access_done = true;
1714             calloutContext->clientAccessCheck();
1715             return;
1716         }
1717
1718 #if USE_ADAPTATION
1719         if (!calloutContext->adaptation_acl_check_done) {
1720             calloutContext->adaptation_acl_check_done = true;
1721             if (Adaptation::AccessCheck::Start(
1722                         Adaptation::methodReqmod, Adaptation::pointPreCache,
1723                         request, NULL, calloutContext->http->al, this))
1724                 return; // will call callback
1725         }
1726 #endif
1727
1728         if (!calloutContext->redirect_done) {
1729             calloutContext->redirect_done = true;
1730
1731             if (Config.Program.redirect) {
1732                 debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1733                 calloutContext->redirect_state = REDIRECT_PENDING;
1734                 calloutContext->clientRedirectStart();
1735                 return;
1736             }
1737         }
1738
1739         if (!calloutContext->adapted_http_access_done) {
1740             debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1741             calloutContext->adapted_http_access_done = true;
1742             calloutContext->clientAccessCheck2();
1743             return;
1744         }
1745
1746         if (!calloutContext->store_id_done) {
1747             calloutContext->store_id_done = true;
1748
1749             if (Config.Program.store_id) {
1750                 debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1751                 calloutContext->store_id_state = REDIRECT_PENDING;
1752                 calloutContext->clientStoreIdStart();
1753                 return;
1754             }
1755         }
1756
1757         if (!calloutContext->interpreted_req_hdrs) {
1758             debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1759             calloutContext->interpreted_req_hdrs = 1;
1760             clientInterpretRequestHeaders(this);
1761         }
1762
1763         if (!calloutContext->no_cache_done) {
1764             calloutContext->no_cache_done = true;
1765
1766             if (Config.accessList.noCache && request->flags.cachable) {
1767                 debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1768                 calloutContext->checkNoCache();
1769                 return;
1770             }
1771         }
1772     } //  if !calloutContext->error
1773
1774     if (!calloutContext->tosToClientDone) {
1775         calloutContext->tosToClientDone = true;
1776         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1777             ACLFilledChecklist ch(NULL, request, NULL);
1778             ch.src_addr = request->client_addr;
1779             ch.my_addr = request->my_addr;
1780             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1781             if (tos)
1782                 Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1783         }
1784     }
1785
1786     if (!calloutContext->nfmarkToClientDone) {
1787         calloutContext->nfmarkToClientDone = true;
1788         if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1789             ACLFilledChecklist ch(NULL, request, NULL);
1790             ch.src_addr = request->client_addr;
1791             ch.my_addr = request->my_addr;
1792             nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1793             if (mark)
1794                 Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1795         }
1796     }
1797
1798 #if USE_OPENSSL
1799     // We need to check for SslBump even if the calloutContext->error is set
1800     // because bumping may require delaying the error until after CONNECT.
1801     if (!calloutContext->sslBumpCheckDone) {
1802         calloutContext->sslBumpCheckDone = true;
1803         if (calloutContext->sslBumpAccessCheck())
1804             return;
1805         /* else no ssl bump required*/
1806     }
1807 #endif
1808
1809     if (calloutContext->error) {
1810         // XXX: prformance regression. c_str() reallocates
1811         SBuf storeUri(request->storeId());
1812         StoreEntry *e = storeCreateEntry(storeUri.c_str(), storeUri.c_str(), request->flags, request->method);
1813 #if USE_OPENSSL
1814         if (sslBumpNeeded()) {
1815             // We have to serve an error, so bump the client first.
1816             sslBumpNeed(Ssl::bumpClientFirst);
1817             // set final error but delay sending until we bump
1818             Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1819             errorAppendEntry(e, calloutContext->error);
1820             calloutContext->error = NULL;
1821             getConn()->setServerBump(srvBump);
1822             e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1823         } else
1824 #endif
1825         {
1826             // send the error to the client now
1827             clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1828             clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1829             assert (repContext);
1830             repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1831             errorAppendEntry(e, calloutContext->error);
1832             calloutContext->error = NULL;
1833             if (calloutContext->readNextRequest)
1834                 getConn()->flags.readMore = true; // resume any pipeline reads.
1835             node = (clientStreamNode *)client_stream.tail->data;
1836             clientStreamRead(node, this, node->readBuffer);
1837             e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1838             return;
1839         }
1840     }
1841
1842     cbdataReferenceDone(calloutContext->http);
1843     delete calloutContext;
1844     calloutContext = NULL;
1845 #if HEADERS_LOG
1846
1847     headersLog(0, 1, request->method, request);
1848 #endif
1849
1850     debugs(83, 3, HERE << "calling processRequest()");
1851     processRequest();
1852
1853 #if ICAP_CLIENT
1854     Adaptation::Icap::History::Pointer ih = request->icapHistory();
1855     if (ih != NULL)
1856         ih->logType = logType;
1857 #endif
1858 }
1859
1860 #if !_USE_INLINE_
1861 #include "client_side_request.cci"
1862 #endif
1863
1864 #if USE_ADAPTATION
1865 /// Initiate an asynchronous adaptation transaction which will call us back.
1866 void
1867 ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1868 {
1869     debugs(85, 3, HERE << "adaptation needed for " << this);
1870     assert(!virginHeadSource);
1871     assert(!adaptedBodySource);
1872     virginHeadSource = initiateAdaptation(
1873                            new Adaptation::Iterator(request, NULL, al, g));
1874
1875     // we could try to guess whether we can bypass this adaptation
1876     // initiation failure, but it should not really happen
1877     Must(initiated(virginHeadSource));
1878 }
1879
1880 void
1881 ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1882 {
1883     assert(cbdataReferenceValid(this));     // indicates bug
1884     clearAdaptation(virginHeadSource);
1885     assert(!adaptedBodySource);
1886
1887     switch (answer.kind) {
1888     case Adaptation::Answer::akForward:
1889         handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1890         break;
1891
1892     case Adaptation::Answer::akBlock:
1893         handleAdaptationBlock(answer);
1894         break;
1895
1896     case Adaptation::Answer::akError:
1897         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1898         break;
1899     }
1900 }
1901
1902 void
1903 ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1904 {
1905     assert(msg);
1906
1907     if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1908         /*
1909          * Replace the old request with the new request.
1910          */
1911         HTTPMSGUNLOCK(request);
1912         request = new_req;
1913         HTTPMSGLOCK(request);
1914
1915         // update the new message to flag whether URL re-writing was done on it
1916         if (request->effectiveRequestUri().cmp(uri) != 0)
1917             request->flags.redirected = 1;
1918
1919         /*
1920          * Store the new URI for logging
1921          */
1922         xfree(uri);
1923         uri = SBufToCstring(request->effectiveRequestUri());
1924         setLogUri(this, urlCanonicalClean(request));
1925         assert(request->method.id());
1926     } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1927         debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1928
1929         // subscribe to receive reply body
1930         if (new_rep->body_pipe != NULL) {
1931             adaptedBodySource = new_rep->body_pipe;
1932             int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1933             assert(consumer_ok);
1934         }
1935
1936         clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1937         clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1938         assert(repContext);
1939         repContext->createStoreEntry(request->method, request->flags);
1940
1941         EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1942         request_satisfaction_mode = true;
1943         request_satisfaction_offset = 0;
1944         storeEntry()->replaceHttpReply(new_rep);
1945         storeEntry()->timestampsSet();
1946
1947         if (!adaptedBodySource) // no body
1948             storeEntry()->complete();
1949         clientGetMoreData(node, this);
1950     }
1951
1952     // we are done with getting headers (but may be receiving body)
1953     clearAdaptation(virginHeadSource);
1954
1955     if (!request_satisfaction_mode)
1956         doCallouts();
1957 }
1958
1959 void
1960 ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1961 {
1962     request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1963     AclMatchedName = answer.ruleId.termedBuf();
1964     assert(calloutContext);
1965     calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1966     AclMatchedName = NULL;
1967 }
1968
1969 void
1970 ClientHttpRequest::resumeBodyStorage()
1971 {
1972     if (!adaptedBodySource)
1973         return;
1974
1975     noteMoreBodyDataAvailable(adaptedBodySource);
1976 }
1977
1978 void
1979 ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1980 {
1981     assert(request_satisfaction_mode);
1982     assert(adaptedBodySource != NULL);
1983
1984     if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1985         const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1986
1987         if (spaceAvailable < contentSize ) {
1988             // No or partial body data consuming
1989             typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1990             AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1991                                                 Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1992             storeEntry()->deferProducer(call);
1993         }
1994
1995         if (!spaceAvailable)
1996             return;
1997
1998         if (spaceAvailable < contentSize )
1999             contentSize = spaceAvailable;
2000
2001         BodyPipeCheckout bpc(*adaptedBodySource);
2002         const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
2003         storeEntry()->write(ioBuf);
2004         // assume StoreEntry::write() writes the entire ioBuf
2005         request_satisfaction_offset += ioBuf.length;
2006         bpc.buf.consume(contentSize);
2007         bpc.checkIn();
2008     }
2009
2010     if (adaptedBodySource->exhausted())
2011         endRequestSatisfaction();
2012     // else wait for more body data
2013 }
2014
2015 void
2016 ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
2017 {
2018     assert(!virginHeadSource);
2019     // should we end request satisfaction now?
2020     if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
2021         endRequestSatisfaction();
2022 }
2023
2024 void
2025 ClientHttpRequest::endRequestSatisfaction()
2026 {
2027     debugs(85,4, HERE << this << " ends request satisfaction");
2028     assert(request_satisfaction_mode);
2029     stopConsumingFrom(adaptedBodySource);
2030
2031     // TODO: anything else needed to end store entry formation correctly?
2032     storeEntry()->complete();
2033 }
2034
2035 void
2036 ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2037 {
2038     assert(!virginHeadSource);
2039     stopConsumingFrom(adaptedBodySource);
2040
2041     debugs(85,3, HERE << "REQMOD body production failed");
2042     if (request_satisfaction_mode) { // too late to recover or serve an error
2043         request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2044         const Comm::ConnectionPointer c = getConn()->clientConnection;
2045         Must(Comm::IsConnOpen(c));
2046         c->close(); // drastic, but we may be writing a response already
2047     } else {
2048         handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2049     }
2050 }
2051
2052 void
2053 ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2054 {
2055     debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2056
2057     const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2058     const bool usedPipe = request->body_pipe != NULL &&
2059                           request->body_pipe->consumedSize() > 0;
2060
2061     if (bypassable && !usedStore && !usedPipe) {
2062         debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2063         if (calloutContext)
2064             doCallouts();
2065         return;
2066     }
2067
2068     debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2069
2070     clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2071     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2072     assert(repContext);
2073
2074     calloutsError(ERR_ICAP_FAILURE, errDetail);
2075
2076     if (calloutContext)
2077         doCallouts();
2078 }
2079
2080 #endif
2081
2082 // XXX: modify and use with ClientRequestContext::clientAccessCheckDone too.
2083 void
2084 ClientHttpRequest::calloutsError(const err_type error, const int errDetail)
2085 {
2086     // The original author of the code also wanted to pass an errno to
2087     // setReplyToError, but it seems unlikely that the errno reflects the
2088     // true cause of the error at this point, so I did not pass it.
2089     if (calloutContext) {
2090         Ip::Address noAddr;
2091         noAddr.setNoAddr();
2092         ConnStateData * c = getConn();
2093         calloutContext->error = clientBuildError(error, Http::scInternalServerError,
2094                                 NULL,
2095                                 c != NULL ? c->clientConnection->remote : noAddr,
2096                                 request
2097                                                 );
2098 #if USE_AUTH
2099         calloutContext->error->auth_user_request =
2100             c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2101 #endif
2102         calloutContext->error->detailError(errDetail);
2103         calloutContext->readNextRequest = true;
2104         if (c != NULL)
2105             c->expectNoForwarding();
2106     }
2107     //else if(calloutContext == NULL) is it possible?
2108 }
2109