src/FwdState.cc

   1 /*
   2  * DEBUG: section 17    Request Forwarding
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "AccessLogEntry.h"
  35 #include "acl/AclAddress.h"
  36 #include "acl/FilledChecklist.h"
  37 #include "acl/Gadgets.h"
  38 #include "anyp/PortCfg.h"
  39 #include "CacheManager.h"
  40 #include "CachePeer.h"
  41 #include "client_side.h"
  42 #include "comm/Connection.h"
  43 #include "comm/ConnOpener.h"
  44 #include "comm/Loops.h"
  45 #include "CommCalls.h"
  46 #include "errorpage.h"
  47 #include "event.h"
  48 #include "fd.h"
  49 #include "fde.h"
  50 #include "ftp.h"
  51 #include "FwdState.h"
  52 #include "globals.h"
  53 #include "gopher.h"
  54 #include "hier_code.h"
  55 #include "http.h"
  56 #include "HttpReply.h"
  57 #include "HttpRequest.h"
  58 #include "icmp/net_db.h"
  59 #include "internal.h"
  60 #include "ip/Intercept.h"
  61 #include "ip/QosConfig.h"
  62 #include "ip/tools.h"
  63 #include "MemObject.h"
  64 #include "mgr/Registration.h"
  65 #include "neighbors.h"
  66 #include "pconn.h"
  67 #include "PeerSelectState.h"
  68 #include "SquidConfig.h"
  69 #include "SquidTime.h"
  70 #include "Store.h"
  71 #include "StoreClient.h"
  72 #include "urn.h"
  73 #include "whois.h"
  74 #if USE_OPENSSL
  75 #include "ssl/cert_validate_message.h"
  76 #include "ssl/Config.h"
  77 #include "ssl/ErrorDetail.h"
  78 #include "ssl/helper.h"
  79 #include "ssl/PeerConnector.h"
  80 #include "ssl/ServerBump.h"
  81 #include "ssl/support.h"
  82 #endif
  83 #if HAVE_ERRNO_H
  84 #include <errno.h>
  85 #endif
  86
  87 static PSC fwdPeerSelectionCompleteWrapper;
  88 static CLCB fwdServerClosedWrapper;
  89 static CNCB fwdConnectDoneWrapper;
  90
  91 static OBJH fwdStats;
  92
  93 #define MAX_FWD_STATS_IDX 9
  94 static int FwdReplyCodes[MAX_FWD_STATS_IDX + 1][Http::scInvalidHeader + 1];
  95
  96 static PconnPool *fwdPconnPool = new PconnPool("server-side");
  97 CBDATA_CLASS_INIT(FwdState);
  98
  99 #if USE_OPENSSL
 100 class FwdStatePeerAnswerDialer: public CallDialer, public Ssl::PeerConnector::CbDialer
 101 {
 102 public:
 103     typedef void (FwdState::*Method)(Ssl::PeerConnectorAnswer &);
 104
 105     FwdStatePeerAnswerDialer(Method method, FwdState *fwd):
 106         method_(method), fwd_(fwd), answer_() {}
 107
 108     /* CallDialer API */
 109     virtual bool canDial(AsyncCall &call) { return fwd_.valid(); }
 110     void dial(AsyncCall &call) { ((&(*fwd_))->*method_)(answer_); }
 111     virtual void print(std::ostream &os) const {
 112         os << '(' << fwd_.get() << ", " << answer_ << ')'; }
 113
 114     /* Ssl::PeerConnector::CbDialer API */
 115     virtual Ssl::PeerConnectorAnswer &answer() { return answer_; }
 116
 117 private:
 118     Method method_;
 119     CbcPointer<FwdState> fwd_;
 120     Ssl::PeerConnectorAnswer answer_;
 121 };
 122 #endif
 123
 124
 125 void
 126 FwdState::abort(void* d)
 127 {
 128     FwdState* fwd = (FwdState*)d;
 129     Pointer tmp = fwd; // Grab a temporary pointer to keep the object alive during our scope.
 130
 131     if (Comm::IsConnOpen(fwd->serverConnection())) {
 132         comm_remove_close_handler(fwd->serverConnection()->fd, fwdServerClosedWrapper, fwd);
 133         debugs(17, 3, HERE << "store entry aborted; closing " <<
 134                fwd->serverConnection());
 135         fwd->serverConnection()->close();
 136     } else {
 137         debugs(17, 7, HERE << "store entry aborted; no connection to close");
 138     }
 139     fwd->serverDestinations.clear();
 140     fwd->self = NULL;
 141 }
 142
 143 /**** PUBLIC INTERFACE ********************************************************/
 144
 145 FwdState::FwdState(const Comm::ConnectionPointer &client, StoreEntry * e, HttpRequest * r, const AccessLogEntryPointer &alp):
 146         al(alp)
 147 {
 148     debugs(17, 2, HERE << "Forwarding client request " << client << ", url=" << e->url() );
 149     entry = e;
 150     clientConn = client;
 151     request = r;
 152     HTTPMSGLOCK(request);
 153     pconnRace = raceImpossible;
 154     start_t = squid_curtime;
 155     serverDestinations.reserve(Config.forward_max_tries);
 156     e->lock("FwdState");
 157     EBIT_SET(e->flags, ENTRY_FWD_HDR_WAIT);
 158 }
 159
 160 // Called once, right after object creation, when it is safe to set self
 161 void FwdState::start(Pointer aSelf)
 162 {
 163     // Protect ourselves from being destroyed when the only Server pointing
 164     // to us is gone (while we expect to talk to more Servers later).
 165     // Once we set self, we are responsible for clearing it when we do not
 166     // expect to talk to any servers.
 167     self = aSelf; // refcounted
 168
 169     // We hope that either the store entry aborts or peer is selected.
 170     // Otherwise we are going to leak our object.
 171
 172     entry->registerAbort(FwdState::abort, this);
 173
 174 #if STRICT_ORIGINAL_DST
 175     // Bug 3243: CVE 2009-0801
 176     // Bypass of browser same-origin access control in intercepted communication
 177     // To resolve this we must force DIRECT and only to the original client destination.
 178     const bool isIntercepted = request && !request->flags.redirected && (request->flags.intercepted || request->flags.interceptTproxy);
 179     const bool useOriginalDst = Config.onoff.client_dst_passthru || (request && !request->flags.hostVerified);
 180     if (isIntercepted && useOriginalDst) {
 181         selectPeerForIntercepted();
 182         // 3.2 does not suppro re-wrapping inside CONNECT.
 183         // our only alternative is to fake destination "found" and continue with the forwarding.
 184         startConnectionOrFail();
 185         return;
 186     }
 187 #endif
 188
 189     // do full route options selection
 190     peerSelect(&serverDestinations, request, al, entry, fwdPeerSelectionCompleteWrapper, this);
 191 }
 192
 193 #if STRICT_ORIGINAL_DST
 194 /// bypasses peerSelect() when dealing with intercepted requests
 195 void
 196 FwdState::selectPeerForIntercepted()
 197 {
 198     // use pinned connection if available
 199     Comm::ConnectionPointer p;
 200     if (ConnStateData *client = request->pinnedConnection()) {
 201         p = client->validatePinnedConnection(request, NULL);
 202         if (Comm::IsConnOpen(p)) {
 203             /* duplicate peerSelectPinned() effects */
 204             p->peerType = PINNED;
 205             entry->ping_status = PING_DONE;     /* Skip ICP */
 206
 207             debugs(17, 3, "reusing a pinned conn: " << *p);
 208             serverDestinations.push_back(p);
 209         } else {
 210             debugs(17,2, "Pinned connection is not valid: " << p);
 211             ErrorState *anErr = new ErrorState(ERR_ZERO_SIZE_OBJECT, Http::scServiceUnavailable, request);
 212             fail(anErr);
 213         }
 214         // Either use the valid pinned connection or fail if it is invalid.
 215         return;
 216     }
 217
 218     // use client original destination as second preferred choice
 219     p = new Comm::Connection();
 220     p->peerType = ORIGINAL_DST;
 221     p->remote = clientConn->local;
 222     getOutgoingAddress(request, p);
 223
 224     debugs(17, 3, HERE << "using client original destination: " << *p);
 225     serverDestinations.push_back(p);
 226 }
 227 #endif
 228
 229 void
 230 FwdState::completed()
 231 {
 232     if (flags.forward_completed) {
 233         debugs(17, DBG_IMPORTANT, HERE << "FwdState::completed called on a completed request! Bad!");
 234         return;
 235     }
 236
 237     flags.forward_completed = true;
 238
 239     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 240         debugs(17, 3, HERE << "entry aborted");
 241         return ;
 242     }
 243
 244 #if URL_CHECKSUM_DEBUG
 245
 246     entry->mem_obj->checkUrlChecksum();
 247 #endif
 248
 249     if (entry->store_status == STORE_PENDING) {
 250         if (entry->isEmpty()) {
 251             if (!err) // we quit (e.g., fd closed) before an error or content
 252                 fail(new ErrorState(ERR_READ_ERROR, Http::scBadGateway, request));
 253             assert(err);
 254             errorAppendEntry(entry, err);
 255             err = NULL;
 256 #if USE_OPENSSL
 257             if (request->flags.sslPeek && request->clientConnectionManager.valid()) {
 258                 CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
 259                              ConnStateData::httpsPeeked, Comm::ConnectionPointer(NULL));
 260             }
 261 #endif
 262         } else {
 263             EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 264             entry->complete();
 265             entry->releaseRequest();
 266         }
 267     }
 268
 269     if (storePendingNClients(entry) > 0)
 270         assert(!EBIT_TEST(entry->flags, ENTRY_FWD_HDR_WAIT));
 271
 272 }
 273
 274 FwdState::~FwdState()
 275 {
 276     debugs(17, 3, HERE << "FwdState destructor starting");
 277
 278     if (! flags.forward_completed)
 279         completed();
 280
 281     doneWithRetries();
 282
 283     HTTPMSGUNLOCK(request);
 284
 285     delete err;
 286
 287     entry->unregisterAbort();
 288
 289     entry->unlock("FwdState");
 290
 291     entry = NULL;
 292
 293     if (calls.connector != NULL) {
 294         calls.connector->cancel("FwdState destructed");
 295         calls.connector = NULL;
 296     }
 297
 298     if (Comm::IsConnOpen(serverConn)) {
 299         comm_remove_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
 300         debugs(17, 3, HERE << "closing FD " << serverConnection()->fd);
 301         serverConn->close();
 302     }
 303
 304     serverDestinations.clear();
 305
 306     debugs(17, 3, HERE << "FwdState destructor done");
 307 }
 308
 309 /**
 310  * This is the entry point for client-side to start forwarding
 311  * a transaction.  It is a static method that may or may not
 312  * allocate a FwdState.
 313  */
 314 void
 315 FwdState::Start(const Comm::ConnectionPointer &clientConn, StoreEntry *entry, HttpRequest *request, const AccessLogEntryPointer &al)
 316 {
 317     /** \note
 318      * client_addr == no_addr indicates this is an "internal" request
 319      * from peer_digest.c, asn.c, netdb.c, etc and should always
 320      * be allowed.  yuck, I know.
 321      */
 322
 323     if ( Config.accessList.miss && !request->client_addr.isNoAddr() &&
 324             request->protocol != AnyP::PROTO_INTERNAL && request->protocol != AnyP::PROTO_CACHE_OBJECT) {
 325         /**
 326          * Check if this host is allowed to fetch MISSES from us (miss_access).
 327          * Intentionally replace the src_addr automatically selected by the checklist code
 328          * we do NOT want the indirect client address to be tested here.
 329          */
 330         ACLFilledChecklist ch(Config.accessList.miss, request, NULL);
 331         ch.src_addr = request->client_addr;
 332         if (ch.fastCheck() == ACCESS_DENIED) {
 333             err_type page_id;
 334             page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
 335
 336             if (page_id == ERR_NONE)
 337                 page_id = ERR_FORWARDING_DENIED;
 338
 339             ErrorState *anErr = new ErrorState(page_id, Http::scForbidden, request);
 340             errorAppendEntry(entry, anErr);     // frees anErr
 341             return;
 342         }
 343     }
 344
 345     debugs(17, 3, HERE << "'" << entry->url() << "'");
 346     /*
 347      * This seems like an odd place to bind mem_obj and request.
 348      * Might want to assert that request is NULL at this point
 349      */
 350     entry->mem_obj->request = request;
 351     HTTPMSGLOCK(entry->mem_obj->request);
 352 #if URL_CHECKSUM_DEBUG
 353
 354     entry->mem_obj->checkUrlChecksum();
 355 #endif
 356
 357     if (shutting_down) {
 358         /* more yuck */
 359         ErrorState *anErr = new ErrorState(ERR_SHUTTING_DOWN, Http::scServiceUnavailable, request);
 360         errorAppendEntry(entry, anErr); // frees anErr
 361         return;
 362     }
 363
 364     switch (request->protocol) {
 365
 366     case AnyP::PROTO_INTERNAL:
 367         internalStart(clientConn, request, entry);
 368         return;
 369
 370     case AnyP::PROTO_CACHE_OBJECT:
 371         CacheManager::GetInstance()->Start(clientConn, request, entry);
 372         return;
 373
 374     case AnyP::PROTO_URN:
 375         urnStart(request, entry);
 376         return;
 377
 378     default:
 379         FwdState::Pointer fwd = new FwdState(clientConn, entry, request, al);
 380         fwd->start(fwd);
 381         return;
 382     }
 383
 384     /* NOTREACHED */
 385 }
 386
 387 void
 388 FwdState::fwdStart(const Comm::ConnectionPointer &clientConn, StoreEntry *entry, HttpRequest *request)
 389 {
 390     // Hides AccessLogEntry.h from code that does not supply ALE anyway.
 391     Start(clientConn, entry, request, NULL);
 392 }
 393
 394 void
 395 FwdState::startConnectionOrFail()
 396 {
 397     debugs(17, 3, HERE << entry->url());
 398
 399     if (serverDestinations.size() > 0) {
 400         // Ditch error page if it was created before.
 401         // A new one will be created if there's another problem
 402         delete err;
 403         err = NULL;
 404
 405         // Update the logging information about this new server connection.
 406         // Done here before anything else so the errors get logged for
 407         // this server link regardless of what happens when connecting to it.
 408         // IF sucessfuly connected this top destination will become the serverConnection().
 409         request->hier.note(serverDestinations[0], request->GetHost());
 410         request->clearError();
 411
 412         connectStart();
 413     } else {
 414         debugs(17, 3, HERE << "Connection failed: " << entry->url());
 415         if (!err) {
 416             ErrorState *anErr = new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request);
 417             fail(anErr);
 418         } // else use actual error from last connection attempt
 419         self = NULL;       // refcounted
 420     }
 421 }
 422
 423 void
 424 FwdState::fail(ErrorState * errorState)
 425 {
 426     debugs(17, 3, err_type_str[errorState->type] << " \"" << Http::StatusCodeString(errorState->httpStatus) << "\"\n\t" << entry->url());
 427
 428     delete err;
 429     err = errorState;
 430
 431     if (!errorState->request) {
 432         errorState->request = request;
 433         HTTPMSGLOCK(errorState->request);
 434     }
 435
 436     if (err->type != ERR_ZERO_SIZE_OBJECT)
 437         return;
 438
 439     if (pconnRace == racePossible) {
 440         debugs(17, 5, HERE << "pconn race happened");
 441         pconnRace = raceHappened;
 442     }
 443
 444     if (ConnStateData *pinned_connection = request->pinnedConnection()) {
 445         pinned_connection->pinning.zeroReply = true;
 446         flags.dont_retry = true; // we want to propagate failure to the client
 447         debugs(17, 4, "zero reply on pinned connection");
 448     }
 449 }
 450
 451 /**
 452  * Frees fwdState without closing FD or generating an abort
 453  */
 454 void
 455 FwdState::unregister(Comm::ConnectionPointer &conn)
 456 {
 457     debugs(17, 3, HERE << entry->url() );
 458     assert(serverConnection() == conn);
 459     assert(Comm::IsConnOpen(conn));
 460     comm_remove_close_handler(conn->fd, fwdServerClosedWrapper, this);
 461     serverConn = NULL;
 462 }
 463
 464 // Legacy method to be removed in favor of the above as soon as possible
 465 void
 466 FwdState::unregister(int fd)
 467 {
 468     debugs(17, 3, HERE << entry->url() );
 469     assert(fd == serverConnection()->fd);
 470     unregister(serverConn);
 471 }
 472
 473 /**
 474  * server-side modules call fwdComplete() when they are done
 475  * downloading an object.  Then, we either 1) re-forward the
 476  * request somewhere else if needed, or 2) call storeComplete()
 477  * to finish it off
 478  */
 479 void
 480 FwdState::complete()
 481 {
 482     debugs(17, 3, HERE << entry->url() << "\n\tstatus " << entry->getReply()->sline.status());
 483 #if URL_CHECKSUM_DEBUG
 484
 485     entry->mem_obj->checkUrlChecksum();
 486 #endif
 487
 488     logReplyStatus(n_tries, entry->getReply()->sline.status());
 489
 490     if (reforward()) {
 491         debugs(17, 3, HERE << "re-forwarding " << entry->getReply()->sline.status() << " " << entry->url());
 492
 493         if (Comm::IsConnOpen(serverConn))
 494             unregister(serverConn);
 495
 496         entry->reset();
 497
 498         // drop the last path off the selection list. try the next one.
 499         serverDestinations.erase(serverDestinations.begin());
 500         startConnectionOrFail();
 501
 502     } else {
 503         if (Comm::IsConnOpen(serverConn))
 504             debugs(17, 3, HERE << "server FD " << serverConnection()->fd << " not re-forwarding status " << entry->getReply()->sline.status());
 505         else
 506             debugs(17, 3, HERE << "server (FD closed) not re-forwarding status " << entry->getReply()->sline.status());
 507         EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 508         entry->complete();
 509
 510         if (!Comm::IsConnOpen(serverConn))
 511             completed();
 512
 513         self = NULL; // refcounted
 514     }
 515 }
 516
 517 /**** CALLBACK WRAPPERS ************************************************************/
 518
 519 static void
 520 fwdPeerSelectionCompleteWrapper(Comm::ConnectionList * unused, ErrorState *err, void *data)
 521 {
 522     FwdState *fwd = (FwdState *) data;
 523     if (err)
 524         fwd->fail(err);
 525     fwd->startConnectionOrFail();
 526 }
 527
 528 static void
 529 fwdServerClosedWrapper(const CommCloseCbParams &params)
 530 {
 531     FwdState *fwd = (FwdState *)params.data;
 532     fwd->serverClosed(params.fd);
 533 }
 534
 535 void
 536 fwdConnectDoneWrapper(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data)
 537 {
 538     FwdState *fwd = (FwdState *) data;
 539     fwd->connectDone(conn, status, xerrno);
 540 }
 541
 542 /**** PRIVATE *****************************************************************/
 543
 544 /*
 545  * FwdState::checkRetry
 546  *
 547  * Return TRUE if the request SHOULD be retried.  This method is
 548  * called when the HTTP connection fails, or when the connection
 549  * is closed before server-side read the end of HTTP headers.
 550  */
 551 bool
 552 FwdState::checkRetry()
 553 {
 554     if (shutting_down)
 555         return false;
 556
 557     if (!self) { // we have aborted before the server called us back
 558         debugs(17, 5, HERE << "not retrying because of earlier abort");
 559         // we will be destroyed when the server clears its Pointer to us
 560         return false;
 561     }
 562
 563     if (entry->store_status != STORE_PENDING)
 564         return false;
 565
 566     if (!entry->isEmpty())
 567         return false;
 568
 569     if (n_tries > Config.forward_max_tries)
 570         return false;
 571
 572     if (squid_curtime - start_t > Config.Timeout.forward)
 573         return false;
 574
 575     if (flags.dont_retry)
 576         return false;
 577
 578     if (request->bodyNibbled())
 579         return false;
 580
 581     // NP: not yet actually connected anywhere. retry is safe.
 582     if (!flags.connected_okay)
 583         return true;
 584
 585     if (!checkRetriable())
 586         return false;
 587
 588     return true;
 589 }
 590
 591 /*
 592  * FwdState::checkRetriable
 593  *
 594  * Return TRUE if this is the kind of request that can be retried
 595  * after a failure.  If the request is not retriable then we don't
 596  * want to risk sending it on a persistent connection.  Instead we'll
 597  * force it to go on a new HTTP connection.
 598  */
 599 bool
 600 FwdState::checkRetriable()
 601 {
 602     // Optimize: A compliant proxy may retry PUTs, but Squid lacks the [rather
 603     // complicated] code required to protect the PUT request body from being
 604     // nibbled during the first try. Thus, Squid cannot retry some PUTs today.
 605     if (request->body_pipe != NULL)
 606         return false;
 607
 608     // RFC2616 9.1 Safe and Idempotent Methods
 609     return (request->method.isHttpSafe() || request->method.isIdempotent());
 610 }
 611
 612 void
 613 FwdState::serverClosed(int fd)
 614 {
 615     debugs(17, 2, HERE << "FD " << fd << " " << entry->url());
 616     retryOrBail();
 617 }
 618
 619 void
 620 FwdState::retryOrBail()
 621 {
 622     if (checkRetry()) {
 623         debugs(17, 3, HERE << "re-forwarding (" << n_tries << " tries, " << (squid_curtime - start_t) << " secs)");
 624         // we should retry the same destination if it failed due to pconn race
 625         if (pconnRace == raceHappened)
 626             debugs(17, 4, HERE << "retrying the same destination");
 627         else
 628             serverDestinations.erase(serverDestinations.begin()); // last one failed. try another.
 629         startConnectionOrFail();
 630         return;
 631     }
 632
 633     // TODO: should we call completed() here and move doneWithRetries there?
 634     doneWithRetries();
 635
 636     if (self != NULL && !err && shutting_down) {
 637         ErrorState *anErr = new ErrorState(ERR_SHUTTING_DOWN, Http::scServiceUnavailable, request);
 638         errorAppendEntry(entry, anErr);
 639     }
 640
 641     self = NULL;        // refcounted
 642 }
 643
 644 // If the Server quits before nibbling at the request body, the body sender
 645 // will not know (so that we can retry). Call this if we will not retry. We
 646 // will notify the sender so that it does not get stuck waiting for space.
 647 void
 648 FwdState::doneWithRetries()
 649 {
 650     if (request && request->body_pipe != NULL)
 651         request->body_pipe->expectNoConsumption();
 652 }
 653
 654 // called by the server that failed after calling unregister()
 655 void
 656 FwdState::handleUnregisteredServerEnd()
 657 {
 658     debugs(17, 2, HERE << "self=" << self << " err=" << err << ' ' << entry->url());
 659     assert(!Comm::IsConnOpen(serverConn));
 660     retryOrBail();
 661 }
 662
 663 void
 664 FwdState::connectDone(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno)
 665 {
 666     if (status != COMM_OK) {
 667         ErrorState *const anErr = makeConnectingError(ERR_CONNECT_FAIL);
 668         anErr->xerrno = xerrno;
 669         fail(anErr);
 670
 671         /* it might have been a timeout with a partially open link */
 672         if (conn != NULL) {
 673             if (conn->getPeer())
 674                 peerConnectFailed(conn->getPeer());
 675
 676             conn->close();
 677         }
 678         retryOrBail();
 679         return;
 680     }
 681
 682     serverConn = conn;
 683     flags.connected_okay = true;
 684
 685     debugs(17, 3, HERE << serverConnection() << ": '" << entry->url() << "'" );
 686
 687     comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
 688
 689     if (serverConnection()->getPeer())
 690         peerConnectSucceded(serverConnection()->getPeer());
 691
 692 #if USE_OPENSSL
 693     if (!request->flags.pinned) {
 694         if ((serverConnection()->getPeer() && serverConnection()->getPeer()->use_ssl) ||
 695                 (!serverConnection()->getPeer() && request->protocol == AnyP::PROTO_HTTPS) ||
 696                 request->flags.sslPeek) {
 697
 698             HttpRequest::Pointer requestPointer = request;
 699             AsyncCall::Pointer callback = asyncCall(17,4,
 700                 "FwdState::ConnectedToPeer",
 701                 FwdStatePeerAnswerDialer(&FwdState::connectedToPeer, this));
 702             Ssl::PeerConnector *connector =
 703                 new Ssl::PeerConnector(requestPointer, serverConnection(), callback);
 704             AsyncJob::Start(connector); // will call our callback
 705             return;
 706         }
 707     }
 708 #endif
 709
 710     dispatch();
 711 }
 712
 713 #if USE_OPENSSL
 714 void
 715 FwdState::connectedToPeer(Ssl::PeerConnectorAnswer &answer)
 716 {
 717     if (ErrorState *error = answer.error.get()) {
 718         fail(error);
 719         answer.error.clear(); // preserve error for errorSendComplete()
 720         self = NULL;
 721         return;
 722     }
 723
 724     dispatch();
 725 }
 726 #endif
 727
 728 void
 729 FwdState::connectTimeout(int fd)
 730 {
 731     debugs(17, 2, "fwdConnectTimeout: FD " << fd << ": '" << entry->url() << "'" );
 732     assert(serverDestinations[0] != NULL);
 733     assert(fd == serverDestinations[0]->fd);
 734
 735     if (entry->isEmpty()) {
 736         ErrorState *anErr = new ErrorState(ERR_CONNECT_FAIL, Http::scGatewayTimeout, request);
 737         anErr->xerrno = ETIMEDOUT;
 738         fail(anErr);
 739
 740         /* This marks the peer DOWN ... */
 741         if (serverDestinations[0]->getPeer())
 742             peerConnectFailed(serverDestinations[0]->getPeer());
 743     }
 744
 745     if (Comm::IsConnOpen(serverDestinations[0])) {
 746         serverDestinations[0]->close();
 747     }
 748 }
 749
 750 /**
 751  * Called after Forwarding path selection (via peer select) has taken place.
 752  * And whenever forwarding needs to attempt a new connection (routing failover)
 753  * We have a vector of possible localIP->remoteIP paths now ready to start being connected.
 754  */
 755 void
 756 FwdState::connectStart()
 757 {
 758     assert(serverDestinations.size() > 0);
 759
 760     debugs(17, 3, "fwdConnectStart: " << entry->url());
 761
 762     if (!request->hier.first_conn_start.tv_sec) // first attempt
 763         request->hier.first_conn_start = current_time;
 764
 765     /* connection timeout */
 766     int ctimeout;
 767     if (serverDestinations[0]->getPeer()) {
 768         ctimeout = serverDestinations[0]->getPeer()->connect_timeout > 0 ?
 769                    serverDestinations[0]->getPeer()->connect_timeout : Config.Timeout.peer_connect;
 770     } else {
 771         ctimeout = Config.Timeout.connect;
 772     }
 773
 774     /* calculate total forwarding timeout ??? */
 775     int ftimeout = Config.Timeout.forward - (squid_curtime - start_t);
 776     if (ftimeout < 0)
 777         ftimeout = 5;
 778
 779     if (ftimeout < ctimeout)
 780         ctimeout = ftimeout;
 781
 782     if (serverDestinations[0]->getPeer() && request->flags.sslBumped) {
 783         debugs(50, 4, "fwdConnectStart: Ssl bumped connections through parent proxy are not allowed");
 784         ErrorState *anErr = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request);
 785         fail(anErr);
 786         self = NULL; // refcounted
 787         return;
 788     }
 789
 790     request->flags.pinned = false; // XXX: what if the ConnStateData set this to flag existing credentials?
 791     // XXX: answer: the peer selection *should* catch it and give us only the pinned peer. so we reverse the =0 step below.
 792     // XXX: also, logs will now lie if pinning is broken and leads to an error message.
 793     if (serverDestinations[0]->peerType == PINNED) {
 794         ConnStateData *pinned_connection = request->pinnedConnection();
 795         debugs(17,7, "pinned peer connection: " << pinned_connection);
 796         // pinned_connection may become nil after a pconn race
 797         if (pinned_connection)
 798             serverConn = pinned_connection->validatePinnedConnection(request, serverDestinations[0]->getPeer());
 799         else
 800             serverConn = NULL;
 801         if (Comm::IsConnOpen(serverConn)) {
 802             pinned_connection->stopPinnedConnectionMonitoring();
 803             flags.connected_okay = true;
 804             ++n_tries;
 805             request->flags.pinned = true;
 806             request->hier.note(serverConn, pinned_connection->pinning.host);
 807             if (pinned_connection->pinnedAuth())
 808                 request->flags.auth = true;
 809             comm_add_close_handler(serverConn->fd, fwdServerClosedWrapper, this);
 810             // the server may close the pinned connection before this request
 811             pconnRace = racePossible;
 812             dispatch();
 813             return;
 814         }
 815         // Pinned connection failure.
 816         debugs(17,2,HERE << "Pinned connection failed: " << pinned_connection);
 817         ErrorState *anErr = new ErrorState(ERR_ZERO_SIZE_OBJECT, Http::scServiceUnavailable, request);
 818         fail(anErr);
 819         self = NULL; // refcounted
 820         return;
 821     }
 822
 823     // Use pconn to avoid opening a new connection.
 824     const char *host = NULL;
 825     if (!serverDestinations[0]->getPeer())
 826         host = request->GetHost();
 827
 828     Comm::ConnectionPointer temp;
 829     // Avoid pconns after races so that the same client does not suffer twice.
 830     // This does not increase the total number of connections because we just
 831     // closed the connection that failed the race. And re-pinning assumes this.
 832     if (pconnRace != raceHappened)
 833         temp = fwdPconnPool->pop(serverDestinations[0], host, checkRetriable());
 834
 835     const bool openedPconn = Comm::IsConnOpen(temp);
 836     pconnRace = openedPconn ? racePossible : raceImpossible;
 837
 838     // if we found an open persistent connection to use. use it.
 839     if (openedPconn) {
 840         serverConn = temp;
 841         flags.connected_okay = true;
 842         debugs(17, 3, HERE << "reusing pconn " << serverConnection());
 843         ++n_tries;
 844
 845         comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
 846
 847         /* Update server side TOS and Netfilter mark on the connection. */
 848         if (Ip::Qos::TheConfig.isAclTosActive()) {
 849             const tos_t tos = GetTosToServer(request);
 850             Ip::Qos::setSockTos(temp, tos);
 851         }
 852 #if SO_MARK
 853         if (Ip::Qos::TheConfig.isAclNfmarkActive()) {
 854             const nfmark_t nfmark = GetNfmarkToServer(request);
 855             Ip::Qos::setSockNfmark(temp, nfmark);
 856         }
 857 #endif
 858
 859         dispatch();
 860         return;
 861     }
 862
 863     // We will try to open a new connection, possibly to the same destination.
 864     // We reset serverDestinations[0] in case we are using it again because
 865     // ConnOpener modifies its destination argument.
 866     serverDestinations[0]->local.port(0);
 867     serverConn = NULL;
 868
 869 #if URL_CHECKSUM_DEBUG
 870     entry->mem_obj->checkUrlChecksum();
 871 #endif
 872
 873     GetMarkingsToServer(request, *serverDestinations[0]);
 874
 875     calls.connector = commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper, this));
 876     Comm::ConnOpener *cs = new Comm::ConnOpener(serverDestinations[0], calls.connector, ctimeout);
 877     if (host)
 878         cs->setHost(host);
 879     AsyncJob::Start(cs);
 880 }
 881
 882 void
 883 FwdState::dispatch()
 884 {
 885     debugs(17, 3, clientConn << ": Fetching " << request->method << ' ' << entry->url());
 886     /*
 887      * Assert that server_fd is set.  This is to guarantee that fwdState
 888      * is attached to something and will be deallocated when server_fd
 889      * is closed.
 890      */
 891     assert(Comm::IsConnOpen(serverConn));
 892
 893     fd_note(serverConnection()->fd, entry->url());
 894
 895     fd_table[serverConnection()->fd].noteUse(fwdPconnPool);
 896
 897     /*assert(!EBIT_TEST(entry->flags, ENTRY_DISPATCHED)); */
 898     assert(entry->ping_status != PING_WAITING);
 899
 900     assert(entry->locked());
 901
 902     EBIT_SET(entry->flags, ENTRY_DISPATCHED);
 903
 904     netdbPingSite(request->GetHost());
 905
 906     /* Retrieves remote server TOS or MARK value, and stores it as part of the
 907      * original client request FD object. It is later used to forward
 908      * remote server's TOS/MARK in the response to the client in case of a MISS.
 909      */
 910     if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
 911         if (Comm::IsConnOpen(clientConn) && Comm::IsConnOpen(serverConnection())) {
 912             fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
 913             /* Get the netfilter mark for the connection */
 914             Ip::Qos::getNfmarkFromServer(serverConnection(), clientFde);
 915         }
 916     }
 917
 918 #if _SQUID_LINUX_
 919     /* Bug 2537: The TOS forward part of QOS only applies to patched Linux kernels. */
 920     if (Ip::Qos::TheConfig.isHitTosActive()) {
 921         if (Comm::IsConnOpen(clientConn)) {
 922             fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
 923             /* Get the TOS value for the packet */
 924             Ip::Qos::getTosFromServer(serverConnection(), clientFde);
 925         }
 926     }
 927 #endif
 928
 929 #if USE_OPENSSL
 930     if (request->flags.sslPeek) {
 931         CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
 932                      ConnStateData::httpsPeeked, serverConnection());
 933         unregister(serverConn); // async call owns it now
 934         complete(); // destroys us
 935         return;
 936     }
 937 #endif
 938
 939     if (serverConnection()->getPeer() != NULL) {
 940         ++ serverConnection()->getPeer()->stats.fetches;
 941         request->peer_login = serverConnection()->getPeer()->login;
 942         request->peer_domain = serverConnection()->getPeer()->domain;
 943         httpStart(this);
 944     } else {
 945         assert(!request->flags.sslPeek);
 946         request->peer_login = NULL;
 947         request->peer_domain = NULL;
 948
 949         switch (request->protocol) {
 950 #if USE_OPENSSL
 951
 952         case AnyP::PROTO_HTTPS:
 953             httpStart(this);
 954             break;
 955 #endif
 956
 957         case AnyP::PROTO_HTTP:
 958             httpStart(this);
 959             break;
 960
 961         case AnyP::PROTO_GOPHER:
 962             gopherStart(this);
 963             break;
 964
 965         case AnyP::PROTO_FTP:
 966             ftpStart(this);
 967             break;
 968
 969         case AnyP::PROTO_CACHE_OBJECT:
 970
 971         case AnyP::PROTO_INTERNAL:
 972
 973         case AnyP::PROTO_URN:
 974             fatal_dump("Should never get here");
 975             break;
 976
 977         case AnyP::PROTO_WHOIS:
 978             whoisStart(this);
 979             break;
 980
 981         case AnyP::PROTO_WAIS:  /* Not implemented */
 982
 983         default:
 984             debugs(17, DBG_IMPORTANT, "WARNING: Cannot retrieve '" << entry->url() << "'.");
 985             ErrorState *anErr = new ErrorState(ERR_UNSUP_REQ, Http::scBadRequest, request);
 986             fail(anErr);
 987             // Set the dont_retry flag because this is not a transient (network) error.
 988             flags.dont_retry = true;
 989             if (Comm::IsConnOpen(serverConn)) {
 990                 serverConn->close();
 991             }
 992             break;
 993         }
 994     }
 995 }
 996
 997 /*
 998  * FwdState::reforward
 999  *
1000  * returns TRUE if the transaction SHOULD be re-forwarded to the
1001  * next choice in the serverDestinations list.  This method is called when
1002  * server-side communication completes normally, or experiences
1003  * some error after receiving the end of HTTP headers.
1004  */
1005 int
1006 FwdState::reforward()
1007 {
1008     StoreEntry *e = entry;
1009
1010     if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
1011         debugs(17, 3, HERE << "entry aborted");
1012         return 0;
1013     }
1014
1015     assert(e->store_status == STORE_PENDING);
1016     assert(e->mem_obj);
1017 #if URL_CHECKSUM_DEBUG
1018
1019     e->mem_obj->checkUrlChecksum();
1020 #endif
1021
1022     debugs(17, 3, HERE << e->url() << "?" );
1023
1024     if (!EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) {
1025         debugs(17, 3, HERE << "No, ENTRY_FWD_HDR_WAIT isn't set");
1026         return 0;
1027     }
1028
1029     if (n_tries > Config.forward_max_tries)
1030         return 0;
1031
1032     if (request->bodyNibbled())
1033         return 0;
1034
1035     if (serverDestinations.size() <= 1) {
1036         // NP: <= 1 since total count includes the recently failed one.
1037         debugs(17, 3, HERE << "No alternative forwarding paths left");
1038         return 0;
1039     }
1040
1041     const Http::StatusCode s = e->getReply()->sline.status();
1042     debugs(17, 3, HERE << "status " << s);
1043     return reforwardableStatus(s);
1044 }
1045
1046 /**
1047  * Create "503 Service Unavailable" or "504 Gateway Timeout" error depending
1048  * on whether this is a validation request. RFC 2616 says that we MUST reply
1049  * with "504 Gateway Timeout" if validation fails and cached reply has
1050  * proxy-revalidate, must-revalidate or s-maxage Cache-Control directive.
1051  */
1052 ErrorState *
1053 FwdState::makeConnectingError(const err_type type) const
1054 {
1055     return new ErrorState(type, request->flags.needValidation ?
1056                           Http::scGatewayTimeout : Http::scServiceUnavailable, request);
1057 }
1058
1059 static void
1060 fwdStats(StoreEntry * s)
1061 {
1062     int i;
1063     int j;
1064     storeAppendPrintf(s, "Status");
1065
1066     for (j = 1; j < MAX_FWD_STATS_IDX; ++j) {
1067         storeAppendPrintf(s, "\ttry#%d", j);
1068     }
1069
1070     storeAppendPrintf(s, "\n");
1071
1072     for (i = 0; i <= (int) Http::scInvalidHeader; ++i) {
1073         if (FwdReplyCodes[0][i] == 0)
1074             continue;
1075
1076         storeAppendPrintf(s, "%3d", i);
1077
1078         for (j = 0; j <= MAX_FWD_STATS_IDX; ++j) {
1079             storeAppendPrintf(s, "\t%d", FwdReplyCodes[j][i]);
1080         }
1081
1082         storeAppendPrintf(s, "\n");
1083     }
1084 }
1085
1086 /**** STATIC MEMBER FUNCTIONS *************************************************/
1087
1088 bool
1089 FwdState::reforwardableStatus(const Http::StatusCode s) const
1090 {
1091     switch (s) {
1092
1093     case Http::scBadGateway:
1094
1095     case Http::scGatewayTimeout:
1096         return true;
1097
1098     case Http::scForbidden:
1099
1100     case Http::scInternalServerError:
1101
1102     case Http::scNotImplemented:
1103
1104     case Http::scServiceUnavailable:
1105         return Config.retry.onerror;
1106
1107     default:
1108         return false;
1109     }
1110
1111     /* NOTREACHED */
1112 }
1113
1114 /**
1115  * Decide where details need to be gathered to correctly describe a persistent connection.
1116  * What is needed:
1117  *  -  the address/port details about this link
1118  *  -  domain name of server at other end of this link (either peer or requested host)
1119  */
1120 void
1121 FwdState::pconnPush(Comm::ConnectionPointer &conn, const char *domain)
1122 {
1123     if (conn->getPeer()) {
1124         fwdPconnPool->push(conn, NULL);
1125     } else {
1126         fwdPconnPool->push(conn, domain);
1127     }
1128 }
1129
1130 void
1131 FwdState::initModule()
1132 {
1133     RegisterWithCacheManager();
1134 }
1135
1136 void
1137 FwdState::RegisterWithCacheManager(void)
1138 {
1139     Mgr::RegisterAction("forward", "Request Forwarding Statistics", fwdStats, 0, 1);
1140 }
1141
1142 void
1143 FwdState::logReplyStatus(int tries, const Http::StatusCode status)
1144 {
1145     if (status > Http::scInvalidHeader)
1146         return;
1147
1148     assert(tries >= 0);
1149
1150     if (tries > MAX_FWD_STATS_IDX)
1151         tries = MAX_FWD_STATS_IDX;
1152
1153     ++ FwdReplyCodes[tries][status];
1154 }
1155
1156 /**** PRIVATE NON-MEMBER FUNCTIONS ********************************************/
1157
1158 /*
1159  * DPW 2007-05-19
1160  * Formerly static, but now used by client_side_request.cc
1161  */
1162 /// Checks for a TOS value to apply depending on the ACL
1163 tos_t
1164 aclMapTOS(acl_tos * head, ACLChecklist * ch)
1165 {
1166     acl_tos *l;
1167
1168     for (l = head; l; l = l->next) {
1169         if (!l->aclList || ch->fastCheck(l->aclList) == ACCESS_ALLOWED)
1170             return l->tos;
1171     }
1172
1173     return 0;
1174 }
1175
1176 /// Checks for a netfilter mark value to apply depending on the ACL
1177 nfmark_t
1178 aclMapNfmark(acl_nfmark * head, ACLChecklist * ch)
1179 {
1180     acl_nfmark *l;
1181
1182     for (l = head; l; l = l->next) {
1183         if (!l->aclList || ch->fastCheck(l->aclList) == ACCESS_ALLOWED)
1184             return l->nfmark;
1185     }
1186
1187     return 0;
1188 }
1189
1190 void
1191 getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn)
1192 {
1193     // skip if an outgoing address is already set.
1194     if (!conn->local.isAnyAddr()) return;
1195
1196     // ensure that at minimum the wildcard local matches remote protocol
1197     if (conn->remote.isIPv4())
1198         conn->local.setIPv4();
1199
1200     // maybe use TPROXY client address
1201     if (request && request->flags.spoofClientIp) {
1202         if (!conn->getPeer() || !conn->getPeer()->options.no_tproxy) {
1203 #if FOLLOW_X_FORWARDED_FOR && LINUX_NETFILTER
1204             if (Config.onoff.tproxy_uses_indirect_client)
1205                 conn->local = request->indirect_client_addr;
1206             else
1207 #endif
1208                 conn->local = request->client_addr;
1209             // some flags need setting on the socket to use this address
1210             conn->flags |= COMM_DOBIND;
1211             conn->flags |= COMM_TRANSPARENT;
1212             return;
1213         }
1214         // else no tproxy today ...
1215     }
1216
1217     if (!Config.accessList.outgoing_address) {
1218         return; // anything will do.
1219     }
1220
1221     ACLFilledChecklist ch(NULL, request, NULL);
1222     ch.dst_peer = conn->getPeer();
1223     ch.dst_addr = conn->remote;
1224
1225     // TODO use the connection details in ACL.
1226     // needs a bit of rework in ACLFilledChecklist to use Comm::Connection instead of ConnStateData
1227
1228     AclAddress *l;
1229     for (l = Config.accessList.outgoing_address; l; l = l->next) {
1230
1231         /* check if the outgoing address is usable to the destination */
1232         if (conn->remote.isIPv4() != l->addr.isIPv4()) continue;
1233
1234         /* check ACLs for this outgoing address */
1235         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
1236             conn->local = l->addr;
1237             return;
1238         }
1239     }
1240 }
1241
1242 tos_t
1243 GetTosToServer(HttpRequest * request)
1244 {
1245     ACLFilledChecklist ch(NULL, request, NULL);
1246     return aclMapTOS(Ip::Qos::TheConfig.tosToServer, &ch);
1247 }
1248
1249 nfmark_t
1250 GetNfmarkToServer(HttpRequest * request)
1251 {
1252     ACLFilledChecklist ch(NULL, request, NULL);
1253     return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch);
1254 }
1255
1256 void
1257 GetMarkingsToServer(HttpRequest * request, Comm::Connection &conn)
1258 {
1259     // Get the server side TOS and Netfilter mark to be set on the connection.
1260     if (Ip::Qos::TheConfig.isAclTosActive()) {
1261         conn.tos = GetTosToServer(request);
1262         debugs(17, 3, "from " << conn.local << " tos " << int(conn.tos));
1263     }
1264
1265 #if SO_MARK && USE_LIBCAP
1266     conn.nfmark = GetNfmarkToServer(request);
1267     debugs(17, 3, "from " << conn.local << " netfilter mark " << conn.nfmark);
1268 #else
1269     conn.nfmark = 0;
1270 #endif
1271 }