src/FwdState.cc

   1 /*
   2  * DEBUG: section 17    Request Forwarding
   3  * AUTHOR: Duane Wessels
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "AccessLogEntry.h"
  35 #include "acl/AclAddress.h"
  36 #include "acl/FilledChecklist.h"
  37 #include "acl/Gadgets.h"
  38 #include "anyp/PortCfg.h"
  39 #include "CacheManager.h"
  40 #include "CachePeer.h"
  41 #include "client_side.h"
  42 #include "comm/Connection.h"
  43 #include "comm/ConnOpener.h"
  44 #include "comm/Loops.h"
  45 #include "CommCalls.h"
  46 #include "errorpage.h"
  47 #include "event.h"
  48 #include "fd.h"
  49 #include "fde.h"
  50 #include "ftp.h"
  51 #include "FwdState.h"
  52 #include "globals.h"
  53 #include "gopher.h"
  54 #include "hier_code.h"
  55 #include "http.h"
  56 #include "HttpReply.h"
  57 #include "HttpRequest.h"
  58 #include "icmp/net_db.h"
  59 #include "internal.h"
  60 #include "ip/Intercept.h"
  61 #include "ip/QosConfig.h"
  62 #include "ip/tools.h"
  63 #include "MemObject.h"
  64 #include "mgr/Registration.h"
  65 #include "neighbors.h"
  66 #include "pconn.h"
  67 #include "PeerSelectState.h"
  68 #include "SquidConfig.h"
  69 #include "SquidTime.h"
  70 #include "Store.h"
  71 #include "StoreClient.h"
  72 #include "urn.h"
  73 #include "whois.h"
  74 #if USE_OPENSSL
  75 #include "ssl/cert_validate_message.h"
  76 #include "ssl/Config.h"
  77 #include "ssl/ErrorDetail.h"
  78 #include "ssl/helper.h"
  79 #include "ssl/ServerBump.h"
  80 #include "ssl/support.h"
  81 #endif
  82 #if HAVE_ERRNO_H
  83 #include <errno.h>
  84 #endif
  85
  86 static PSC fwdPeerSelectionCompleteWrapper;
  87 static CLCB fwdServerClosedWrapper;
  88 #if USE_OPENSSL
  89 static PF fwdNegotiateSSLWrapper;
  90 #endif
  91 static CNCB fwdConnectDoneWrapper;
  92
  93 static OBJH fwdStats;
  94
  95 #define MAX_FWD_STATS_IDX 9
  96 static int FwdReplyCodes[MAX_FWD_STATS_IDX + 1][Http::scInvalidHeader + 1];
  97
  98 static PconnPool *fwdPconnPool = new PconnPool("server-side");
  99 CBDATA_CLASS_INIT(FwdState);
 100
 101 void
 102 FwdState::abort(void* d)
 103 {
 104     FwdState* fwd = (FwdState*)d;
 105     Pointer tmp = fwd; // Grab a temporary pointer to keep the object alive during our scope.
 106
 107     if (Comm::IsConnOpen(fwd->serverConnection())) {
 108         comm_remove_close_handler(fwd->serverConnection()->fd, fwdServerClosedWrapper, fwd);
 109         debugs(17, 3, HERE << "store entry aborted; closing " <<
 110                fwd->serverConnection());
 111         fwd->serverConnection()->close();
 112     } else {
 113         debugs(17, 7, HERE << "store entry aborted; no connection to close");
 114     }
 115     fwd->serverDestinations.clear();
 116     fwd->self = NULL;
 117 }
 118
 119 /**** PUBLIC INTERFACE ********************************************************/
 120
 121 FwdState::FwdState(const Comm::ConnectionPointer &client, StoreEntry * e, HttpRequest * r, const AccessLogEntryPointer &alp):
 122         al(alp)
 123 {
 124     debugs(17, 2, HERE << "Forwarding client request " << client << ", url=" << e->url() );
 125     entry = e;
 126     clientConn = client;
 127     request = r;
 128     HTTPMSGLOCK(request);
 129     pconnRace = raceImpossible;
 130     start_t = squid_curtime;
 131     serverDestinations.reserve(Config.forward_max_tries);
 132     e->lock("FwdState");
 133     EBIT_SET(e->flags, ENTRY_FWD_HDR_WAIT);
 134 }
 135
 136 // Called once, right after object creation, when it is safe to set self
 137 void FwdState::start(Pointer aSelf)
 138 {
 139     // Protect ourselves from being destroyed when the only Server pointing
 140     // to us is gone (while we expect to talk to more Servers later).
 141     // Once we set self, we are responsible for clearing it when we do not
 142     // expect to talk to any servers.
 143     self = aSelf; // refcounted
 144
 145     // We hope that either the store entry aborts or peer is selected.
 146     // Otherwise we are going to leak our object.
 147
 148     entry->registerAbort(FwdState::abort, this);
 149
 150 #if STRICT_ORIGINAL_DST
 151     // Bug 3243: CVE 2009-0801
 152     // Bypass of browser same-origin access control in intercepted communication
 153     // To resolve this we must force DIRECT and only to the original client destination.
 154     const bool isIntercepted = request && !request->flags.redirected && (request->flags.intercepted || request->flags.interceptTproxy);
 155     const bool useOriginalDst = Config.onoff.client_dst_passthru || (request && !request->flags.hostVerified);
 156     if (isIntercepted && useOriginalDst) {
 157         selectPeerForIntercepted();
 158         // 3.2 does not suppro re-wrapping inside CONNECT.
 159         // our only alternative is to fake destination "found" and continue with the forwarding.
 160         startConnectionOrFail();
 161         return;
 162     }
 163 #endif
 164
 165     // do full route options selection
 166     peerSelect(&serverDestinations, request, al, entry, fwdPeerSelectionCompleteWrapper, this);
 167 }
 168
 169 #if STRICT_ORIGINAL_DST
 170 /// bypasses peerSelect() when dealing with intercepted requests
 171 void
 172 FwdState::selectPeerForIntercepted()
 173 {
 174     // use pinned connection if available
 175     Comm::ConnectionPointer p;
 176     if (ConnStateData *client = request->pinnedConnection()) {
 177         p = client->validatePinnedConnection(request, NULL);
 178         if (Comm::IsConnOpen(p)) {
 179             /* duplicate peerSelectPinned() effects */
 180             p->peerType = PINNED;
 181             entry->ping_status = PING_DONE;     /* Skip ICP */
 182
 183             debugs(17, 3, "reusing a pinned conn: " << *p);
 184             serverDestinations.push_back(p);
 185         } else {
 186             debugs(17,2, "Pinned connection is not valid: " << p);
 187             ErrorState *anErr = new ErrorState(ERR_ZERO_SIZE_OBJECT, Http::scServiceUnavailable, request);
 188             fail(anErr);
 189         }
 190         // Either use the valid pinned connection or fail if it is invalid.
 191         return;
 192     }
 193
 194     // use client original destination as second preferred choice
 195     p = new Comm::Connection();
 196     p->peerType = ORIGINAL_DST;
 197     p->remote = clientConn->local;
 198     getOutgoingAddress(request, p);
 199
 200     debugs(17, 3, HERE << "using client original destination: " << *p);
 201     serverDestinations.push_back(p);
 202 }
 203 #endif
 204
 205 void
 206 FwdState::completed()
 207 {
 208     if (flags.forward_completed) {
 209         debugs(17, DBG_IMPORTANT, HERE << "FwdState::completed called on a completed request! Bad!");
 210         return;
 211     }
 212
 213     flags.forward_completed = true;
 214
 215     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 216         debugs(17, 3, HERE << "entry aborted");
 217         return ;
 218     }
 219
 220 #if URL_CHECKSUM_DEBUG
 221
 222     entry->mem_obj->checkUrlChecksum();
 223 #endif
 224
 225     if (entry->store_status == STORE_PENDING) {
 226         if (entry->isEmpty()) {
 227             if (!err) // we quit (e.g., fd closed) before an error or content
 228                 fail(new ErrorState(ERR_READ_ERROR, Http::scBadGateway, request));
 229             assert(err);
 230             errorAppendEntry(entry, err);
 231             err = NULL;
 232 #if USE_OPENSSL
 233             if (request->flags.sslPeek && request->clientConnectionManager.valid()) {
 234                 CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
 235                              ConnStateData::httpsPeeked, Comm::ConnectionPointer(NULL));
 236             }
 237 #endif
 238         } else {
 239             EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 240             entry->complete();
 241             entry->releaseRequest();
 242         }
 243     }
 244
 245     if (storePendingNClients(entry) > 0)
 246         assert(!EBIT_TEST(entry->flags, ENTRY_FWD_HDR_WAIT));
 247
 248 }
 249
 250 FwdState::~FwdState()
 251 {
 252     debugs(17, 3, HERE << "FwdState destructor starting");
 253
 254     if (! flags.forward_completed)
 255         completed();
 256
 257     doneWithRetries();
 258
 259     HTTPMSGUNLOCK(request);
 260
 261     delete err;
 262
 263     entry->unregisterAbort();
 264
 265     entry->unlock("FwdState");
 266
 267     entry = NULL;
 268
 269     if (calls.connector != NULL) {
 270         calls.connector->cancel("FwdState destructed");
 271         calls.connector = NULL;
 272     }
 273
 274     if (Comm::IsConnOpen(serverConn)) {
 275         comm_remove_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
 276         debugs(17, 3, HERE << "closing FD " << serverConnection()->fd);
 277         serverConn->close();
 278     }
 279
 280     serverDestinations.clear();
 281
 282     debugs(17, 3, HERE << "FwdState destructor done");
 283 }
 284
 285 /**
 286  * This is the entry point for client-side to start forwarding
 287  * a transaction.  It is a static method that may or may not
 288  * allocate a FwdState.
 289  */
 290 void
 291 FwdState::Start(const Comm::ConnectionPointer &clientConn, StoreEntry *entry, HttpRequest *request, const AccessLogEntryPointer &al)
 292 {
 293     /** \note
 294      * client_addr == no_addr indicates this is an "internal" request
 295      * from peer_digest.c, asn.c, netdb.c, etc and should always
 296      * be allowed.  yuck, I know.
 297      */
 298
 299     if ( Config.accessList.miss && !request->client_addr.isNoAddr() &&
 300             request->protocol != AnyP::PROTO_INTERNAL && request->protocol != AnyP::PROTO_CACHE_OBJECT) {
 301         /**
 302          * Check if this host is allowed to fetch MISSES from us (miss_access).
 303          * Intentionally replace the src_addr automatically selected by the checklist code
 304          * we do NOT want the indirect client address to be tested here.
 305          */
 306         ACLFilledChecklist ch(Config.accessList.miss, request, NULL);
 307         ch.src_addr = request->client_addr;
 308         if (ch.fastCheck() == ACCESS_DENIED) {
 309             err_type page_id;
 310             page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
 311
 312             if (page_id == ERR_NONE)
 313                 page_id = ERR_FORWARDING_DENIED;
 314
 315             ErrorState *anErr = new ErrorState(page_id, Http::scForbidden, request);
 316             errorAppendEntry(entry, anErr);     // frees anErr
 317             return;
 318         }
 319     }
 320
 321     debugs(17, 3, HERE << "'" << entry->url() << "'");
 322     /*
 323      * This seems like an odd place to bind mem_obj and request.
 324      * Might want to assert that request is NULL at this point
 325      */
 326     entry->mem_obj->request = request;
 327     HTTPMSGLOCK(entry->mem_obj->request);
 328 #if URL_CHECKSUM_DEBUG
 329
 330     entry->mem_obj->checkUrlChecksum();
 331 #endif
 332
 333     if (shutting_down) {
 334         /* more yuck */
 335         ErrorState *anErr = new ErrorState(ERR_SHUTTING_DOWN, Http::scServiceUnavailable, request);
 336         errorAppendEntry(entry, anErr); // frees anErr
 337         return;
 338     }
 339
 340     switch (request->protocol) {
 341
 342     case AnyP::PROTO_INTERNAL:
 343         internalStart(clientConn, request, entry);
 344         return;
 345
 346     case AnyP::PROTO_CACHE_OBJECT:
 347         CacheManager::GetInstance()->Start(clientConn, request, entry);
 348         return;
 349
 350     case AnyP::PROTO_URN:
 351         urnStart(request, entry);
 352         return;
 353
 354     default:
 355         FwdState::Pointer fwd = new FwdState(clientConn, entry, request, al);
 356         fwd->start(fwd);
 357         return;
 358     }
 359
 360     /* NOTREACHED */
 361 }
 362
 363 void
 364 FwdState::fwdStart(const Comm::ConnectionPointer &clientConn, StoreEntry *entry, HttpRequest *request)
 365 {
 366     // Hides AccessLogEntry.h from code that does not supply ALE anyway.
 367     Start(clientConn, entry, request, NULL);
 368 }
 369
 370 void
 371 FwdState::startConnectionOrFail()
 372 {
 373     debugs(17, 3, HERE << entry->url());
 374
 375     if (serverDestinations.size() > 0) {
 376         // Ditch error page if it was created before.
 377         // A new one will be created if there's another problem
 378         delete err;
 379         err = NULL;
 380
 381         // Update the logging information about this new server connection.
 382         // Done here before anything else so the errors get logged for
 383         // this server link regardless of what happens when connecting to it.
 384         // IF sucessfuly connected this top destination will become the serverConnection().
 385         request->hier.note(serverDestinations[0], request->GetHost());
 386         request->clearError();
 387
 388         connectStart();
 389     } else {
 390         debugs(17, 3, HERE << "Connection failed: " << entry->url());
 391         if (!err) {
 392             ErrorState *anErr = new ErrorState(ERR_CANNOT_FORWARD, Http::scInternalServerError, request);
 393             fail(anErr);
 394         } // else use actual error from last connection attempt
 395         self = NULL;       // refcounted
 396     }
 397 }
 398
 399 void
 400 FwdState::fail(ErrorState * errorState)
 401 {
 402     debugs(17, 3, err_type_str[errorState->type] << " \"" << Http::StatusCodeString(errorState->httpStatus) << "\"\n\t" << entry->url());
 403
 404     delete err;
 405     err = errorState;
 406
 407     if (!errorState->request) {
 408         errorState->request = request;
 409         HTTPMSGLOCK(errorState->request);
 410     }
 411
 412     if (err->type != ERR_ZERO_SIZE_OBJECT)
 413         return;
 414
 415     if (pconnRace == racePossible) {
 416         debugs(17, 5, HERE << "pconn race happened");
 417         pconnRace = raceHappened;
 418     }
 419
 420     if (ConnStateData *pinned_connection = request->pinnedConnection()) {
 421         pinned_connection->pinning.zeroReply = true;
 422         flags.dont_retry = true; // we want to propagate failure to the client
 423         debugs(17, 4, "zero reply on pinned connection");
 424     }
 425 }
 426
 427 /**
 428  * Frees fwdState without closing FD or generating an abort
 429  */
 430 void
 431 FwdState::unregister(Comm::ConnectionPointer &conn)
 432 {
 433     debugs(17, 3, HERE << entry->url() );
 434     assert(serverConnection() == conn);
 435     assert(Comm::IsConnOpen(conn));
 436     comm_remove_close_handler(conn->fd, fwdServerClosedWrapper, this);
 437     serverConn = NULL;
 438 }
 439
 440 // Legacy method to be removed in favor of the above as soon as possible
 441 void
 442 FwdState::unregister(int fd)
 443 {
 444     debugs(17, 3, HERE << entry->url() );
 445     assert(fd == serverConnection()->fd);
 446     unregister(serverConn);
 447 }
 448
 449 /**
 450  * server-side modules call fwdComplete() when they are done
 451  * downloading an object.  Then, we either 1) re-forward the
 452  * request somewhere else if needed, or 2) call storeComplete()
 453  * to finish it off
 454  */
 455 void
 456 FwdState::complete()
 457 {
 458     debugs(17, 3, HERE << entry->url() << "\n\tstatus " << entry->getReply()->sline.status());
 459 #if URL_CHECKSUM_DEBUG
 460
 461     entry->mem_obj->checkUrlChecksum();
 462 #endif
 463
 464     logReplyStatus(n_tries, entry->getReply()->sline.status());
 465
 466     if (reforward()) {
 467         debugs(17, 3, HERE << "re-forwarding " << entry->getReply()->sline.status() << " " << entry->url());
 468
 469         if (Comm::IsConnOpen(serverConn))
 470             unregister(serverConn);
 471
 472         entry->reset();
 473
 474         // drop the last path off the selection list. try the next one.
 475         serverDestinations.erase(serverDestinations.begin());
 476         startConnectionOrFail();
 477
 478     } else {
 479         if (Comm::IsConnOpen(serverConn))
 480             debugs(17, 3, HERE << "server FD " << serverConnection()->fd << " not re-forwarding status " << entry->getReply()->sline.status());
 481         else
 482             debugs(17, 3, HERE << "server (FD closed) not re-forwarding status " << entry->getReply()->sline.status());
 483         EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 484         entry->complete();
 485
 486         if (!Comm::IsConnOpen(serverConn))
 487             completed();
 488
 489         self = NULL; // refcounted
 490     }
 491 }
 492
 493 /**** CALLBACK WRAPPERS ************************************************************/
 494
 495 static void
 496 fwdPeerSelectionCompleteWrapper(Comm::ConnectionList * unused, ErrorState *err, void *data)
 497 {
 498     FwdState *fwd = (FwdState *) data;
 499     if (err)
 500         fwd->fail(err);
 501     fwd->startConnectionOrFail();
 502 }
 503
 504 static void
 505 fwdServerClosedWrapper(const CommCloseCbParams &params)
 506 {
 507     FwdState *fwd = (FwdState *)params.data;
 508     fwd->serverClosed(params.fd);
 509 }
 510
 511 #if USE_OPENSSL
 512 static void
 513 fwdNegotiateSSLWrapper(int fd, void *data)
 514 {
 515     FwdState *fwd = (FwdState *) data;
 516     fwd->negotiateSSL(fd);
 517 }
 518
 519 #endif
 520
 521 void
 522 fwdConnectDoneWrapper(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data)
 523 {
 524     FwdState *fwd = (FwdState *) data;
 525     fwd->connectDone(conn, status, xerrno);
 526 }
 527
 528 /**** PRIVATE *****************************************************************/
 529
 530 /*
 531  * FwdState::checkRetry
 532  *
 533  * Return TRUE if the request SHOULD be retried.  This method is
 534  * called when the HTTP connection fails, or when the connection
 535  * is closed before server-side read the end of HTTP headers.
 536  */
 537 bool
 538 FwdState::checkRetry()
 539 {
 540     if (shutting_down)
 541         return false;
 542
 543     if (!self) { // we have aborted before the server called us back
 544         debugs(17, 5, HERE << "not retrying because of earlier abort");
 545         // we will be destroyed when the server clears its Pointer to us
 546         return false;
 547     }
 548
 549     if (entry->store_status != STORE_PENDING)
 550         return false;
 551
 552     if (!entry->isEmpty())
 553         return false;
 554
 555     if (n_tries > Config.forward_max_tries)
 556         return false;
 557
 558     if (squid_curtime - start_t > Config.Timeout.forward)
 559         return false;
 560
 561     if (flags.dont_retry)
 562         return false;
 563
 564     if (request->bodyNibbled())
 565         return false;
 566
 567     // NP: not yet actually connected anywhere. retry is safe.
 568     if (!flags.connected_okay)
 569         return true;
 570
 571     if (!checkRetriable())
 572         return false;
 573
 574     return true;
 575 }
 576
 577 /*
 578  * FwdState::checkRetriable
 579  *
 580  * Return TRUE if this is the kind of request that can be retried
 581  * after a failure.  If the request is not retriable then we don't
 582  * want to risk sending it on a persistent connection.  Instead we'll
 583  * force it to go on a new HTTP connection.
 584  */
 585 bool
 586 FwdState::checkRetriable()
 587 {
 588     // Optimize: A compliant proxy may retry PUTs, but Squid lacks the [rather
 589     // complicated] code required to protect the PUT request body from being
 590     // nibbled during the first try. Thus, Squid cannot retry some PUTs today.
 591     if (request->body_pipe != NULL)
 592         return false;
 593
 594     // RFC2616 9.1 Safe and Idempotent Methods
 595     return (request->method.isHttpSafe() || request->method.isIdempotent());
 596 }
 597
 598 void
 599 FwdState::serverClosed(int fd)
 600 {
 601     debugs(17, 2, HERE << "FD " << fd << " " << entry->url());
 602     retryOrBail();
 603 }
 604
 605 void
 606 FwdState::retryOrBail()
 607 {
 608     if (checkRetry()) {
 609         debugs(17, 3, HERE << "re-forwarding (" << n_tries << " tries, " << (squid_curtime - start_t) << " secs)");
 610         // we should retry the same destination if it failed due to pconn race
 611         if (pconnRace == raceHappened)
 612             debugs(17, 4, HERE << "retrying the same destination");
 613         else
 614             serverDestinations.erase(serverDestinations.begin()); // last one failed. try another.
 615         startConnectionOrFail();
 616         return;
 617     }
 618
 619     // TODO: should we call completed() here and move doneWithRetries there?
 620     doneWithRetries();
 621
 622     if (self != NULL && !err && shutting_down) {
 623         ErrorState *anErr = new ErrorState(ERR_SHUTTING_DOWN, Http::scServiceUnavailable, request);
 624         errorAppendEntry(entry, anErr);
 625     }
 626
 627     self = NULL;        // refcounted
 628 }
 629
 630 // If the Server quits before nibbling at the request body, the body sender
 631 // will not know (so that we can retry). Call this if we will not retry. We
 632 // will notify the sender so that it does not get stuck waiting for space.
 633 void
 634 FwdState::doneWithRetries()
 635 {
 636     if (request && request->body_pipe != NULL)
 637         request->body_pipe->expectNoConsumption();
 638 }
 639
 640 // called by the server that failed after calling unregister()
 641 void
 642 FwdState::handleUnregisteredServerEnd()
 643 {
 644     debugs(17, 2, HERE << "self=" << self << " err=" << err << ' ' << entry->url());
 645     assert(!Comm::IsConnOpen(serverConn));
 646     retryOrBail();
 647 }
 648
 649 #if USE_OPENSSL
 650 void
 651 FwdState::negotiateSSL(int fd)
 652 {
 653     unsigned long ssl_lib_error = SSL_ERROR_NONE;
 654     SSL *ssl = fd_table[fd].ssl;
 655     int ret;
 656
 657     if ((ret = SSL_connect(ssl)) <= 0) {
 658         int ssl_error = SSL_get_error(ssl, ret);
 659 #ifdef EPROTO
 660         int sysErrNo = EPROTO;
 661 #else
 662         int sysErrNo = EACCES;
 663 #endif
 664
 665         switch (ssl_error) {
 666
 667         case SSL_ERROR_WANT_READ:
 668             Comm::SetSelect(fd, COMM_SELECT_READ, fwdNegotiateSSLWrapper, this, 0);
 669             return;
 670
 671         case SSL_ERROR_WANT_WRITE:
 672             Comm::SetSelect(fd, COMM_SELECT_WRITE, fwdNegotiateSSLWrapper, this, 0);
 673             return;
 674
 675         case SSL_ERROR_SSL:
 676         case SSL_ERROR_SYSCALL:
 677             ssl_lib_error = ERR_get_error();
 678             debugs(81, DBG_IMPORTANT, "fwdNegotiateSSL: Error negotiating SSL connection on FD " << fd <<
 679                    ": " << ERR_error_string(ssl_lib_error, NULL) << " (" << ssl_error <<
 680                    "/" << ret << "/" << errno << ")");
 681
 682             // store/report errno when ssl_error is SSL_ERROR_SYSCALL, ssl_lib_error is 0, and ret is -1
 683             if (ssl_error == SSL_ERROR_SYSCALL && ret == -1 && ssl_lib_error == 0)
 684                 sysErrNo = errno;
 685
 686             // falling through to complete error handling
 687
 688         default:
 689             // TODO: move into a method before merge
 690             Ssl::ErrorDetail *errDetails;
 691             Ssl::ErrorDetail *errFromFailure = (Ssl::ErrorDetail *)SSL_get_ex_data(ssl, ssl_ex_index_ssl_error_detail);
 692             if (errFromFailure != NULL) {
 693                 // The errFromFailure is attached to the ssl object
 694                 // and will be released when ssl object destroyed.
 695                 // Copy errFromFailure to a new Ssl::ErrorDetail object.
 696                 errDetails = new Ssl::ErrorDetail(*errFromFailure);
 697             } else {
 698                 // server_cert can be NULL here
 699                 X509 *server_cert = SSL_get_peer_certificate(ssl);
 700                 errDetails = new Ssl::ErrorDetail(SQUID_ERR_SSL_HANDSHAKE, server_cert, NULL);
 701                 X509_free(server_cert);
 702             }
 703
 704             if (ssl_lib_error != SSL_ERROR_NONE)
 705                 errDetails->setLibError(ssl_lib_error);
 706
 707             if (request->clientConnectionManager.valid()) {
 708                 // remember the server certificate from the ErrorDetail object
 709                 if (Ssl::ServerBump *serverBump = request->clientConnectionManager->serverBump()) {
 710                     serverBump->serverCert.resetAndLock(errDetails->peerCert());
 711
 712                     // remember validation errors, if any
 713                     if (Ssl::CertErrors *errs = static_cast<Ssl::CertErrors*>(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors)))
 714                         serverBump->sslErrors = cbdataReference(errs);
 715                 }
 716
 717                 // For intercepted connections, set the host name to the server
 718                 // certificate CN. Otherwise, we just hope that CONNECT is using
 719                 // a user-entered address (a host name or a user-entered IP).
 720                 const bool isConnectRequest = !request->clientConnectionManager->port->flags.isIntercepted();
 721                 if (request->flags.sslPeek && !isConnectRequest) {
 722                     if (X509 *srvX509 = errDetails->peerCert()) {
 723                         if (const char *name = Ssl::CommonHostName(srvX509)) {
 724                             request->SetHost(name);
 725                             debugs(83, 3, HERE << "reset request host: " << name);
 726                         }
 727                     }
 728                 }
 729             }
 730
 731             ErrorState *const anErr = makeConnectingError(ERR_SECURE_CONNECT_FAIL);
 732             anErr->xerrno = sysErrNo;
 733             anErr->detail = errDetails;
 734             fail(anErr);
 735
 736             if (serverConnection()->getPeer()) {
 737                 peerConnectFailed(serverConnection()->getPeer());
 738             }
 739
 740             serverConn->close();
 741             return;
 742         }
 743     }
 744
 745     if (request->clientConnectionManager.valid()) {
 746         // remember the server certificate from the ErrorDetail object
 747         if (Ssl::ServerBump *serverBump = request->clientConnectionManager->serverBump()) {
 748             serverBump->serverCert.reset(SSL_get_peer_certificate(ssl));
 749
 750             // remember validation errors, if any
 751             if (Ssl::CertErrors *errs = static_cast<Ssl::CertErrors *>(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors)))
 752                 serverBump->sslErrors = cbdataReference(errs);
 753         }
 754     }
 755
 756     if (serverConnection()->getPeer() && !SSL_session_reused(ssl)) {
 757         if (serverConnection()->getPeer()->sslSession)
 758             SSL_SESSION_free(serverConnection()->getPeer()->sslSession);
 759
 760         serverConnection()->getPeer()->sslSession = SSL_get1_session(ssl);
 761     }
 762
 763     if (Ssl::TheConfig.ssl_crt_validator) {
 764         Ssl::CertValidationRequest validationRequest;
 765         // WARNING: Currently we do not use any locking for any of the
 766         // members of the Ssl::CertValidationRequest class. In this code the
 767         // Ssl::CertValidationRequest object used only to pass data to
 768         // Ssl::CertValidationHelper::submit method.
 769         validationRequest.ssl = ssl;
 770         validationRequest.domainName = request->GetHost();
 771         if (Ssl::CertErrors *errs = static_cast<Ssl::CertErrors *>(SSL_get_ex_data(ssl, ssl_ex_index_ssl_errors)))
 772             // validationRequest disappears on return so no need to cbdataReference
 773             validationRequest.errors = errs;
 774         else
 775             validationRequest.errors = NULL;
 776         try {
 777             debugs(83, 5, "Sending SSL certificate for validation to ssl_crtvd.");
 778             Ssl::CertValidationHelper::GetInstance()->sslSubmit(validationRequest, sslCrtvdHandleReplyWrapper, this);
 779             return;
 780         } catch (const std::exception &e) {
 781             debugs(33, DBG_IMPORTANT, "ERROR: Failed to compose ssl_crtvd " <<
 782                    "request for " << validationRequest.domainName <<
 783                    " certificate: " << e.what() << "; will now block to " <<
 784                    "validate that certificate.");
 785             // fall through to do blocking in-process generation.
 786             ErrorState *anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request);
 787             fail(anErr);
 788             if (serverConnection()->getPeer()) {
 789                 peerConnectFailed(serverConnection()->getPeer());
 790             }
 791             serverConn->close();
 792             self = NULL;
 793             return;
 794         }
 795     }
 796
 797     dispatch();
 798 }
 799
 800 void
 801 FwdState::sslCrtvdHandleReplyWrapper(void *data, Ssl::CertValidationResponse const &validationResponse)
 802 {
 803     FwdState * fwd = (FwdState *)(data);
 804     fwd->sslCrtvdHandleReply(validationResponse);
 805 }
 806
 807 void
 808 FwdState::sslCrtvdHandleReply(Ssl::CertValidationResponse const &validationResponse)
 809 {
 810     Ssl::CertErrors *errs = NULL;
 811     Ssl::ErrorDetail *errDetails = NULL;
 812     bool validatorFailed = false;
 813     if (!Comm::IsConnOpen(serverConnection())) {
 814         return;
 815     }
 816
 817     debugs(83,5, request->GetHost() << " cert validation result: " << validationResponse.resultCode);
 818
 819     if (validationResponse.resultCode == HelperReply::Error)
 820         errs = sslCrtvdCheckForErrors(validationResponse, errDetails);
 821     else if (validationResponse.resultCode != HelperReply::Okay)
 822         validatorFailed = true;
 823
 824     if (!errDetails && !validatorFailed) {
 825         dispatch();
 826         return;
 827     }
 828
 829     ErrorState *anErr = NULL;
 830     if (validatorFailed) {
 831         anErr = new ErrorState(ERR_GATEWAY_FAILURE, Http::scInternalServerError, request);
 832     }  else {
 833
 834         // Check the list error with
 835         if (errDetails && request->clientConnectionManager.valid()) {
 836             // remember the server certificate from the ErrorDetail object
 837             if (Ssl::ServerBump *serverBump = request->clientConnectionManager->serverBump()) {
 838                 // remember validation errors, if any
 839                 if (errs) {
 840                     if (serverBump->sslErrors)
 841                         cbdataReferenceDone(serverBump->sslErrors);
 842                     serverBump->sslErrors = cbdataReference(errs);
 843                 }
 844             }
 845         }
 846
 847         anErr = makeConnectingError(ERR_SECURE_CONNECT_FAIL);
 848         anErr->detail = errDetails;
 849         /*anErr->xerrno= Should preserved*/
 850     }
 851
 852     fail(anErr);
 853     if (serverConnection()->getPeer()) {
 854         peerConnectFailed(serverConnection()->getPeer());
 855     }
 856     serverConn->close();
 857     self = NULL;
 858     return;
 859 }
 860
 861 /// Checks errors in the cert. validator response against sslproxy_cert_error.
 862 /// The first honored error, if any, is returned via errDetails parameter.
 863 /// The method returns all seen errors except SSL_ERROR_NONE as Ssl::CertErrors.
 864 Ssl::CertErrors *
 865 FwdState::sslCrtvdCheckForErrors(Ssl::CertValidationResponse const &resp, Ssl::ErrorDetail *& errDetails)
 866 {
 867     Ssl::CertErrors *errs = NULL;
 868
 869     ACLFilledChecklist *check = NULL;
 870     if (acl_access *acl = Config.ssl_client.cert_error)
 871         check = new ACLFilledChecklist(acl, request, dash_str);
 872
 873     SSL *ssl = fd_table[serverConnection()->fd].ssl;
 874     typedef Ssl::CertValidationResponse::RecvdErrors::const_iterator SVCRECI;
 875     for (SVCRECI i = resp.errors.begin(); i != resp.errors.end(); ++i) {
 876         debugs(83, 7, "Error item: " << i->error_no << " " << i->error_reason);
 877
 878         assert(i->error_no != SSL_ERROR_NONE);
 879
 880         if (!errDetails) {
 881             bool allowed = false;
 882             if (check) {
 883                 check->sslErrors = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get()));
 884                 if (check->fastCheck() == ACCESS_ALLOWED)
 885                     allowed = true;
 886             }
 887             // else the Config.ssl_client.cert_error access list is not defined
 888             // and the first error will cause the error page
 889
 890             if (allowed) {
 891                 debugs(83, 3, "bypassing SSL error " << i->error_no << " in " << "buffer");
 892             } else {
 893                 debugs(83, 5, "confirming SSL error " << i->error_no);
 894                 X509 *brokenCert = i->cert.get();
 895                 Ssl::X509_Pointer peerCert(SSL_get_peer_certificate(ssl));
 896                 const char *aReason = i->error_reason.empty() ? NULL : i->error_reason.c_str();
 897                 errDetails = new Ssl::ErrorDetail(i->error_no, peerCert.get(), brokenCert, aReason);
 898             }
 899             if (check) {
 900                 delete check->sslErrors;
 901                 check->sslErrors = NULL;
 902             }
 903         }
 904
 905         if (!errs)
 906             errs = new Ssl::CertErrors(Ssl::CertError(i->error_no, i->cert.get()));
 907         else
 908             errs->push_back_unique(Ssl::CertError(i->error_no, i->cert.get()));
 909     }
 910     if (check)
 911         delete check;
 912
 913     return errs;
 914 }
 915
 916 void
 917 FwdState::initiateSSL()
 918 {
 919     SSL *ssl;
 920     SSL_CTX *sslContext = NULL;
 921     const CachePeer *peer = serverConnection()->getPeer();
 922     int fd = serverConnection()->fd;
 923
 924     if (peer) {
 925         assert(peer->use_ssl);
 926         sslContext = peer->sslContext;
 927     } else {
 928         sslContext = Config.ssl_client.sslContext;
 929     }
 930
 931     assert(sslContext);
 932
 933     if ((ssl = SSL_new(sslContext)) == NULL) {
 934         debugs(83, DBG_IMPORTANT, "fwdInitiateSSL: Error allocating handle: " << ERR_error_string(ERR_get_error(), NULL)  );
 935         ErrorState *anErr = new ErrorState(ERR_SOCKET_FAILURE, Http::scInternalServerError, request);
 936         // TODO: create Ssl::ErrorDetail with OpenSSL-supplied error code
 937         fail(anErr);
 938         self = NULL;            // refcounted
 939         return;
 940     }
 941
 942     SSL_set_fd(ssl, fd);
 943
 944     if (peer) {
 945         if (peer->ssldomain)
 946             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->ssldomain);
 947
 948 #if NOT_YET
 949
 950         else if (peer->name)
 951             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->name);
 952
 953 #endif
 954
 955         else
 956             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->host);
 957
 958         if (peer->sslSession)
 959             SSL_set_session(ssl, peer->sslSession);
 960
 961     } else {
 962         // While we are peeking at the certificate, we may not know the server
 963         // name that the client will request (after interception or CONNECT)
 964         // unless it was the CONNECT request with a user-typed address.
 965         const char *hostname = request->GetHost();
 966         const bool hostnameIsIp = request->GetHostIsNumeric();
 967         const bool isConnectRequest = request->clientConnectionManager.valid() &&
 968                                       !request->clientConnectionManager->port->flags.isIntercepted();
 969         if (!request->flags.sslPeek || isConnectRequest)
 970             SSL_set_ex_data(ssl, ssl_ex_index_server, (void*)hostname);
 971
 972         // Use SNI TLS extension only when we connect directly
 973         // to the origin server and we know the server host name.
 974         if (!hostnameIsIp)
 975             Ssl::setClientSNI(ssl, hostname);
 976     }
 977
 978     // If CertValidation Helper used do not lookup checklist for errors,
 979     // but keep a list of errors to send it to CertValidator
 980     if (!Ssl::TheConfig.ssl_crt_validator) {
 981         // Create the ACL check list now, while we have access to more info.
 982         // The list is used in ssl_verify_cb() and is freed in ssl_free().
 983         if (acl_access *acl = Config.ssl_client.cert_error) {
 984             ACLFilledChecklist *check = new ACLFilledChecklist(acl, request, dash_str);
 985             SSL_set_ex_data(ssl, ssl_ex_index_cert_error_check, check);
 986         }
 987     }
 988
 989     // store peeked cert to check SQUID_X509_V_ERR_CERT_CHANGE
 990     X509 *peeked_cert;
 991     if (request->clientConnectionManager.valid() &&
 992             request->clientConnectionManager->serverBump() &&
 993             (peeked_cert = request->clientConnectionManager->serverBump()->serverCert.get())) {
 994         CRYPTO_add(&(peeked_cert->references),1,CRYPTO_LOCK_X509);
 995         SSL_set_ex_data(ssl, ssl_ex_index_ssl_peeked_cert, peeked_cert);
 996     }
 997
 998     fd_table[fd].ssl = ssl;
 999     fd_table[fd].read_method = &ssl_read_method;
1000     fd_table[fd].write_method = &ssl_write_method;
1001     negotiateSSL(fd);
1002 }
1003
1004 #endif
1005
1006 void
1007 FwdState::connectDone(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno)
1008 {
1009     if (status != COMM_OK) {
1010         ErrorState *const anErr = makeConnectingError(ERR_CONNECT_FAIL);
1011         anErr->xerrno = xerrno;
1012         fail(anErr);
1013
1014         /* it might have been a timeout with a partially open link */
1015         if (conn != NULL) {
1016             if (conn->getPeer())
1017                 peerConnectFailed(conn->getPeer());
1018
1019             conn->close();
1020         }
1021         retryOrBail();
1022         return;
1023     }
1024
1025     serverConn = conn;
1026     flags.connected_okay = true;
1027
1028     debugs(17, 3, HERE << serverConnection() << ": '" << entry->url() << "'" );
1029
1030     comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
1031
1032     if (serverConnection()->getPeer())
1033         peerConnectSucceded(serverConnection()->getPeer());
1034
1035 #if USE_OPENSSL
1036     if (!request->flags.pinned) {
1037         if ((serverConnection()->getPeer() && serverConnection()->getPeer()->use_ssl) ||
1038                 (!serverConnection()->getPeer() && request->protocol == AnyP::PROTO_HTTPS) ||
1039                 request->flags.sslPeek) {
1040             initiateSSL();
1041             return;
1042         }
1043     }
1044 #endif
1045
1046     dispatch();
1047 }
1048
1049 void
1050 FwdState::connectTimeout(int fd)
1051 {
1052     debugs(17, 2, "fwdConnectTimeout: FD " << fd << ": '" << entry->url() << "'" );
1053     assert(serverDestinations[0] != NULL);
1054     assert(fd == serverDestinations[0]->fd);
1055
1056     if (entry->isEmpty()) {
1057         ErrorState *anErr = new ErrorState(ERR_CONNECT_FAIL, Http::scGatewayTimeout, request);
1058         anErr->xerrno = ETIMEDOUT;
1059         fail(anErr);
1060
1061         /* This marks the peer DOWN ... */
1062         if (serverDestinations[0]->getPeer())
1063             peerConnectFailed(serverDestinations[0]->getPeer());
1064     }
1065
1066     if (Comm::IsConnOpen(serverDestinations[0])) {
1067         serverDestinations[0]->close();
1068     }
1069 }
1070
1071 /**
1072  * Called after Forwarding path selection (via peer select) has taken place.
1073  * And whenever forwarding needs to attempt a new connection (routing failover)
1074  * We have a vector of possible localIP->remoteIP paths now ready to start being connected.
1075  */
1076 void
1077 FwdState::connectStart()
1078 {
1079     assert(serverDestinations.size() > 0);
1080
1081     debugs(17, 3, "fwdConnectStart: " << entry->url());
1082
1083     if (!request->hier.first_conn_start.tv_sec) // first attempt
1084         request->hier.first_conn_start = current_time;
1085
1086     /* connection timeout */
1087     int ctimeout;
1088     if (serverDestinations[0]->getPeer()) {
1089         ctimeout = serverDestinations[0]->getPeer()->connect_timeout > 0 ?
1090                    serverDestinations[0]->getPeer()->connect_timeout : Config.Timeout.peer_connect;
1091     } else {
1092         ctimeout = Config.Timeout.connect;
1093     }
1094
1095     /* calculate total forwarding timeout ??? */
1096     int ftimeout = Config.Timeout.forward - (squid_curtime - start_t);
1097     if (ftimeout < 0)
1098         ftimeout = 5;
1099
1100     if (ftimeout < ctimeout)
1101         ctimeout = ftimeout;
1102
1103     if (serverDestinations[0]->getPeer() && request->flags.sslBumped) {
1104         debugs(50, 4, "fwdConnectStart: Ssl bumped connections through parent proxy are not allowed");
1105         ErrorState *anErr = new ErrorState(ERR_CANNOT_FORWARD, Http::scServiceUnavailable, request);
1106         fail(anErr);
1107         self = NULL; // refcounted
1108         return;
1109     }
1110
1111     request->flags.pinned = false; // XXX: what if the ConnStateData set this to flag existing credentials?
1112     // XXX: answer: the peer selection *should* catch it and give us only the pinned peer. so we reverse the =0 step below.
1113     // XXX: also, logs will now lie if pinning is broken and leads to an error message.
1114     if (serverDestinations[0]->peerType == PINNED) {
1115         ConnStateData *pinned_connection = request->pinnedConnection();
1116         debugs(17,7, "pinned peer connection: " << pinned_connection);
1117         // pinned_connection may become nil after a pconn race
1118         if (pinned_connection)
1119             serverConn = pinned_connection->validatePinnedConnection(request, serverDestinations[0]->getPeer());
1120         else
1121             serverConn = NULL;
1122         if (Comm::IsConnOpen(serverConn)) {
1123             pinned_connection->stopPinnedConnectionMonitoring();
1124             flags.connected_okay = true;
1125             ++n_tries;
1126             request->flags.pinned = true;
1127             request->hier.note(serverConn, pinned_connection->pinning.host);
1128             if (pinned_connection->pinnedAuth())
1129                 request->flags.auth = true;
1130             comm_add_close_handler(serverConn->fd, fwdServerClosedWrapper, this);
1131             // the server may close the pinned connection before this request
1132             pconnRace = racePossible;
1133             dispatch();
1134             return;
1135         }
1136         // Pinned connection failure.
1137         debugs(17,2,HERE << "Pinned connection failed: " << pinned_connection);
1138         ErrorState *anErr = new ErrorState(ERR_ZERO_SIZE_OBJECT, Http::scServiceUnavailable, request);
1139         fail(anErr);
1140         self = NULL; // refcounted
1141         return;
1142     }
1143
1144     // Use pconn to avoid opening a new connection.
1145     const char *host = NULL;
1146     if (!serverDestinations[0]->getPeer())
1147         host = request->GetHost();
1148
1149     Comm::ConnectionPointer temp;
1150     // Avoid pconns after races so that the same client does not suffer twice.
1151     // This does not increase the total number of connections because we just
1152     // closed the connection that failed the race. And re-pinning assumes this.
1153     if (pconnRace != raceHappened)
1154         temp = fwdPconnPool->pop(serverDestinations[0], host, checkRetriable());
1155
1156     const bool openedPconn = Comm::IsConnOpen(temp);
1157     pconnRace = openedPconn ? racePossible : raceImpossible;
1158
1159     // if we found an open persistent connection to use. use it.
1160     if (openedPconn) {
1161         serverConn = temp;
1162         flags.connected_okay = true;
1163         debugs(17, 3, HERE << "reusing pconn " << serverConnection());
1164         ++n_tries;
1165
1166         comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
1167
1168         /* Update server side TOS and Netfilter mark on the connection. */
1169         if (Ip::Qos::TheConfig.isAclTosActive()) {
1170             const tos_t tos = GetTosToServer(request);
1171             Ip::Qos::setSockTos(temp, tos);
1172         }
1173 #if SO_MARK
1174         if (Ip::Qos::TheConfig.isAclNfmarkActive()) {
1175             const nfmark_t nfmark = GetNfmarkToServer(request);
1176             Ip::Qos::setSockNfmark(temp, nfmark);
1177         }
1178 #endif
1179
1180         dispatch();
1181         return;
1182     }
1183
1184     // We will try to open a new connection, possibly to the same destination.
1185     // We reset serverDestinations[0] in case we are using it again because
1186     // ConnOpener modifies its destination argument.
1187     serverDestinations[0]->local.port(0);
1188     serverConn = NULL;
1189
1190 #if URL_CHECKSUM_DEBUG
1191     entry->mem_obj->checkUrlChecksum();
1192 #endif
1193
1194     GetMarkingsToServer(request, *serverDestinations[0]);
1195
1196     calls.connector = commCbCall(17,3, "fwdConnectDoneWrapper", CommConnectCbPtrFun(fwdConnectDoneWrapper, this));
1197     Comm::ConnOpener *cs = new Comm::ConnOpener(serverDestinations[0], calls.connector, ctimeout);
1198     if (host)
1199         cs->setHost(host);
1200     AsyncJob::Start(cs);
1201 }
1202
1203 void
1204 FwdState::dispatch()
1205 {
1206     debugs(17, 3, HERE << clientConn << ": Fetching '" << RequestMethodStr(request->method) << " " << entry->url() << "'");
1207     /*
1208      * Assert that server_fd is set.  This is to guarantee that fwdState
1209      * is attached to something and will be deallocated when server_fd
1210      * is closed.
1211      */
1212     assert(Comm::IsConnOpen(serverConn));
1213
1214     fd_note(serverConnection()->fd, entry->url());
1215
1216     fd_table[serverConnection()->fd].noteUse(fwdPconnPool);
1217
1218     /*assert(!EBIT_TEST(entry->flags, ENTRY_DISPATCHED)); */
1219     assert(entry->ping_status != PING_WAITING);
1220
1221     assert(entry->locked());
1222
1223     EBIT_SET(entry->flags, ENTRY_DISPATCHED);
1224
1225     netdbPingSite(request->GetHost());
1226
1227     /* Retrieves remote server TOS or MARK value, and stores it as part of the
1228      * original client request FD object. It is later used to forward
1229      * remote server's TOS/MARK in the response to the client in case of a MISS.
1230      */
1231     if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
1232         if (Comm::IsConnOpen(clientConn) && Comm::IsConnOpen(serverConnection())) {
1233             fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
1234             /* Get the netfilter mark for the connection */
1235             Ip::Qos::getNfmarkFromServer(serverConnection(), clientFde);
1236         }
1237     }
1238
1239 #if _SQUID_LINUX_
1240     /* Bug 2537: The TOS forward part of QOS only applies to patched Linux kernels. */
1241     if (Ip::Qos::TheConfig.isHitTosActive()) {
1242         if (Comm::IsConnOpen(clientConn)) {
1243             fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
1244             /* Get the TOS value for the packet */
1245             Ip::Qos::getTosFromServer(serverConnection(), clientFde);
1246         }
1247     }
1248 #endif
1249
1250 #if USE_OPENSSL
1251     if (request->flags.sslPeek) {
1252         CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData,
1253                      ConnStateData::httpsPeeked, serverConnection());
1254         unregister(serverConn); // async call owns it now
1255         complete(); // destroys us
1256         return;
1257     }
1258 #endif
1259
1260     if (serverConnection()->getPeer() != NULL) {
1261         ++ serverConnection()->getPeer()->stats.fetches;
1262         request->peer_login = serverConnection()->getPeer()->login;
1263         request->peer_domain = serverConnection()->getPeer()->domain;
1264         httpStart(this);
1265     } else {
1266         assert(!request->flags.sslPeek);
1267         request->peer_login = NULL;
1268         request->peer_domain = NULL;
1269
1270         switch (request->protocol) {
1271 #if USE_OPENSSL
1272
1273         case AnyP::PROTO_HTTPS:
1274             httpStart(this);
1275             break;
1276 #endif
1277
1278         case AnyP::PROTO_HTTP:
1279             httpStart(this);
1280             break;
1281
1282         case AnyP::PROTO_GOPHER:
1283             gopherStart(this);
1284             break;
1285
1286         case AnyP::PROTO_FTP:
1287             ftpStart(this);
1288             break;
1289
1290         case AnyP::PROTO_CACHE_OBJECT:
1291
1292         case AnyP::PROTO_INTERNAL:
1293
1294         case AnyP::PROTO_URN:
1295             fatal_dump("Should never get here");
1296             break;
1297
1298         case AnyP::PROTO_WHOIS:
1299             whoisStart(this);
1300             break;
1301
1302         case AnyP::PROTO_WAIS:  /* Not implemented */
1303
1304         default:
1305             debugs(17, DBG_IMPORTANT, "WARNING: Cannot retrieve '" << entry->url() << "'.");
1306             ErrorState *anErr = new ErrorState(ERR_UNSUP_REQ, Http::scBadRequest, request);
1307             fail(anErr);
1308             // Set the dont_retry flag because this is not a transient (network) error.
1309             flags.dont_retry = true;
1310             if (Comm::IsConnOpen(serverConn)) {
1311                 serverConn->close();
1312             }
1313             break;
1314         }
1315     }
1316 }
1317
1318 /*
1319  * FwdState::reforward
1320  *
1321  * returns TRUE if the transaction SHOULD be re-forwarded to the
1322  * next choice in the serverDestinations list.  This method is called when
1323  * server-side communication completes normally, or experiences
1324  * some error after receiving the end of HTTP headers.
1325  */
1326 int
1327 FwdState::reforward()
1328 {
1329     StoreEntry *e = entry;
1330
1331     if (EBIT_TEST(e->flags, ENTRY_ABORTED)) {
1332         debugs(17, 3, HERE << "entry aborted");
1333         return 0;
1334     }
1335
1336     assert(e->store_status == STORE_PENDING);
1337     assert(e->mem_obj);
1338 #if URL_CHECKSUM_DEBUG
1339
1340     e->mem_obj->checkUrlChecksum();
1341 #endif
1342
1343     debugs(17, 3, HERE << e->url() << "?" );
1344
1345     if (!EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) {
1346         debugs(17, 3, HERE << "No, ENTRY_FWD_HDR_WAIT isn't set");
1347         return 0;
1348     }
1349
1350     if (n_tries > Config.forward_max_tries)
1351         return 0;
1352
1353     if (request->bodyNibbled())
1354         return 0;
1355
1356     if (serverDestinations.size() <= 1) {
1357         // NP: <= 1 since total count includes the recently failed one.
1358         debugs(17, 3, HERE << "No alternative forwarding paths left");
1359         return 0;
1360     }
1361
1362     const Http::StatusCode s = e->getReply()->sline.status();
1363     debugs(17, 3, HERE << "status " << s);
1364     return reforwardableStatus(s);
1365 }
1366
1367 /**
1368  * Create "503 Service Unavailable" or "504 Gateway Timeout" error depending
1369  * on whether this is a validation request. RFC 2616 says that we MUST reply
1370  * with "504 Gateway Timeout" if validation fails and cached reply has
1371  * proxy-revalidate, must-revalidate or s-maxage Cache-Control directive.
1372  */
1373 ErrorState *
1374 FwdState::makeConnectingError(const err_type type) const
1375 {
1376     return new ErrorState(type, request->flags.needValidation ?
1377                           Http::scGatewayTimeout : Http::scServiceUnavailable, request);
1378 }
1379
1380 static void
1381 fwdStats(StoreEntry * s)
1382 {
1383     int i;
1384     int j;
1385     storeAppendPrintf(s, "Status");
1386
1387     for (j = 1; j < MAX_FWD_STATS_IDX; ++j) {
1388         storeAppendPrintf(s, "\ttry#%d", j);
1389     }
1390
1391     storeAppendPrintf(s, "\n");
1392
1393     for (i = 0; i <= (int) Http::scInvalidHeader; ++i) {
1394         if (FwdReplyCodes[0][i] == 0)
1395             continue;
1396
1397         storeAppendPrintf(s, "%3d", i);
1398
1399         for (j = 0; j <= MAX_FWD_STATS_IDX; ++j) {
1400             storeAppendPrintf(s, "\t%d", FwdReplyCodes[j][i]);
1401         }
1402
1403         storeAppendPrintf(s, "\n");
1404     }
1405 }
1406
1407 /**** STATIC MEMBER FUNCTIONS *************************************************/
1408
1409 bool
1410 FwdState::reforwardableStatus(const Http::StatusCode s) const
1411 {
1412     switch (s) {
1413
1414     case Http::scBadGateway:
1415
1416     case Http::scGatewayTimeout:
1417         return true;
1418
1419     case Http::scForbidden:
1420
1421     case Http::scInternalServerError:
1422
1423     case Http::scNotImplemented:
1424
1425     case Http::scServiceUnavailable:
1426         return Config.retry.onerror;
1427
1428     default:
1429         return false;
1430     }
1431
1432     /* NOTREACHED */
1433 }
1434
1435 /**
1436  * Decide where details need to be gathered to correctly describe a persistent connection.
1437  * What is needed:
1438  *  -  the address/port details about this link
1439  *  -  domain name of server at other end of this link (either peer or requested host)
1440  */
1441 void
1442 FwdState::pconnPush(Comm::ConnectionPointer &conn, const char *domain)
1443 {
1444     if (conn->getPeer()) {
1445         fwdPconnPool->push(conn, NULL);
1446     } else {
1447         fwdPconnPool->push(conn, domain);
1448     }
1449 }
1450
1451 void
1452 FwdState::initModule()
1453 {
1454     RegisterWithCacheManager();
1455 }
1456
1457 void
1458 FwdState::RegisterWithCacheManager(void)
1459 {
1460     Mgr::RegisterAction("forward", "Request Forwarding Statistics", fwdStats, 0, 1);
1461 }
1462
1463 void
1464 FwdState::logReplyStatus(int tries, const Http::StatusCode status)
1465 {
1466     if (status > Http::scInvalidHeader)
1467         return;
1468
1469     assert(tries >= 0);
1470
1471     if (tries > MAX_FWD_STATS_IDX)
1472         tries = MAX_FWD_STATS_IDX;
1473
1474     ++ FwdReplyCodes[tries][status];
1475 }
1476
1477 /**** PRIVATE NON-MEMBER FUNCTIONS ********************************************/
1478
1479 /*
1480  * DPW 2007-05-19
1481  * Formerly static, but now used by client_side_request.cc
1482  */
1483 /// Checks for a TOS value to apply depending on the ACL
1484 tos_t
1485 aclMapTOS(acl_tos * head, ACLChecklist * ch)
1486 {
1487     acl_tos *l;
1488
1489     for (l = head; l; l = l->next) {
1490         if (!l->aclList || ch->fastCheck(l->aclList) == ACCESS_ALLOWED)
1491             return l->tos;
1492     }
1493
1494     return 0;
1495 }
1496
1497 /// Checks for a netfilter mark value to apply depending on the ACL
1498 nfmark_t
1499 aclMapNfmark(acl_nfmark * head, ACLChecklist * ch)
1500 {
1501     acl_nfmark *l;
1502
1503     for (l = head; l; l = l->next) {
1504         if (!l->aclList || ch->fastCheck(l->aclList) == ACCESS_ALLOWED)
1505             return l->nfmark;
1506     }
1507
1508     return 0;
1509 }
1510
1511 void
1512 getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn)
1513 {
1514     // skip if an outgoing address is already set.
1515     if (!conn->local.isAnyAddr()) return;
1516
1517     // ensure that at minimum the wildcard local matches remote protocol
1518     if (conn->remote.isIPv4())
1519         conn->local.setIPv4();
1520
1521     // maybe use TPROXY client address
1522     if (request && request->flags.spoofClientIp) {
1523         if (!conn->getPeer() || !conn->getPeer()->options.no_tproxy) {
1524 #if FOLLOW_X_FORWARDED_FOR && LINUX_NETFILTER
1525             if (Config.onoff.tproxy_uses_indirect_client)
1526                 conn->local = request->indirect_client_addr;
1527             else
1528 #endif
1529                 conn->local = request->client_addr;
1530             // some flags need setting on the socket to use this address
1531             conn->flags |= COMM_DOBIND;
1532             conn->flags |= COMM_TRANSPARENT;
1533             return;
1534         }
1535         // else no tproxy today ...
1536     }
1537
1538     if (!Config.accessList.outgoing_address) {
1539         return; // anything will do.
1540     }
1541
1542     ACLFilledChecklist ch(NULL, request, NULL);
1543     ch.dst_peer = conn->getPeer();
1544     ch.dst_addr = conn->remote;
1545
1546     // TODO use the connection details in ACL.
1547     // needs a bit of rework in ACLFilledChecklist to use Comm::Connection instead of ConnStateData
1548
1549     AclAddress *l;
1550     for (l = Config.accessList.outgoing_address; l; l = l->next) {
1551
1552         /* check if the outgoing address is usable to the destination */
1553         if (conn->remote.isIPv4() != l->addr.isIPv4()) continue;
1554
1555         /* check ACLs for this outgoing address */
1556         if (!l->aclList || ch.fastCheck(l->aclList) == ACCESS_ALLOWED) {
1557             conn->local = l->addr;
1558             return;
1559         }
1560     }
1561 }
1562
1563 tos_t
1564 GetTosToServer(HttpRequest * request)
1565 {
1566     ACLFilledChecklist ch(NULL, request, NULL);
1567     return aclMapTOS(Ip::Qos::TheConfig.tosToServer, &ch);
1568 }
1569
1570 nfmark_t
1571 GetNfmarkToServer(HttpRequest * request)
1572 {
1573     ACLFilledChecklist ch(NULL, request, NULL);
1574     return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch);
1575 }
1576
1577 void
1578 GetMarkingsToServer(HttpRequest * request, Comm::Connection &conn)
1579 {
1580     // Get the server side TOS and Netfilter mark to be set on the connection.
1581     if (Ip::Qos::TheConfig.isAclTosActive()) {
1582         conn.tos = GetTosToServer(request);
1583         debugs(17, 3, "from " << conn.local << " tos " << int(conn.tos));
1584     }
1585
1586 #if SO_MARK && USE_LIBCAP
1587     conn.nfmark = GetNfmarkToServer(request);
1588     debugs(17, 3, "from " << conn.local << " netfilter mark " << conn.nfmark);
1589 #else
1590     conn.nfmark = 0;
1591 #endif
1592 }