src/forward.cc

   1
   2 /*
   3  * $Id: forward.cc,v 1.175 2008/02/11 22:26:39 rousskov Exp $
   4  *
   5  * DEBUG: section 17    Request Forwarding
   6  * AUTHOR: Duane Wessels
   7  *
   8  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   9  * ----------------------------------------------------------
  10  *
  11  *  Squid is the result of efforts by numerous individuals from
  12  *  the Internet community; see the CONTRIBUTORS file for full
  13  *  details.   Many organizations have provided support for Squid's
  14  *  development; see the SPONSORS file for full details.  Squid is
  15  *  Copyrighted (C) 2001 by the Regents of the University of
  16  *  California; see the COPYRIGHT file for full details.  Squid
  17  *  incorporates software developed and/or copyrighted by other
  18  *  sources; see the CREDITS file for full details.
  19  *
  20  *  This program is free software; you can redistribute it and/or modify
  21  *  it under the terms of the GNU General Public License as published by
  22  *  the Free Software Foundation; either version 2 of the License, or
  23  *  (at your option) any later version.
  24  *
  25  *  This program is distributed in the hope that it will be useful,
  26  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  28  *  GNU General Public License for more details.
  29  *
  30  *  You should have received a copy of the GNU General Public License
  31  *  along with this program; if not, write to the Free Software
  32  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  33  *
  34  */
  35
  36
  37 #include "squid.h"
  38 #include "forward.h"
  39 #include "ACLChecklist.h"
  40 #include "ACL.h"
  41 #include "CacheManager.h"
  42 #include "event.h"
  43 #include "errorpage.h"
  44 #include "fde.h"
  45 #include "HttpReply.h"
  46 #include "HttpRequest.h"
  47 #include "MemObject.h"
  48 #include "pconn.h"
  49 #include "SquidTime.h"
  50 #include "Store.h"
  51 #include "icmp/net_db.h"
  52 #include "IPInterception.h"
  53
  54 static PSC fwdStartCompleteWrapper;
  55 static PF fwdServerClosedWrapper;
  56 #if USE_SSL
  57 static PF fwdNegotiateSSLWrapper;
  58 #endif
  59 static PF fwdConnectTimeoutWrapper;
  60 static EVH fwdConnectStartWrapper;
  61 static CNCB fwdConnectDoneWrapper;
  62
  63 static OBJH fwdStats;
  64 static void fwdServerFree(FwdServer * fs);
  65
  66 #define MAX_FWD_STATS_IDX 9
  67 static int FwdReplyCodes[MAX_FWD_STATS_IDX + 1][HTTP_INVALID_HEADER + 1];
  68
  69 #if WIP_FWD_LOG
  70 static void fwdLog(FwdState * fwdState);
  71 static Logfile *logfile = NULL;
  72 #endif
  73
  74 static PconnPool *fwdPconnPool = new PconnPool("server-side");
  75 CBDATA_CLASS_INIT(FwdState);
  76
  77 void
  78 FwdState::abort(void* d)
  79 {
  80     FwdState* fwd = (FwdState*)d;
  81     Pointer tmp = fwd; // Grab a temporary pointer to keep the object alive during our scope.
  82
  83     if (fwd->server_fd >= 0) {
  84         comm_close(fwd->server_fd);
  85         fwd->server_fd = -1;
  86     }
  87
  88     fwd->self = NULL;
  89 }
  90
  91 /**** PUBLIC INTERFACE ********************************************************/
  92
  93 FwdState::FwdState(int fd, StoreEntry * e, HttpRequest * r)
  94 {
  95     entry = e;
  96     client_fd = fd;
  97     server_fd = -1;
  98     request = HTTPMSGLOCK(r);
  99     start_t = squid_curtime;
 100
 101     e->lock();
 102     EBIT_SET(e->flags, ENTRY_FWD_HDR_WAIT);
 103 }
 104
 105 // Called once, right after object creation, when it is safe to set self
 106 void FwdState::start(Pointer aSelf)
 107 {
 108     // Protect ourselves from being destroyed when the only Server pointing
 109     // to us is gone (while we expect to talk to more Servers later).
 110     // Once we set self, we are responsible for clearing it when we do not
 111     // expect to talk to any servers.
 112     self = aSelf; // refcounted
 113
 114     // We hope that either the store entry aborts or peer is selected.
 115     // Otherwise we are going to leak our object.
 116
 117     entry->registerAbort(FwdState::abort, this);
 118     peerSelect(request, entry, fwdStartCompleteWrapper, this);
 119
 120     // TODO: set self _after_ the peer is selected because we do not need
 121     // self until we start talking to some Server.
 122 }
 123
 124 void
 125 FwdState::completed()
 126 {
 127     if (flags.forward_completed == 1) {
 128         debugs(17, 1, HERE << "FwdState::completed called on a completed request! Bad!");
 129         return;
 130     }
 131
 132     flags.forward_completed = 1;
 133
 134 #if URL_CHECKSUM_DEBUG
 135
 136     entry->mem_obj->checkUrlChecksum();
 137 #endif
 138 #if WIP_FWD_LOG
 139
 140     log();
 141 #endif
 142
 143     if (entry->store_status == STORE_PENDING) {
 144         if (entry->isEmpty()) {
 145             assert(err);
 146             errorAppendEntry(entry, err);
 147             err = NULL;
 148         } else {
 149             EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 150             entry->complete();
 151             entry->releaseRequest();
 152         }
 153     }
 154
 155     if (storePendingNClients(entry) > 0)
 156         assert(!EBIT_TEST(entry->flags, ENTRY_FWD_HDR_WAIT));
 157
 158 }
 159
 160 FwdState::~FwdState()
 161 {
 162     debugs(17, 3, HERE << "FwdState destructor starting");
 163
 164     if (! flags.forward_completed)
 165         completed();
 166
 167     serversFree(&servers);
 168
 169     HTTPMSGUNLOCK(request);
 170
 171     if (err)
 172         errorStateFree(err);
 173
 174     entry->unregisterAbort();
 175
 176     entry->unlock();
 177
 178     entry = NULL;
 179
 180     int fd = server_fd;
 181
 182     if (fd > -1) {
 183         server_fd = -1;
 184         comm_remove_close_handler(fd, fwdServerClosedWrapper, this);
 185         debugs(17, 3, "fwdStateFree: closing FD " << fd);
 186         comm_close(fd);
 187     }
 188
 189     debugs(17, 3, HERE << "FwdState destructor done");
 190 }
 191
 192 /**
 193  * This is the entry point for client-side to start forwarding
 194  * a transaction.  It is a static method that may or may not
 195  * allocate a FwdState.
 196  */
 197 void
 198 FwdState::fwdStart(int client_fd, StoreEntry *entry, HttpRequest *request)
 199 {
 200     /*
 201      * client_addr == no_addr indicates this is an "internal" request
 202      * from peer_digest.c, asn.c, netdb.c, etc and should always
 203      * be allowed.  yuck, I know.
 204      */
 205
 206     if ( !request->client_addr.IsNoAddr() && request->protocol != PROTO_INTERNAL && request->protocol != PROTO_CACHEOBJ) {
 207         /*
 208          * Check if this host is allowed to fetch MISSES from us (miss_access)
 209          */
 210         ACLChecklist ch;
 211         ch.src_addr = request->client_addr;
 212         ch.my_addr = request->my_addr;
 213         ch.request = HTTPMSGLOCK(request);
 214         ch.accessList = cbdataReference(Config.accessList.miss);
 215         /* cbdataReferenceDone() happens in either fastCheck() or ~ACLCheckList */
 216         int answer = ch.fastCheck();
 217
 218         if (answer == 0) {
 219             err_type page_id;
 220             page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, 1);
 221
 222             if (page_id == ERR_NONE)
 223                 page_id = ERR_FORWARDING_DENIED;
 224
 225             ErrorState *anErr = errorCon(page_id, HTTP_FORBIDDEN, request);
 226
 227             errorAppendEntry(entry, anErr);     // frees anErr
 228
 229             return;
 230         }
 231     }
 232
 233     debugs(17, 3, "FwdState::start() '" << entry->url() << "'");
 234     /*
 235      * This seems like an odd place to bind mem_obj and request.
 236      * Might want to assert that request is NULL at this point
 237      */
 238     entry->mem_obj->request = HTTPMSGLOCK(request);
 239 #if URL_CHECKSUM_DEBUG
 240
 241     entry->mem_obj->checkUrlChecksum();
 242 #endif
 243
 244     if (shutting_down) {
 245         /* more yuck */
 246         ErrorState *anErr = errorCon(ERR_SHUTTING_DOWN, HTTP_SERVICE_UNAVAILABLE, request);
 247         errorAppendEntry(entry, anErr); // frees anErr
 248         return;
 249     }
 250
 251     switch (request->protocol) {
 252
 253     case PROTO_INTERNAL:
 254         internalStart(request, entry);
 255         return;
 256
 257     case PROTO_CACHEOBJ:
 258         CacheManager::GetInstance()->Start(client_fd, request, entry);
 259         return;
 260
 261     case PROTO_URN:
 262         urnStart(request, entry);
 263         return;
 264
 265     default:
 266         FwdState::Pointer fwd = new FwdState(client_fd, entry, request);
 267
 268         /* If we need to transparently proxy the request
 269          * then we need the client source protocol, address and port */
 270         if (request->flags.spoof_client_ip) {
 271             fwd->src = request->client_addr;
 272             // AYJ: do we need to pass on the transparent flag also?
 273         }
 274
 275         fwd->start(fwd);
 276         return;
 277     }
 278
 279     /* NOTREACHED */
 280 }
 281
 282 void
 283 FwdState::fail(ErrorState * errorState)
 284 {
 285     debugs(17, 3, HERE << err_type_str[errorState->type] << " \"" << httpStatusString(errorState->httpStatus) << "\"\n\t" << entry->url()  );
 286
 287     if (err)
 288         errorStateFree(err);
 289
 290     err = errorState;
 291
 292     if (!errorState->request)
 293         errorState->request = HTTPMSGLOCK(request);
 294 }
 295
 296 /**
 297  * Frees fwdState without closing FD or generating an abort
 298  */
 299 void
 300 FwdState::unregister(int fd)
 301 {
 302     debugs(17, 3, HERE << entry->url()  );
 303     assert(fd == server_fd);
 304     assert(fd > -1);
 305     comm_remove_close_handler(fd, fwdServerClosedWrapper, this);
 306     server_fd = -1;
 307 }
 308
 309 /**
 310  * server-side modules call fwdComplete() when they are done
 311  * downloading an object.  Then, we either 1) re-forward the
 312  * request somewhere else if needed, or 2) call storeComplete()
 313  * to finish it off
 314  */
 315 void
 316 FwdState::complete()
 317 {
 318     StoreEntry *e = entry;
 319     assert(entry->store_status == STORE_PENDING);
 320     debugs(17, 3, HERE << e->url() << "\n\tstatus " << entry->getReply()->sline.status  );
 321 #if URL_CHECKSUM_DEBUG
 322
 323     entry->mem_obj->checkUrlChecksum();
 324 #endif
 325
 326     logReplyStatus(n_tries, entry->getReply()->sline.status);
 327
 328     if (reforward()) {
 329         debugs(17, 3, "fwdComplete: re-forwarding " << entry->getReply()->sline.status << " " << e->url());
 330
 331         if (server_fd > -1)
 332             unregister(server_fd);
 333
 334         e->reset();
 335
 336         startComplete(servers);
 337     } else {
 338         debugs(17, 3, "fwdComplete: server FD " << server_fd << " not re-forwarding status " << entry->getReply()->sline.status);
 339         if (entry->isEmpty() && !err) {
 340             EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
 341             entry->complete();
 342         }
 343
 344         if (server_fd < 0)
 345             completed();
 346
 347         self = NULL; // refcounted
 348     }
 349 }
 350
 351
 352 /**** CALLBACK WRAPPERS ************************************************************/
 353
 354 static void
 355 fwdStartCompleteWrapper(FwdServer * servers, void *data)
 356 {
 357     FwdState *fwd = (FwdState *) data;
 358     fwd->startComplete(servers);
 359 }
 360
 361 static void
 362 fwdServerClosedWrapper(int fd, void *data)
 363 {
 364     FwdState *fwd = (FwdState *) data;
 365     fwd->serverClosed(fd);
 366 }
 367
 368 static void
 369 fwdConnectStartWrapper(void *data)
 370 {
 371     FwdState *fwd = (FwdState *) data;
 372     fwd->connectStart();
 373 }
 374
 375 #if USE_SSL
 376 static void
 377 fwdNegotiateSSLWrapper(int fd, void *data)
 378 {
 379     FwdState *fwd = (FwdState *) data;
 380     fwd->negotiateSSL(fd);
 381 }
 382
 383 #endif
 384
 385 static void
 386 fwdConnectDoneWrapper(int server_fd, comm_err_t status, int xerrno, void *data)
 387 {
 388     FwdState *fwd = (FwdState *) data;
 389     fwd->connectDone(server_fd, status, xerrno);
 390 }
 391
 392 static void
 393 fwdConnectTimeoutWrapper(int fd, void *data)
 394 {
 395     FwdState *fwd = (FwdState *) data;
 396     fwd->connectTimeout(fd);
 397 }
 398
 399 /*
 400  * Accounts for closed persistent connections
 401  */
 402 static void
 403 fwdPeerClosed(int fd, void *data)
 404 {
 405     peer *p = (peer *)data;
 406     p->stats.conn_open--;
 407 }
 408
 409 /**** PRIVATE *****************************************************************/
 410
 411 /*
 412  * FwdState::checkRetry
 413  *
 414  * Return TRUE if the request SHOULD be retried.  This method is
 415  * called when the HTTP connection fails, or when the connection
 416  * is closed before server-side read the end of HTTP headers.
 417  */
 418 bool
 419 FwdState::checkRetry()
 420 {
 421     if (shutting_down)
 422         return false;
 423
 424     if (entry->store_status != STORE_PENDING)
 425         return false;
 426
 427     if (!entry->isEmpty())
 428         return false;
 429
 430     if (n_tries > 10)
 431         return false;
 432
 433     if (origin_tries > 2)
 434         return false;
 435
 436     if (squid_curtime - start_t > Config.Timeout.forward)
 437         return false;
 438
 439     if (flags.dont_retry)
 440         return false;
 441
 442     if (!checkRetriable())
 443         return false;
 444
 445     if (request->bodyNibbled())
 446         return false;
 447
 448     return true;
 449 }
 450
 451 /*
 452  * FwdState::checkRetriable
 453  *
 454  * Return TRUE if this is the kind of request that can be retried
 455  * after a failure.  If the request is not retriable then we don't
 456  * want to risk sending it on a persistent connection.  Instead we'll
 457  * force it to go on a new HTTP connection.
 458  */
 459 bool
 460 FwdState::checkRetriable()
 461 {
 462     /* If there is a request body then Squid can only try once
 463      * even if the method is indempotent
 464      */
 465
 466     if (request->body_pipe != NULL)
 467         return false;
 468
 469     /* RFC2616 9.1 Safe and Idempotent Methods */
 470     switch (request->method.id()) {
 471         /* 9.1.1 Safe Methods */
 472
 473     case METHOD_GET:
 474
 475     case METHOD_HEAD:
 476         /* 9.1.2 Idempotent Methods */
 477
 478     case METHOD_PUT:
 479
 480     case METHOD_DELETE:
 481
 482     case METHOD_OPTIONS:
 483
 484     case METHOD_TRACE:
 485         break;
 486
 487     default:
 488         return false;
 489     }
 490
 491     return true;
 492 }
 493
 494 void
 495 FwdState::serverClosed(int fd)
 496 {
 497     debugs(17, 2, "fwdServerClosed: FD " << fd << " " << entry->url());
 498     assert(server_fd == fd);
 499     server_fd = -1;
 500
 501     retryOrBail();
 502 }
 503
 504 void
 505 FwdState::retryOrBail()
 506 {
 507     if (!self) { // we have aborted before the server called us back
 508         debugs(17, 5, HERE << "not retrying because of earlier abort");
 509         // we will be destroyed when the server clears its Pointer to us
 510         return;
 511     }
 512
 513     if (checkRetry()) {
 514         int originserver = (servers->_peer == NULL);
 515         debugs(17, 3, "fwdServerClosed: re-forwarding (" << n_tries << " tries, " << (squid_curtime - start_t) << " secs)");
 516
 517         if (servers->next) {
 518             /* use next, or cycle if origin server isn't last */
 519             FwdServer *fs = servers;
 520             FwdServer **T, *T2 = NULL;
 521             servers = fs->next;
 522
 523             for (T = &servers; *T; T2 = *T, T = &(*T)->next);
 524             if (T2 && T2->_peer) {
 525                 /* cycle */
 526                 *T = fs;
 527                 fs->next = NULL;
 528             } else {
 529                 /* Use next. The last "direct" entry is retried multiple times */
 530                 servers = fs->next;
 531                 fwdServerFree(fs);
 532                 originserver = 0;
 533             }
 534         }
 535
 536         /* use eventAdd to break potential call sequence loops and to slow things down a little */
 537         eventAdd("fwdConnectStart", fwdConnectStartWrapper, this, originserver ? 0.05 : 0.005, 0);
 538
 539         return;
 540     }
 541
 542     if (!err && shutting_down) {
 543         errorCon(ERR_SHUTTING_DOWN, HTTP_SERVICE_UNAVAILABLE, request);
 544     }
 545
 546     self = NULL;        // refcounted
 547 }
 548
 549 // called by the server that failed after calling unregister()
 550 void
 551 FwdState::handleUnregisteredServerEnd()
 552 {
 553     debugs(17, 2, "handleUnregisteredServerEnd: self=" << self <<
 554            " err=" << err << ' ' << entry->url());
 555     assert(server_fd < 0);
 556     retryOrBail();
 557 }
 558
 559 #if USE_SSL
 560 void
 561 FwdState::negotiateSSL(int fd)
 562 {
 563     FwdServer *fs = servers;
 564     SSL *ssl = fd_table[fd].ssl;
 565     int ret;
 566
 567     if ((ret = SSL_connect(ssl)) <= 0) {
 568         int ssl_error = SSL_get_error(ssl, ret);
 569
 570         switch (ssl_error) {
 571
 572         case SSL_ERROR_WANT_READ:
 573             commSetSelect(fd, COMM_SELECT_READ, fwdNegotiateSSLWrapper, this, 0);
 574             return;
 575
 576         case SSL_ERROR_WANT_WRITE:
 577             commSetSelect(fd, COMM_SELECT_WRITE, fwdNegotiateSSLWrapper, this, 0);
 578             return;
 579
 580         default:
 581             debugs(81, 1, "fwdNegotiateSSL: Error negotiating SSL connection on FD " << fd <<
 582                    ": " << ERR_error_string(ERR_get_error(), NULL) << " (" << ssl_error <<
 583                    "/" << ret << "/" << errno << ")");
 584             ErrorState *anErr = errorCon(ERR_SECURE_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
 585 #ifdef EPROTO
 586
 587             anErr->xerrno = EPROTO;
 588 #else
 589
 590             anErr->xerrno = EACCES;
 591 #endif
 592
 593             fail(anErr);
 594
 595             if (fs->_peer) {
 596                 peerConnectFailed(fs->_peer);
 597                 fs->_peer->stats.conn_open--;
 598             }
 599
 600             comm_close(fd);
 601             return;
 602         }
 603     }
 604
 605     if (fs->_peer && !SSL_session_reused(ssl)) {
 606         if (fs->_peer->sslSession)
 607             SSL_SESSION_free(fs->_peer->sslSession);
 608
 609         fs->_peer->sslSession = SSL_get1_session(ssl);
 610     }
 611
 612     dispatch();
 613 }
 614
 615 void
 616 FwdState::initiateSSL()
 617 {
 618     FwdServer *fs = servers;
 619     int fd = server_fd;
 620     SSL *ssl;
 621     SSL_CTX *sslContext = NULL;
 622     peer *peer = fs->_peer;
 623
 624     if (peer) {
 625         assert(peer->use_ssl);
 626         sslContext = peer->sslContext;
 627     } else {
 628         sslContext = Config.ssl_client.sslContext;
 629     }
 630
 631     assert(sslContext);
 632
 633     if ((ssl = SSL_new(sslContext)) == NULL) {
 634         debugs(83, 1, "fwdInitiateSSL: Error allocating handle: " << ERR_error_string(ERR_get_error(), NULL)  );
 635         ErrorState *anErr = errorCon(ERR_SOCKET_FAILURE, HTTP_INTERNAL_SERVER_ERROR, request);
 636         anErr->xerrno = errno;
 637         fail(anErr);
 638         self = NULL;            // refcounted
 639         return;
 640     }
 641
 642     SSL_set_fd(ssl, fd);
 643
 644     if (peer) {
 645         if (peer->ssldomain)
 646             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->ssldomain);
 647
 648 #if NOT_YET
 649
 650         else if (peer->name)
 651             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->name);
 652
 653 #endif
 654
 655         else
 656             SSL_set_ex_data(ssl, ssl_ex_index_server, peer->host);
 657
 658         if (peer->sslSession)
 659             SSL_set_session(ssl, peer->sslSession);
 660
 661     } else {
 662         SSL_set_ex_data(ssl, ssl_ex_index_server, (void*)request->GetHost());
 663     }
 664
 665     // Create the ACL check list now, while we have access to more info.
 666     // The list is used in ssl_verify_cb() and is freed in ssl_free().
 667     if (acl_access *acl = Config.ssl_client.cert_error) {
 668         ACLChecklist *check = aclChecklistCreate(acl, request, dash_str);
 669         check->fd(fd);
 670         SSL_set_ex_data(ssl, ssl_ex_index_cert_error_check, check);
 671     }
 672
 673     fd_table[fd].ssl = ssl;
 674     fd_table[fd].read_method = &ssl_read_method;
 675     fd_table[fd].write_method = &ssl_write_method;
 676     negotiateSSL(fd);
 677 }
 678
 679 #endif
 680
 681 void
 682 FwdState::connectDone(int aServerFD, comm_err_t status, int xerrno)
 683 {
 684     FwdServer *fs = servers;
 685     assert(server_fd == aServerFD);
 686
 687     if (Config.onoff.log_ip_on_direct && status != COMM_ERR_DNS && fs->code == HIER_DIRECT)
 688         updateHierarchyInfo();
 689
 690     if (status == COMM_ERR_DNS) {
 691         /*
 692          * Only set the dont_retry flag if the DNS lookup fails on
 693          * a direct connection.  If DNS lookup fails when trying
 694          * a neighbor cache, we may want to retry another option.
 695          */
 696
 697         if (NULL == fs->_peer)
 698             flags.dont_retry = 1;
 699
 700         debugs(17, 4, "fwdConnectDone: Unknown host: " << request->GetHost());
 701
 702         ErrorState *anErr = errorCon(ERR_DNS_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
 703
 704         anErr->dnsserver_msg = xstrdup(dns_error_message_safe());
 705
 706         fail(anErr);
 707
 708         comm_close(server_fd);
 709     } else if (status != COMM_OK) {
 710         assert(fs);
 711         ErrorState *anErr = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
 712         anErr->xerrno = xerrno;
 713
 714         fail(anErr);
 715
 716         if (fs->_peer)
 717             peerConnectFailed(fs->_peer);
 718
 719         comm_close(server_fd);
 720     } else {
 721         debugs(17, 3, "fwdConnectDone: FD " << server_fd << ": '" << entry->url() << "'" );
 722
 723         if (fs->_peer)
 724             peerConnectSucceded(fs->_peer);
 725
 726 #if USE_SSL
 727
 728         if ((fs->_peer && fs->_peer->use_ssl) ||
 729                 (!fs->_peer && request->protocol == PROTO_HTTPS)) {
 730             initiateSSL();
 731             return;
 732         }
 733
 734 #endif
 735         dispatch();
 736     }
 737 }
 738
 739 void
 740 FwdState::connectTimeout(int fd)
 741 {
 742     FwdServer *fs = servers;
 743
 744     debugs(17, 2, "fwdConnectTimeout: FD " << fd << ": '" << entry->url() << "'" );
 745     assert(fd == server_fd);
 746
 747     if (Config.onoff.log_ip_on_direct && fs->code == HIER_DIRECT && fd_table[fd].ipaddr[0])
 748         updateHierarchyInfo();
 749
 750     if (entry->isEmpty()) {
 751         ErrorState *anErr = errorCon(ERR_CONNECT_FAIL, HTTP_GATEWAY_TIMEOUT, request);
 752         anErr->xerrno = ETIMEDOUT;
 753         fail(anErr);
 754         /*
 755          * This marks the peer DOWN ...
 756          */
 757
 758         if (servers)
 759             if (servers->_peer)
 760                 peerConnectFailed(servers->_peer);
 761     }
 762
 763     comm_close(fd);
 764 }
 765
 766 void
 767 FwdState::connectStart()
 768 {
 769     const char *url = entry->url();
 770     int fd = -1;
 771     FwdServer *fs = servers;
 772     const char *host;
 773     unsigned short port;
 774     const char *domain = NULL;
 775     int ctimeout;
 776     int ftimeout = Config.Timeout.forward - (squid_curtime - start_t);
 777
 778     IPAddress outgoing;
 779     unsigned short tos;
 780
 781     IPAddress client_addr;
 782     assert(fs);
 783     assert(server_fd == -1);
 784     debugs(17, 3, "fwdConnectStart: " << url);
 785
 786     if (fs->_peer) {
 787         host = fs->_peer->host;
 788         port = fs->_peer->http_port;
 789         ctimeout = fs->_peer->connect_timeout > 0 ? fs->_peer->connect_timeout
 790                    : Config.Timeout.peer_connect;
 791
 792         if (fs->_peer->options.originserver)
 793             domain = request->GetHost();
 794     } else {
 795         host = request->GetHost();
 796         port = request->port;
 797         ctimeout = Config.Timeout.connect;
 798     }
 799
 800     if (request->flags.spoof_client_ip)
 801         client_addr = request->client_addr;
 802
 803     if (ftimeout < 0)
 804         ftimeout = 5;
 805
 806     if (ftimeout < ctimeout)
 807         ctimeout = ftimeout;
 808
 809
 810     request->flags.pinned = 0;
 811     if (fs->code == PINNED) {
 812         ConnStateData *pinned_connection = request->pinnedConnection();
 813         assert(pinned_connection);
 814         fd = pinned_connection->validatePinnedConnection(request, fs->_peer);
 815         if (fd >= 0) {
 816             pinned_connection->unpinConnection();
 817 #if 0
 818             if (!fs->_peer)
 819                 fs->code = HIER_DIRECT;
 820 #endif
 821             server_fd = fd;
 822             n_tries++;
 823             request->flags.pinned = 1;
 824             if (pinned_connection->pinnedAuth())
 825                 request->flags.auth = 1;
 826             comm_add_close_handler(fd, fwdServerClosedWrapper, this);
 827             connectDone(fd, COMM_OK, 0);
 828             return;
 829         }
 830         /* Failure. Fall back on next path */
 831         request->releasePinnedConnection();
 832         servers = fs->next;
 833         fwdServerFree(fs);
 834         connectStart();
 835         return;
 836     }
 837
 838     fd = fwdPconnPool->pop(host, port, domain, client_addr, checkRetriable());
 839     if (fd >= 0) {
 840         debugs(17, 3, "fwdConnectStart: reusing pconn FD " << fd);
 841         server_fd = fd;
 842         n_tries++;
 843
 844         if (!fs->_peer)
 845             origin_tries++;
 846
 847         updateHierarchyInfo();
 848
 849         comm_add_close_handler(fd, fwdServerClosedWrapper, this);
 850
 851         dispatch();
 852
 853         return;
 854     }
 855
 856 #if URL_CHECKSUM_DEBUG
 857     entry->mem_obj->checkUrlChecksum();
 858
 859 #endif
 860
 861     outgoing = getOutgoingAddr(request, fs->_peer);
 862
 863     tos = getOutgoingTOS(request);
 864
 865     debugs(17, 3, "fwdConnectStart: got outgoing addr " << outgoing << ", tos " << tos);
 866
 867     if (request->flags.spoof_client_ip) {
 868         fd = comm_openex(SOCK_STREAM, IPPROTO_TCP, outgoing, (COMM_NONBLOCKING|COMM_TRANSPARENT), tos, url);
 869     } else {
 870         fd = comm_openex(SOCK_STREAM, IPPROTO_TCP, outgoing, COMM_NONBLOCKING, tos, url);
 871     }
 872
 873     debugs(17, 3, "fwdConnectStart: got TCP FD " << fd);
 874
 875     if (fd < 0) {
 876         debugs(50, 4, "fwdConnectStart: " << xstrerror());
 877         ErrorState *anErr = errorCon(ERR_SOCKET_FAILURE, HTTP_INTERNAL_SERVER_ERROR, request);
 878         anErr->xerrno = errno;
 879         fail(anErr);
 880         self = NULL;    // refcounted
 881         return;
 882     }
 883
 884     server_fd = fd;
 885     n_tries++;
 886
 887     if (!fs->_peer)
 888         origin_tries++;
 889
 890     /*
 891      * stats.conn_open is used to account for the number of
 892      * connections that we have open to the peer, so we can limit
 893      * based on the max-conn option.  We need to increment here,
 894      * even if the connection may fail.
 895      */
 896
 897     if (fs->_peer) {
 898         fs->_peer->stats.conn_open++;
 899         comm_add_close_handler(fd, fwdPeerClosed, fs->_peer);
 900     }
 901
 902     comm_add_close_handler(fd, fwdServerClosedWrapper, this);
 903
 904     commSetTimeout(fd, ctimeout, fwdConnectTimeoutWrapper, this);
 905
 906 #if LINUX_TPROXY2
 907     if (!fs->_peer && request->flags.spoof_client_ip) {
 908         // try to set the outgoing address using TPROXY v2
 909         // if it fails we abort any further TPROXY actions on this connection
 910         if (IPInterceptor.SetTproxy2OutgoingAddr(int fd, const IPAddress &src) == -1) {
 911             request->flags.spoof_client_ip = 0;
 912         }
 913     }
 914 #endif
 915
 916     updateHierarchyInfo();
 917     commConnectStart(fd, host, port, fwdConnectDoneWrapper, this);
 918 }
 919
 920 void
 921 FwdState::startComplete(FwdServer * theServers)
 922 {
 923     debugs(17, 3, "fwdStartComplete: " << entry->url()  );
 924
 925     if (theServers != NULL) {
 926         servers = theServers;
 927         connectStart();
 928     } else {
 929         startFail();
 930     }
 931 }
 932
 933 void
 934 FwdState::startFail()
 935 {
 936     debugs(17, 3, "fwdStartFail: " << entry->url()  );
 937     ErrorState *anErr = errorCon(ERR_CANNOT_FORWARD, HTTP_SERVICE_UNAVAILABLE, request);
 938     anErr->xerrno = errno;
 939     fail(anErr);
 940     self = NULL;        // refcounted
 941 }
 942
 943 void
 944 FwdState::dispatch()
 945 {
 946     peer *p = NULL;
 947     debugs(17, 3, "fwdDispatch: FD " << client_fd << ": Fetching '" << RequestMethodStr(request->method) << " " << entry->url() << "'" );
 948     /*
 949      * Assert that server_fd is set.  This is to guarantee that fwdState
 950      * is attached to something and will be deallocated when server_fd
 951      * is closed.
 952      */
 953     assert(server_fd > -1);
 954
 955     fd_note(server_fd, entry->url());
 956
 957     fd_table[server_fd].noteUse(fwdPconnPool);
 958
 959     /*assert(!EBIT_TEST(entry->flags, ENTRY_DISPATCHED)); */
 960     assert(entry->ping_status != PING_WAITING);
 961
 962     assert(entry->lock_count);
 963
 964     EBIT_SET(entry->flags, ENTRY_DISPATCHED);
 965
 966     netdbPingSite(request->GetHost());
 967
 968 #if USE_ZPH_QOS && _SQUID_LINUX_
 969     /* Bug 2537: This part of ZPH only applies to patched Linux kernels. */
 970
 971     /* Retrieves remote server TOS value, and stores it as part of the
 972      * original client request FD object. It is later used to forward
 973      * remote server's TOS in the response to the client in case of a MISS.
 974      */
 975     fde * clientFde = &fd_table[client_fd];
 976     if (clientFde) {
 977         int tos = 1;
 978         int tos_len = sizeof(tos);
 979         clientFde->upstreamTOS = 0;
 980         if (setsockopt(server_fd,SOL_IP,IP_RECVTOS,&tos,tos_len)==0) {
 981             unsigned char buf[512];
 982             int len = 512;
 983             if (getsockopt(server_fd,SOL_IP,IP_PKTOPTIONS,buf,(socklen_t*)&len) == 0) {
 984                 /* Parse the PKTOPTIONS structure to locate the TOS data message
 985                  * prepared in the kernel by the ZPH incoming TCP TOS preserving
 986                  * patch.
 987                  */
 988                 unsigned char * p = buf;
 989                 while (p-buf < len) {
 990                     struct cmsghdr *o = (struct cmsghdr*)p;
 991                     if (o->cmsg_len<=0)
 992                         break;
 993
 994                     if (o->cmsg_level == SOL_IP && o->cmsg_type == IP_TOS) {
 995                         clientFde->upstreamTOS = (unsigned char)(*(int*)CMSG_DATA(o));
 996                         break;
 997                     }
 998                     p += CMSG_LEN(o->cmsg_len);
 999                 }
1000             } else {
1001                 debugs(33, 1, "ZPH: error in getsockopt(IP_PKTOPTIONS) on FD "<<server_fd<<" "<<xstrerror());
1002             }
1003         } else {
1004             debugs(33, 1, "ZPH: error in setsockopt(IP_RECVTOS) on FD "<<server_fd<<" "<<xstrerror());
1005         }
1006     }
1007 #endif
1008
1009     if (servers && (p = servers->_peer)) {
1010         p->stats.fetches++;
1011         request->peer_login = p->login;
1012         request->peer_domain = p->domain;
1013         httpStart(this);
1014     } else {
1015         request->peer_login = NULL;
1016         request->peer_domain = NULL;
1017
1018         switch (request->protocol) {
1019 #if USE_SSL
1020
1021         case PROTO_HTTPS:
1022             httpStart(this);
1023             break;
1024 #endif
1025
1026         case PROTO_HTTP:
1027             httpStart(this);
1028             break;
1029
1030         case PROTO_GOPHER:
1031             gopherStart(this);
1032             break;
1033
1034         case PROTO_FTP:
1035             ftpStart(this);
1036             break;
1037
1038         case PROTO_CACHEOBJ:
1039
1040         case PROTO_INTERNAL:
1041
1042         case PROTO_URN:
1043             fatal_dump("Should never get here");
1044             break;
1045
1046         case PROTO_WHOIS:
1047             whoisStart(this);
1048             break;
1049
1050         case PROTO_WAIS:        /* Not implemented */
1051
1052         default:
1053             debugs(17, 1, "fwdDispatch: Cannot retrieve '" << entry->url() << "'" );
1054             ErrorState *anErr = errorCon(ERR_UNSUP_REQ, HTTP_BAD_REQUEST, request);
1055             fail(anErr);
1056             /*
1057              * Force a persistent connection to be closed because
1058              * some Netscape browsers have a bug that sends CONNECT
1059              * requests as GET's over persistent connections.
1060              */
1061             request->flags.proxy_keepalive = 0;
1062             /*
1063              * Set the dont_retry flag becuase this is not a
1064              * transient (network) error; its a bug.
1065              */
1066             flags.dont_retry = 1;
1067             comm_close(server_fd);
1068             break;
1069         }
1070     }
1071 }
1072
1073 /*
1074  * FwdState::reforward
1075  *
1076  * returns TRUE if the transaction SHOULD be re-forwarded to the
1077  * next choice in the FwdServers list.  This method is called when
1078  * server-side communication completes normally, or experiences
1079  * some error after receiving the end of HTTP headers.
1080  */
1081 int
1082 FwdState::reforward()
1083 {
1084     StoreEntry *e = entry;
1085     FwdServer *fs = servers;
1086     http_status s;
1087     assert(e->store_status == STORE_PENDING);
1088     assert(e->mem_obj);
1089 #if URL_CHECKSUM_DEBUG
1090
1091     e->mem_obj->checkUrlChecksum();
1092 #endif
1093
1094     debugs(17, 3, "fwdReforward: " << e->url() << "?" );
1095
1096     if (!EBIT_TEST(e->flags, ENTRY_FWD_HDR_WAIT)) {
1097         debugs(17, 3, "fwdReforward: No, ENTRY_FWD_HDR_WAIT isn't set");
1098         return 0;
1099     }
1100
1101     if (n_tries > 9)
1102         return 0;
1103
1104     if (origin_tries > 1)
1105         return 0;
1106
1107     if (request->bodyNibbled())
1108         return 0;
1109
1110     assert(fs);
1111
1112     servers = fs->next;
1113
1114     fwdServerFree(fs);
1115
1116     if (servers == NULL) {
1117         debugs(17, 3, "fwdReforward: No forward-servers left");
1118         return 0;
1119     }
1120
1121     s = e->getReply()->sline.status;
1122     debugs(17, 3, "fwdReforward: status " << s);
1123     return reforwardableStatus(s);
1124 }
1125
1126 static void
1127 fwdStats(StoreEntry * s)
1128 {
1129     int i;
1130     int j;
1131     storeAppendPrintf(s, "Status");
1132
1133     for (j = 0; j <= MAX_FWD_STATS_IDX; j++) {
1134         storeAppendPrintf(s, "\ttry#%d", j + 1);
1135     }
1136
1137     storeAppendPrintf(s, "\n");
1138
1139     for (i = 0; i <= (int) HTTP_INVALID_HEADER; i++) {
1140         if (FwdReplyCodes[0][i] == 0)
1141             continue;
1142
1143         storeAppendPrintf(s, "%3d", i);
1144
1145         for (j = 0; j <= MAX_FWD_STATS_IDX; j++) {
1146             storeAppendPrintf(s, "\t%d", FwdReplyCodes[j][i]);
1147         }
1148
1149         storeAppendPrintf(s, "\n");
1150     }
1151 }
1152
1153
1154 /**** STATIC MEMBER FUNCTIONS *************************************************/
1155
1156 bool
1157 FwdState::reforwardableStatus(http_status s)
1158 {
1159     switch (s) {
1160
1161     case HTTP_BAD_GATEWAY:
1162
1163     case HTTP_GATEWAY_TIMEOUT:
1164         return true;
1165
1166     case HTTP_FORBIDDEN:
1167
1168     case HTTP_INTERNAL_SERVER_ERROR:
1169
1170     case HTTP_NOT_IMPLEMENTED:
1171
1172     case HTTP_SERVICE_UNAVAILABLE:
1173         return Config.retry.onerror;
1174
1175     default:
1176         return false;
1177     }
1178
1179     /* NOTREACHED */
1180 }
1181
1182 void
1183
1184 FwdState::pconnPush(int fd, const char *host, int port, const char *domain, IPAddress &client_addr)
1185 {
1186     fwdPconnPool->push(fd, host, port, domain, client_addr);
1187 }
1188
1189 void
1190 FwdState::initModule()
1191 {
1192     memDataInit(MEM_FWD_SERVER, "FwdServer", sizeof(FwdServer), 0);
1193
1194 #if WIP_FWD_LOG
1195
1196     if (logfile)
1197         (void) 0;
1198     else if (NULL == Config.Log.forward)
1199         (void) 0;
1200     else
1201         logfile = logfileOpen(Config.Log.forward, 0, 1);
1202
1203 #endif
1204
1205     RegisterWithCacheManager();
1206 }
1207
1208 void
1209 FwdState::RegisterWithCacheManager(void)
1210 {
1211     CacheManager::GetInstance()->
1212     registerAction("forward", "Request Forwarding Statistics", fwdStats, 0, 1);
1213 }
1214
1215 void
1216 FwdState::logReplyStatus(int tries, http_status status)
1217 {
1218     if (status > HTTP_INVALID_HEADER)
1219         return;
1220
1221     assert(tries);
1222
1223     tries--;
1224
1225     if (tries > MAX_FWD_STATS_IDX)
1226         tries = MAX_FWD_STATS_IDX;
1227
1228     FwdReplyCodes[tries][status]++;
1229 }
1230
1231 void
1232 FwdState::serversFree(FwdServer ** FSVR)
1233 {
1234     FwdServer *fs;
1235
1236     while ((fs = *FSVR)) {
1237         *FSVR = fs->next;
1238         fwdServerFree(fs);
1239     }
1240 }
1241
1242 /** From Comment #5 by Henrik Nordstrom made at
1243 http://www.squid-cache.org/bugs/show_bug.cgi?id=2391 on 2008-09-19
1244
1245 updateHierarchyInfo should be called each time a new path has been
1246 selected or when more information about the path is available (i.e. the
1247 server IP), and when it's called it needs to be given reasonable
1248 arguments describing the now selected path..
1249
1250 It does not matter from a functional perspective if it gets called a few
1251 times more than what is really needed, but calling it too often may
1252 obviously hurt performance.
1253
1254 \todo Current code looks fine, even if using !fs->_peer as condition
1255 instead of HIER_DIRECT would be clearer.
1256 */
1257 // updates HierarchyLogEntry, guessing nextHop and its format
1258 void
1259 FwdState::updateHierarchyInfo()
1260 {
1261     assert(request);
1262
1263     FwdServer *fs = servers;
1264     assert(fs);
1265
1266     const char *nextHop = NULL;
1267
1268     if (fs->_peer) {
1269         // went to peer, log peer host name
1270         nextHop = fs->_peer->name;
1271     } else {
1272         // went DIRECT, must honor log_ip_on_direct
1273
1274         // XXX: or should we use request->host_addr here? how?
1275         assert(server_fd >= 0);
1276         nextHop = fd_table[server_fd].ipaddr;
1277         if (!Config.onoff.log_ip_on_direct || !nextHop[0])
1278             nextHop = request->GetHost(); // domain name
1279     }
1280
1281     assert(nextHop);
1282     hierarchyNote(&request->hier, fs->code, nextHop);
1283 }
1284
1285
1286 /**** PRIVATE NON-MEMBER FUNCTIONS ********************************************/
1287
1288 static void
1289 fwdServerFree(FwdServer * fs)
1290 {
1291     cbdataReferenceDone(fs->_peer);
1292     memFree(fs, MEM_FWD_SERVER);
1293 }
1294
1295 static IPAddress
1296 aclMapAddr(acl_address * head, ACLChecklist * ch)
1297 {
1298     acl_address *l;
1299
1300     IPAddress addr;
1301
1302     for (l = head; l; l = l->next) {
1303         if (ch->matchAclListFast(l->aclList))
1304             return l->addr;
1305     }
1306
1307     addr.SetAnyAddr();
1308     return addr;
1309 }
1310
1311 /*
1312  * DPW 2007-05-19
1313  * Formerly static, but now used by client_side_request.cc
1314  */
1315 int
1316 aclMapTOS(acl_tos * head, ACLChecklist * ch)
1317 {
1318     acl_tos *l;
1319
1320     for (l = head; l; l = l->next) {
1321         if (ch->matchAclListFast(l->aclList))
1322             return l->tos;
1323     }
1324
1325     return 0;
1326 }
1327
1328 IPAddress
1329 getOutgoingAddr(HttpRequest * request, struct peer *dst_peer)
1330 {
1331     ACLChecklist ch;
1332
1333     if (request && request->flags.spoof_client_ip)
1334         return request->client_addr;
1335
1336     ch.dst_peer = dst_peer;
1337
1338     if (request) {
1339         ch.src_addr = request->client_addr;
1340         ch.my_addr = request->my_addr;
1341         ch.request = HTTPMSGLOCK(request);
1342     }
1343
1344     return aclMapAddr(Config.accessList.outgoing_address, &ch);
1345 }
1346
1347 unsigned long
1348 getOutgoingTOS(HttpRequest * request)
1349 {
1350     ACLChecklist ch;
1351
1352     if (request) {
1353         ch.src_addr = request->client_addr;
1354         ch.my_addr = request->my_addr;
1355         ch.request = HTTPMSGLOCK(request);
1356     }
1357
1358     return aclMapTOS(Config.accessList.outgoing_tos, &ch);
1359 }
1360
1361
1362 /**** WIP_FWD_LOG *************************************************************/
1363
1364 #if WIP_FWD_LOG
1365 void
1366 fwdUninit(void)
1367 {
1368     if (NULL == logfile)
1369         return;
1370
1371     logfileClose(logfile);
1372
1373     logfile = NULL;
1374 }
1375
1376 void
1377 fwdLogRotate(void)
1378 {
1379     if (logfile)
1380         logfileRotate(logfile);
1381 }
1382
1383 static void
1384 FwdState::log()
1385 {
1386     if (NULL == logfile)
1387         return;
1388
1389     logfilePrintf(logfile, "%9d.%03d %03d %s %s\n",
1390                   (int) current_time.tv_sec,
1391                   (int) current_time.tv_usec / 1000,
1392                   last_status,
1393                   RequestMethodStr(request->method),
1394                   request->canonical);
1395 }
1396
1397 void
1398 FwdState::status(http_status s)
1399 {
1400     last_status = s;
1401 }
1402
1403 #endif