From: Amos Jeffries <squid3@treenet.co.nz>
Date: Fri, 8 Oct 2010 07:45:07 +0000 (+1300)
Subject: Merge from trunk
X-Git-Tag: take08~55^2~124^2~41
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=9c8a6c3befe1928d464f5e14fe3a2d6b094b652d;p=thirdparty%2Fsquid.git

Merge from trunk
---

9c8a6c3befe1928d464f5e14fe3a2d6b094b652d
diff --cc doc/release-notes/release-3.2.sgml
index 1a1d696062,b5b8e9e673..7c6580e91c
--- a/doc/release-notes/release-3.2.sgml
+++ b/doc/release-notes/release-3.2.sgml
@@@ -301,19 -301,14 +301,18 @@@ This section gives a thorough account o
  <sect1>New tags<label id="newtags">
  <p>
  <descrip>
- 	<tag>adapted_http_access</tag>
- 	<p>Access control based on altered HTTP request following adaptation alterations (ICAP, eCAP, URL rewriter).
- 	An upgraded drop-in replacement for <em>http_access2</em> found in Squid-2.
+ 	<tag>cpu_affinity_map</tag>
+ 	<p>New setting for SMP support to map Squid processes onto specific CPU cores.
  
 +	<tag>connect_retries</tag>
 +	<p>Replacement for <em>maximum_single_addr_tries</em>, but instead of only applying to hosts with single addresses.
- 	This directive applies to all hosts, extending the number of connection attempts to each IP address.
++	   This directive applies to all hosts, extending the number of connection attempts to each IP address.
 +
  	<tag>else</tag>
- 	<p>Part of conditional SMP support syyntax. see <em>if</em>
+ 	<p>Part of conditional SMP support syntax. see <em>if</em>
  
  	<tag>endif</tag>
- 	<p>Part of conditional SMP support syyntax. see <em>if</em>
+ 	<p>Part of conditional SMP support syntax. see <em>if</em>
  
  	<tag>eui_lookup</tag>
  	<p>Whether to lookup the EUI or MAC address of a connected client.
@@@ -427,10 -454,9 +458,13 @@@
  	<tag>ftp_list_width</tag>
  	<p>Obsolete.
  
+ 	<tag>ignore_expect_100</tag>
+ 	<p>Obsolete.
+ 
 +	<tag>maximum_single_addr_tries</tag>
 +	<p>The behaviour controlled by this directive is no longer possible.
- 	It has been replaced by <em>connect_retries</em> option which operates a little differently.
++	   It has been replaced by <em>connect_retries</em> option which operates a little differently.
 +
  	<tag>url_rewrite_concurrency</tag>
  	<p>Replaced by url_rewrite_children ... concurrency=N option.
  
diff --cc src/client_side_reply.cc
index d76d97b93f,14d5860f5d..e9a1540e2d
--- a/src/client_side_reply.cc
+++ b/src/client_side_reply.cc
@@@ -1972,23 -1977,15 +1976,15 @@@ clientReplyContext::sendMoreData (Store
          body_buf = buf;
      }
  
- #if USE_ZPH_QOS
      if (reqofs==0 && !logTypeIsATcpHit(http->logType)) {
 -        assert(fd >= 0); // the beginning of this method implies fd may be -1
 +        assert(conn != NULL && Comm::IsConnOpen(conn->clientConn)); // the beginning of this method implies FD may be closed.
-         int tos = 0;
-         if (Ip::Qos::TheConfig.tos_sibling_hit && http->request->hier.code==SIBLING_HIT ) {
-             tos = Ip::Qos::TheConfig.tos_sibling_hit;
-             debugs(33, 2, "ZPH: Sibling Peer hit with hier.code=" << http->request->hier.code << ", TOS=" << tos);
-         } else if (Ip::Qos::TheConfig.tos_parent_hit && http->request->hier.code==PARENT_HIT) {
-             tos = Ip::Qos::TheConfig.tos_parent_hit;
-             debugs(33, 2, "ZPH: Parent Peer hit with hier.code=" << http->request->hier.code << ", TOS=" << tos);
-         } else if (Ip::Qos::TheConfig.preserve_miss_tos && Ip::Qos::TheConfig.preserve_miss_tos_mask) {
-             tos = fd_table[conn->clientConn->fd].upstreamTOS & Ip::Qos::TheConfig.preserve_miss_tos_mask;
-             debugs(33, 2, "ZPH: Preserving TOS on miss, TOS="<<tos);
+         if (Ip::Qos::TheConfig.isHitTosActive()) {
 -            Ip::Qos::doTosLocalMiss(fd, http->request->hier.code);
++            Ip::Qos::doTosLocalMiss(conn->clientConn, http->request->hier.code);
+         }
+         if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
 -            Ip::Qos::doNfmarkLocalMiss(fd, http->request->hier.code);
++            Ip::Qos::doNfmarkLocalMiss(conn->clientConn, http->request->hier.code);
          }
-         comm_set_tos(conn->clientConn->fd,tos);
      }
- #endif
  
      /* We've got the final data to start pushing... */
      flags.storelogiccomplete = 1;
diff --cc src/client_side_request.cc
index 595bfce03e,9c939491d2..a1f9889c27
--- a/src/client_side_request.cc
+++ b/src/client_side_request.cc
@@@ -1333,15 -1337,27 +1335,27 @@@ ClientHttpRequest::doCallouts(
          }
      }
  
-     if (!calloutContext->clientside_tos_done) {
-         calloutContext->clientside_tos_done = true;
+     if (!calloutContext->tosToClientDone) {
+         calloutContext->tosToClientDone = true;
 -        if (getConn() != NULL) {
 +        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConn)) {
              ACLFilledChecklist ch(NULL, request, NULL);
              ch.src_addr = request->client_addr;
              ch.my_addr = request->my_addr;
-             int tos = aclMapTOS(Config.accessList.clientside_tos, &ch);
+             tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
              if (tos)
-                 comm_set_tos(getConn()->clientConn->fd, tos);
 -                Ip::Qos::setSockTos(getConn()->fd, tos);
++                Ip::Qos::setSockTos(getConn()->clientConn->fd, tos);
+         }
+     }
+ 
+     if (!calloutContext->nfmarkToClientDone) {
+         calloutContext->nfmarkToClientDone = true;
 -        if (getConn() != NULL) {
++        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConn)) {
+             ACLFilledChecklist ch(NULL, request, NULL);
+             ch.src_addr = request->client_addr;
+             ch.my_addr = request->my_addr;
+             nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
+             if (mark)
 -                Ip::Qos::setSockNfmark(getConn()->fd, mark);
++                Ip::Qos::setSockNfmark(getConn()->clientConn->fd, mark);
          }
      }
  
diff --cc src/comm.cc
index 014562b927,a5a4a7002a..7f506f2511
--- a/src/comm.cc
+++ b/src/comm.cc
@@@ -72,7 -73,7 +73,7 @@@ typedef enum 
  
  static void commStopHalfClosedMonitor(int fd);
  static IOCB commHalfClosedReader;
- static void comm_init_opened(const Comm::ConnectionPointer &conn, unsigned char TOS, const char *note, struct addrinfo *AI);
 -static void comm_init_opened(int new_socket, Ip::Address &addr, tos_t tos, nfmark_t nfmark, const char *note, struct addrinfo *AI);
++static void comm_init_opened(const Comm::ConnectionPointer &conn, tos_t tos, nfmark_t nfmark, const char *note, struct addrinfo *AI);
  static int comm_apply_flags(int new_socket, Ip::Address &addr, int flags, struct addrinfo *AI);
  
  
@@@ -555,22 -594,9 +556,22 @@@ comm_open(int sock_type
            int flags,
            const char *note)
  {
-     return comm_openex(sock_type, proto, addr, flags, 0, note);
+     return comm_openex(sock_type, proto, addr, flags, 0, 0, note);
  }
  
 +void
 +comm_open_listener(int sock_type,
 +                   int proto,
 +                   Comm::ConnectionPointer &conn,
 +                   const char *note)
 +{
 +    /* all listener sockets require bind() */
 +    conn->flags |= COMM_DOBIND;
 +
 +    /* attempt native enabled port. */
 +    conn->fd = comm_openex(sock_type, proto, conn->local, conn->flags, 0, note);
 +}
 +
  int
  comm_open_listener(int sock_type,
                     int proto,
@@@ -716,11 -731,7 +706,11 @@@ comm_openex(int sock_type
      if ( Ip::EnableIpv6&IPV6_SPECIAL_V4MAPPING && addr.IsIPv6() )
          comm_set_v6only(new_socket, 0);
  
 -    comm_init_opened(new_socket, addr, tos, nfmark, note, AI);
 +    // temporary for te transition. comm_openex will eventually have a conn to play with.
 +    Comm::ConnectionPointer temp = new Comm::Connection;
 +    temp->fd = new_socket;
 +    temp->local = addr;
-     comm_init_opened(temp, TOS, note, AI);
++    comm_init_opened(temp, tos, nfmark, note, AI);
      new_socket = comm_apply_flags(new_socket, addr, flags, AI);
  
      addr.FreeAddrInfo(AI);
@@@ -732,26 -743,36 +722,28 @@@
  
  /// update FD tables after a local or remote (IPC) comm_openex();
  void
 -comm_init_opened(int new_socket,
 -                 Ip::Address &addr,
 +comm_init_opened(const Comm::ConnectionPointer &conn,
-                  unsigned char TOS,
+                  tos_t tos,
+                  nfmark_t nfmark,
                   const char *note,
                   struct addrinfo *AI)
  {
 -    assert(new_socket >= 0);
 +    assert(Comm::IsConnOpen(conn));
      assert(AI);
  
 -    fde *F = NULL;
 -
      /* update fdstat */
 -    debugs(5, 5, "comm_open: FD " << new_socket << " is a new socket");
 -
 -    assert(!isOpen(new_socket));
 -    fd_open(new_socket, FD_SOCKET, note);
 -
 -    fdd_table[new_socket].close_file = NULL;
 +    debugs(5, 5, HERE << conn << " is a new socket");
  
 -    fdd_table[new_socket].close_line = 0;
 +    assert(!isOpen(conn->fd)); // NP: global isOpen checks the fde entry for openness not the Comm::Connection
 +    fd_open(conn->fd, FD_SOCKET, note);
  
 -    F = &fd_table[new_socket];
 -
 -    F->local_addr = addr;
 +    fdd_table[conn->fd].close_file = NULL;
 +    fdd_table[conn->fd].close_line = 0;
  
 +    fde *F = &fd_table[conn->fd];
 +    F->local_addr = conn->local;
-     F->tos = TOS;
+     F->tosToServer = tos;
 -
+     F->nfmarkToServer = nfmark;
 -
      F->sock_family = AI->ai_family;
  }
  
@@@ -823,27 -846,27 +815,27 @@@ comm_import_opened(const Comm::Connecti
                     const char *note,
                     struct addrinfo *AI)
  {
 -    debugs(5, 2, HERE << " FD " << fd << " at " << addr);
 -    assert(fd >= 0);
 +    debugs(5, 2, HERE << conn);
 +    assert(Comm::IsConnOpen(conn));
      assert(AI);
  
-     comm_init_opened(conn, 0, note, AI);
 -    comm_init_opened(fd, addr, 0, 0, note, AI);
++    comm_init_opened(conn, 0, 0, note, AI);
  
 -    if (!(flags & COMM_NOCLOEXEC))
 -        fd_table[fd].flags.close_on_exec = 1;
 +    if (!(conn->flags & COMM_NOCLOEXEC))
 +        fd_table[conn->fd].flags.close_on_exec = 1;
  
 -    if (addr.GetPort() > (u_short) 0) {
 +    if (conn->local.GetPort() > (u_short) 0) {
  #ifdef _SQUID_MSWIN_
 -        if (sock_type != SOCK_DGRAM)
 +        if (AI->ai_socktype != SOCK_DGRAM)
  #endif
 -            fd_table[fd].flags.nolinger = 1;
 +            fd_table[conn->fd].flags.nolinger = 1;
      }
  
 -    if ((flags & COMM_TRANSPARENT))
 -        fd_table[fd].flags.transparent = 1;
 +    if ((conn->flags & COMM_TRANSPARENT))
 +        fd_table[conn->fd].flags.transparent = 1;
  
 -    if (flags & COMM_NONBLOCKING)
 -        fd_table[fd].flags.nonblocking = 1;
 +    if (conn->flags & COMM_NONBLOCKING)
 +        fd_table[conn->fd].flags.nonblocking = 1;
  
  #ifdef TCP_NODELAY
      if (AI->ai_socktype == SOCK_STREAM)
diff --cc src/comm.h
index aadecae261,19e0f0c06b..d2179c3ff4
--- a/src/comm.h
+++ b/src/comm.h
@@@ -53,12 -72,10 +53,11 @@@ SQUIDCEXTERN void comm_import_opened(co
   * (in debugs or cachemgr) will occur in Native IPv4 format.
   * A reconfigure is needed to reset the stored IP in most cases and attempt a port re-open.
   */
 -SQUIDCEXTERN int comm_open_listener(int sock_type, int proto, Ip::Address &addr, int flags, const char *note);
 +extern int comm_open_listener(int sock_type, int proto, Ip::Address &addr, int flags, const char *note);
 +extern void comm_open_listener(int sock_type, int proto, Comm::ConnectionPointer &conn, const char *note);
  
- SQUIDCEXTERN int comm_openex(int, int, Ip::Address &, int, unsigned char TOS, const char *);
+ SQUIDCEXTERN int comm_openex(int, int, Ip::Address &, int, tos_t tos, nfmark_t nfmark, const char *);
  SQUIDCEXTERN u_short comm_local_port(int fd);
- SQUIDCEXTERN int comm_set_tos(int fd, int tos);
  
  SQUIDCEXTERN void commSetSelect(int, unsigned int, PF *, void *, time_t);
  SQUIDCEXTERN void commResetSelect(int);
diff --cc src/forward.cc
index 65d5319a22,331ece4cdc..e2e2546715
--- a/src/forward.cc
+++ b/src/forward.cc
@@@ -45,9 -42,9 +45,10 @@@
  #include "hier_code.h"
  #include "HttpReply.h"
  #include "HttpRequest.h"
+ #include "ip/QosConfig.h"
  #include "MemObject.h"
  #include "pconn.h"
 +#include "PeerSelectState.h"
  #include "SquidTime.h"
  #include "Store.h"
  #include "icmp/net_db.h"
@@@ -678,49 -696,63 +679,49 @@@ FwdState::initiateSSL(
  #endif
  
  void
 -FwdState::connectDone(int aServerFD, const DnsLookupDetails &dns, comm_err_t status, int xerrno)
 +FwdState::connectDone(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno)
  {
 -    FwdServer *fs = servers;
 -    assert(server_fd == aServerFD);
 -
 -    request->recordLookup(dns);
 -
 -    if (Config.onoff.log_ip_on_direct && status != COMM_ERR_DNS && fs->code == HIER_DIRECT)
 -        updateHierarchyInfo();
 -
 -    if (status == COMM_ERR_DNS) {
 -        /*
 -         * Only set the dont_retry flag if the DNS lookup fails on
 -         * a direct connection.  If DNS lookup fails when trying
 -         * a neighbor cache, we may want to retry another option.
 -         */
 -
 -        if (NULL == fs->_peer)
 -            flags.dont_retry = 1;
 -
 -        debugs(17, 4, "fwdConnectDone: Unknown host: " << request->GetHost());
 +    if (status != COMM_OK) {
-         ErrorState *anErr = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
++        ErrorState *const anErr = makeConnectingError(ERR_CONNECT_FAIL);
 +        anErr->xerrno = xerrno;
 +        fail(anErr);
  
 -        ErrorState *const anErr = makeConnectingError(ERR_DNS_FAIL);
 +        /* it might have been a timeout with a partially open link */
 +        if (conn != NULL) {
 +            if (conn->getPeer())
 +                peerConnectFailed(conn->getPeer());
  
 -        anErr->dnsError = dns.error;
 +            conn->close();
 +        }
 +        retryOrBail();
 +        return;
 +    }
  
 -        fail(anErr);
 +    serverConn = conn;
  
 -        comm_close(server_fd);
 -    } else if (status != COMM_OK) {
 -        assert(fs);
 -        ErrorState *const anErr = makeConnectingError(ERR_CONNECT_FAIL);
 -        anErr->xerrno = xerrno;
 +#if REDUNDANT_NOW
 +    if (Config.onoff.log_ip_on_direct && serverConnection()->peerType == HIER_DIRECT)
 +        updateHierarchyInfo();
 +#endif
  
 -        fail(anErr);
 +    debugs(17, 3, HERE << serverConnection() << ": '" << entry->url() << "'" );
  
 -        if (fs->_peer)
 -            peerConnectFailed(fs->_peer);
 +    comm_add_close_handler(serverConnection()->fd, fwdServerClosedWrapper, this);
  
 -        comm_close(server_fd);
 -    } else {
 -        debugs(17, 3, "fwdConnectDone: FD " << server_fd << ": '" << entry->url() << "'" );
 +    if (serverConnection()->getPeer())
 +        peerConnectSucceded(serverConnection()->getPeer());
  
 -        if (fs->_peer)
 -            peerConnectSucceded(fs->_peer);
 +    updateHierarchyInfo();
  
  #if USE_SSL
 -
 -        if ((fs->_peer && fs->_peer->use_ssl) ||
 -                (!fs->_peer && request->protocol == PROTO_HTTPS)) {
 -            initiateSSL();
 -            return;
 -        }
 -
 -#endif
 -        dispatch();
 +    if ((serverConnection()->getPeer() && serverConnection()->getPeer()->use_ssl) ||
 +            (!serverConnection()->getPeer() && request->protocol == PROTO_HTTPS)) {
 +        initiateSSL();
 +        return;
      }
 +#endif
 +
 +    dispatch();
  }
  
  void
@@@ -855,11 -997,11 +856,22 @@@ FwdState::dispatch(
       * is attached to something and will be deallocated when server_fd
       * is closed.
       */
 -    assert(server_fd > -1);
 +    assert(Comm::IsConnOpen(serverConn));
 +
++    tos_t tos = GetTosToServer(request);
++
++#if SO_MARK
++    nfmark_t mark = GetNfmarkToServer(request);
++    debugs(17, 3, HERE << "got outgoing addr " << serverConn << ", tos " << int(tos)
++                    << ", netfilter mark " << mark);
++#else
++    nfmark_t mark = 0;
++    debugs(17, 3, HERE << "got outgoing addr " << serverConn << ", tos " << int(tos));
++#endif
+ 
 -    fd_note(server_fd, entry->url());
 +    fd_note(serverConnection()->fd, entry->url());
  
 -    fd_table[server_fd].noteUse(fwdPconnPool);
 +    fd_table[serverConnection()->fd].noteUse(fwdPconnPool);
  
      /*assert(!EBIT_TEST(entry->flags, ENTRY_DISPATCHED)); */
      assert(entry->ping_status != PING_WAITING);
@@@ -870,44 -1012,40 +882,40 @@@
  
      netdbPingSite(request->GetHost());
  
- #if USE_ZPH_QOS && defined(_SQUID_LINUX_)
-     /* Bug 2537: This part of ZPH only applies to patched Linux kernels. */
+     /* Update server side TOS and Netfilter mark if using persistent connections. */
+     if (Config.onoff.server_pconns) {
+         if (Ip::Qos::TheConfig.isAclTosActive()) {
+             tos_t tos = GetTosToServer(request);
+             Ip::Qos::setSockTos(server_fd, tos);
+         }
+ #if SO_MARK
+         if (Ip::Qos::TheConfig.isAclNfmarkActive()) {
+             nfmark_t mark = GetNfmarkToServer(request);
+             Ip::Qos::setSockNfmark(server_fd, mark);
+         }
+ #endif
+     }
  
-     /* Retrieves remote server TOS value, and stores it as part of the
+     /* Retrieves remote server TOS or MARK value, and stores it as part of the
       * original client request FD object. It is later used to forward
-      * remote server's TOS in the response to the client in case of a MISS.
+      * remote server's TOS/MARK in the response to the client in case of a MISS.
       */
-     fde * clientFde = &fd_table[clientConn->fd];
-     if (clientFde) {
-         int tos = 1;
-         int tos_len = sizeof(tos);
-         clientFde->upstreamTOS = 0;
-         if (setsockopt(serverConnection()->fd, SOL_IP, IP_RECVTOS, &tos, tos_len)==0) {
-             unsigned char buf[512];
-             int len = 512;
-             if (getsockopt(serverConnection()->fd, SOL_IP, IP_PKTOPTIONS, buf, (socklen_t*)&len) == 0) {
-                 /* Parse the PKTOPTIONS structure to locate the TOS data message
-                  * prepared in the kernel by the ZPH incoming TCP TOS preserving
-                  * patch.
-                  */
-                 unsigned char * pbuf = buf;
-                 while (pbuf-buf < len) {
-                     struct cmsghdr *o = (struct cmsghdr*)pbuf;
-                     if (o->cmsg_len<=0)
-                         break;
- 
-                     if (o->cmsg_level == SOL_IP && o->cmsg_type == IP_TOS) {
-                         int *tmp = (int*)CMSG_DATA(o);
-                         clientFde->upstreamTOS = (unsigned char)*tmp;
-                         break;
-                     }
-                     pbuf += CMSG_LEN(o->cmsg_len);
-                 }
-             } else {
-                 debugs(33, DBG_IMPORTANT, "ZPH: error in getsockopt(IP_PKTOPTIONS) on FD " << serverConnection()->fd << " " << xstrerror());
-             }
-         } else {
-             debugs(33, DBG_IMPORTANT, "ZPH: error in setsockopt(IP_RECVTOS) on FD " << serverConnection()->fd << " " << xstrerror());
+     if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
 -        fde * clientFde = &fd_table[client_fd];
 -        fde * servFde = &fd_table[server_fd];
++        fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
++        fde * servFde = &fd_table[serverConnection()->fd];
+         if (clientFde && servFde) {
+             /* Get the netfilter mark for the connection */
 -            Ip::Qos::getNfmarkFromServer(server_fd, servFde, clientFde);
++            Ip::Qos::getNfmarkFromServer(serverConnection()->fd, servFde, clientFde);
+         }
+     }
+ 
+ #if _SQUID_LINUX_
+     /* Bug 2537: The TOS forward part of QOS only applies to patched Linux kernels. */
+     if (Ip::Qos::TheConfig.isHitTosActive()) {
 -        fde * clientFde = &fd_table[client_fd];
++        fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos
+         if (clientFde) {
+             /* Get the TOS value for the packet */
 -            Ip::Qos::getTosFromServer(server_fd, clientFde);
++            Ip::Qos::getTosFromServer(serverConnection()->fd, clientFde);
          }
      }
  #endif
@@@ -1195,25 -1393,31 +1217,39 @@@ aclMapTOS(acl_tos * head, ACLChecklist 
      return 0;
  }
  
+ /// Checks for a netfilter mark value to apply depending on the ACL
+ nfmark_t
+ aclMapNfmark(acl_nfmark * head, ACLChecklist * ch)
+ {
+     acl_nfmark *l;
+ 
+     for (l = head; l; l = l->next) {
+         if (!l->aclList || ch->matchAclListFast(l->aclList))
+             return l->nfmark;
+     }
+ 
+     return 0;
+ }
+ 
 -Ip::Address
 -getOutgoingAddr(HttpRequest * request, struct peer *dst_peer)
 +void
 +getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn)
  {
 +    /* skip if an outgoing address is already set. */
 +    if (!conn->local.IsAnyAddr()) return;
 +
 +    // maybe use TPROXY client address
      if (request && request->flags.spoof_client_ip) {
 -        if (!dst_peer || !dst_peer->options.no_tproxy) {
 +        if (!conn->getPeer() || !conn->getPeer()->options.no_tproxy) {
  #if FOLLOW_X_FORWARDED_FOR && LINUX_NETFILTER
              if (Config.onoff.tproxy_uses_indirect_client)
 -                return request->indirect_client_addr;
 +                conn->local = request->indirect_client_addr;
              else
  #endif
 -                return request->client_addr;
 +                conn->local = request->client_addr;
 +            // some flags need setting on the socket to use this address
 +            conn->flags |= COMM_DOBIND;
 +            conn->flags |= COMM_TRANSPARENT;
 +            return;
          }
          // else no tproxy today ...
      }
@@@ -1239,22 -1439,24 +1275,35 @@@
          ch.my_addr = request->my_addr;
      }
  
 -    return aclMapAddr(Config.accessList.outgoing_address, &ch);
 +    acl_address *l;
 +    for (l = Config.accessList.outgoing_address; l; l = l->next) {
 +
 +        /* check if the outgoing address is usable to the destination */
 +        if (conn->remote.IsIPv4() != l->addr.IsIPv4()) continue;
 +
 +        /* check ACLs for this outgoing address */
 +        if (!l->aclList || ch.matchAclListFast(l->aclList)) {
 +            conn->local = l->addr;
 +            return;
 +        }
 +    }
  }
  
- unsigned long
- getOutgoingTOS(HttpRequest * request)
+ tos_t
+ GetTosToServer(HttpRequest * request)
+ {
+     ACLFilledChecklist ch(NULL, request, NULL);
+ 
+     if (request) {
+         ch.src_addr = request->client_addr;
+         ch.my_addr = request->my_addr;
+     }
+ 
+     return aclMapTOS(Ip::Qos::TheConfig.tosToServer, &ch);
+ }
+ 
+ nfmark_t
+ GetNfmarkToServer(HttpRequest * request)
  {
      ACLFilledChecklist ch(NULL, request, NULL);
  
diff --cc src/forward.h
index 7cef702f79,5cf645bd41..34a97d3749
--- a/src/forward.h
+++ b/src/forward.h
@@@ -7,11 -7,30 +7,24 @@@ class ErrorState
  class HttpRequest;
  
  #include "comm.h"
 -#include "hier_code.h"
 +#include "comm/Connection.h"
 +#include "fde.h"
  #include "ip/Address.h"
 +#include "Array.h"
  
+ /**
+  * Returns the TOS value that we should be setting on the connection
+  * to the server, based on the ACL.
+  */
+ tos_t GetTosToServer(HttpRequest * request);
+ 
+ /**
+  * Returns the Netfilter mark value that we should be setting on the
+  * connection to the server, based on the ACL.
+  */
+ nfmark_t GetNfmarkToServer(HttpRequest * request);
+ 
+ 
 -class FwdServer
 -{
 -public:
 -    peer *_peer;                /* NULL --> origin server */
 -    hier_code code;
 -    FwdServer *next;
 -};
 -
  class FwdState : public RefCountable
  {
  public:
diff --cc src/protos.h
index c04965c91d,2d284779dc..2edde17fcc
--- a/src/protos.h
+++ b/src/protos.h
@@@ -407,9 -410,7 +407,9 @@@ SQUIDCEXTERN void peerDigestNeeded(Peer
  SQUIDCEXTERN void peerDigestNotePeerGone(PeerDigest * pd);
  SQUIDCEXTERN void peerDigestStatsReport(const PeerDigest * pd, StoreEntry * e);
  
 +#include "comm/forward.h"
 +extern void getOutgoingAddress(HttpRequest * request, Comm::ConnectionPointer conn);
- unsigned long getOutgoingTOS(HttpRequest * request);
+ extern Ip::Address getOutgoingAddr(HttpRequest * request, struct peer *dst_peer);
  
  SQUIDCEXTERN void urnStart(HttpRequest *, StoreEntry *);
  
diff --cc src/tools.cc
index 7f931bec36,2ccfb99a13..0799c65919
--- a/src/tools.cc
+++ b/src/tools.cc
@@@ -33,18 -33,13 +33,14 @@@
   */
  
  #include "squid.h"
- // XXX: these should be already pulled in...
- //#include "compat/initgroups.h"
- //#include "compat/getaddrinfo.h"
- //#include "compat/getnameinfo.h"
- //#include "compat/tempnam.h"
 -#include "compat/initgroups.h"
 -#include "compat/getaddrinfo.h"
 -#include "compat/getnameinfo.h"
 -#include "compat/tempnam.h"
 +
 +#include "base/Subscription.h"
  #include "fde.h"
 +#include "ICP.h"
  #include "ip/Intercept.h"
+ #include "ip/QosConfig.h"
 +#include "ipc/Coordinator.h"
 +#include "ipc/Kids.h"
  #include "MemBuf.h"
  #include "ProtoPort.h"
  #include "SquidMath.h"
diff --cc src/tunnel.cc
index 1c1c0a6a09,b0489e0ea4..64605a6178
--- a/src/tunnel.cc
+++ b/src/tunnel.cc
@@@ -508,67 -560,51 +508,70 @@@ tunnelConnectDone(const Comm::Connectio
      HttpRequest *request = tunnelState->request;
      ErrorState *err = NULL;
  
 -    request->recordLookup(dns);
 +#if DELAY_POOLS
 +    /* no point using the delayIsNoDelay stuff since tunnel is nice and simple */
 +    if (conn->getPeer() && conn->getPeer()->options.no_delay)
 +        tunnelState->server.setDelayId(DelayId());
 +#endif
  
 -    if (tunnelState->servers->_peer)
 -        hierarchyNote(&tunnelState->request->hier, tunnelState->servers->code,
 -                      tunnelState->servers->_peer->host);
 +    if (conn != NULL && conn->getPeer())
 +        hierarchyNote(&tunnelState->request->hier, conn->peerType, conn->getPeer()->host);
      else if (Config.onoff.log_ip_on_direct)
 -        hierarchyNote(&tunnelState->request->hier, tunnelState->servers->code,
 -                      fd_table[tunnelState->server.fd()].ipaddr);
 +        hierarchyNote(&tunnelState->request->hier, conn->peerType, fd_table[conn->fd].ipaddr);
      else
 -        hierarchyNote(&tunnelState->request->hier, tunnelState->servers->code,
 -                      tunnelState->host);
 -
 -    if (status == COMM_ERR_DNS) {
 -        debugs(26, 4, "tunnelConnect: Unknown host: " << tunnelState->host);
 -        err = errorCon(ERR_DNS_FAIL, HTTP_NOT_FOUND, request);
 -        *tunnelState->status_ptr = HTTP_NOT_FOUND;
 -        err->dnsError = dns.error;
 -        err->callback = tunnelErrorComplete;
 -        err->callback_data = tunnelState;
 -        errorSend(tunnelState->client.fd(), err);
 -    } else if (status != COMM_OK) {
 -        err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
 -        *tunnelState->status_ptr = HTTP_SERVICE_UNAVAILABLE;
 -        err->xerrno = xerrno;
 -        err->port = tunnelState->port;
 -        err->callback = tunnelErrorComplete;
 -        err->callback_data = tunnelState;
 -        errorSend(tunnelState->client.fd(), err);
 -    } else {
 -        if (tunnelState->servers->_peer)
 -            tunnelProxyConnected(tunnelState->server.fd(), tunnelState);
 -        else {
 -            tunnelConnected(tunnelState->server.fd(), tunnelState);
 +        hierarchyNote(&tunnelState->request->hier, conn->peerType, tunnelState->getHost());
 +
 +    // TODO: merge this into hierarchyNote with a conn parameter instead of peerType
 +    request->hier.peer_local_port = conn->local.GetPort(); // for %<lp logging
 +
 +    if (status != COMM_OK) {
 +        /* At this point only the TCP handshake has failed. no data has been passed.
 +         * we are allowed to re-try the TCP-level connection to alternate IPs for CONNECT.
 +         */
 +        tunnelState->serverDestinations.shift();
 +        if (status != COMM_TIMEOUT && tunnelState->serverDestinations.size() > 0) {
 +            /* Try another IP of this destination host */
 +            AsyncCall::Pointer call = commCbCall(26,3, "tunnelConnectDone", CommConnectCbPtrFun(tunnelConnectDone, tunnelState));
 +            Comm::ConnOpener *cs = new Comm::ConnOpener(tunnelState->serverDestinations[0], call, Config.Timeout.connect);
 +            cs->setHost(tunnelState->url);
 +            AsyncJob::Start(cs);
 +        } else {
 +            err = errorCon(ERR_CONNECT_FAIL, HTTP_SERVICE_UNAVAILABLE, request);
 +            *tunnelState->status_ptr = HTTP_SERVICE_UNAVAILABLE;
 +            err->xerrno = xerrno;
 +            // on timeout is this still:    err->xerrno = ETIMEDOUT;
 +            err->port = conn->remote.GetPort();
 +            err->callback = tunnelErrorComplete;
 +            err->callback_data = tunnelState;
 +            errorSend(tunnelState->client.conn, err);
          }
 +        return;
 +    }
 +
 +    tunnelState->server.conn = conn;
 +    request->peer_host = conn->getPeer() ? conn->getPeer()->host : NULL;
 +    comm_add_close_handler(conn->fd, tunnelServerClosed, tunnelState);
 +
 +    if (conn->getPeer()) {
 +        tunnelState->request->peer_login = conn->getPeer()->login;
 +        tunnelState->request->flags.proxying = 1;
 +    } else {
 +        tunnelState->request->peer_login = NULL;
 +        tunnelState->request->flags.proxying = 0;
 +    }
  
 -        commSetTimeout(tunnelState->server.fd(),
 -                       Config.Timeout.read,
 -                       tunnelTimeout,
 -                       tunnelState);
 +    if (conn->getPeer())
 +        tunnelRelayConnectRequest(conn, tunnelState);
 +    else {
 +        tunnelConnected(conn, tunnelState);
      }
 +
 +    commSetTimeout(conn->fd, Config.Timeout.read, tunnelTimeout, tunnelState);
  }
  
+ extern tos_t GetTosToServer(HttpRequest * request);
+ extern nfmark_t GetNfmarkToServer(HttpRequest * request);
+ 
  void
  tunnelStart(ClientHttpRequest * http, int64_t * size_ptr, int *status_ptr)
  {