From: Amos Jeffries <squid3@treenet.co.nz>
Date: Wed, 6 Oct 2010 03:50:45 +0000 (+1300)
Subject: Author: Andrew Beverley <andy@andybev.com>
X-Git-Tag: take1~198
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=425de4c8495f778cb43fc61df608575cedf8b60c;p=thirdparty%2Fsquid.git

Author: Andrew Beverley <andy@andybev.com>
Netfilter MARK support for QoS
---

diff --git a/CREDITS b/CREDITS
index 617e47a133..3138fc61af 100644
--- a/CREDITS
+++ b/CREDITS
@@ -342,6 +342,28 @@ lib/strnstr.cc:
 
 ==============================================================================
 
+lib/xstrto.cc:
+
+/*
+ * (C) 2000-2006 by the netfilter coreteam <coreteam@netfilter.org>:
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+==============================================================================
+
 lib/getopt.c:
 
 /*
diff --git a/configure.in b/configure.in
index 98c8be27ba..b30f1bf856 100644
--- a/configure.in
+++ b/configure.in
@@ -1309,6 +1309,38 @@ AC_ARG_ENABLE(linux-netfilter,
 AC_MSG_NOTICE([Linux Netfilter support requested: ${enable_linux_netfilter:=auto}])
 #will be AC_DEFINEd later, after checking for appropriate infrastructure
 
+
+dnl Look for libnetfilter_conntrack options (needed for QOS netfilter marking)
+AC_ARG_WITH(netfilter-conntrack,
+  AS_HELP_STRING([--without-netfilter-conntrack],
+                 [Do not use Netfilter conntrack libraries for packet marking.
+                  A path to alternative library location may be specified by
+                  using --with-netfilter-conntrack=PATH. Default: auto-detect.]), [
+    case "$withval" in
+      yes|no) with_netfilter_conntrack=$withval ;;
+      *)   squid_opt_netfilterconntrackpath=$withval
+           with_netfilter_conntrack=yes ;;
+    esac], [
+    with_netfilter_conntrack=yes
+])
+if test x"$with_netfilter_conntrack" = "xyes"; then
+    if test "x$squid_opt_netfilterconntrackpath" != "x"; then
+        if ! test -d "$squid_opt_netfilterconntrackpath"; then
+            AC_MSG_ERROR([--with-netfilter-conntrack path '$squid_opt_netfilterconntrackpath' does not exist])
+        fi
+        LDFLAGS="-L$squid_opt_netfilterconntrackpath/lib $LDFLAGS"
+        CPPFLAGS="-I$squid_opt_netfilterconntrackpath/include $CPPFLAGS"
+    fi
+    AC_SEARCH_LIBS([nfct_query], [netfilter_conntrack],,
+        with_netfilter_conntrack=no
+        if test x"$withval" = "xyes"; then
+            AC_MSG_ERROR([--with-netfilter-conntrack specified but libnetfilter-conntrack libraries not found])
+        fi )
+    AC_CHECK_HEADERS([libnetfilter_conntrack/libnetfilter_conntrack.h \
+        libnetfilter_conntrack/libnetfilter_conntrack_tcp.h],,with_netfilter_conntrack=no)
+fi
+
+
 dnl Enable Large file support
 buildmodel=""
 squid_opt_enable_large_files=no
@@ -2039,10 +2071,15 @@ AC_ARG_ENABLE(zph-qos,
 SQUID_YESNO([$enableval],
             [unrecognized argument to --enable-zph-qos: $enableval])
 ])
-SQUID_DEFINE_BOOL(USE_ZPH_QOS,${enable_zph_qos:=no},
+SQUID_DEFINE_BOOL(USE_QOS_TOS,${enable_zph_qos:=yes},
           [Enable Zero Penalty Hit QOS. When set, Squid will alter the
            TOS field of HIT responses to help policing network traffic])
 AC_MSG_NOTICE([ZPH QOS enabled: $enable_zph_qos])
+if test x"$enable_zph_qos" = "xyes" ; then
+        AC_MSG_NOTICE([QOS netfilter mark preservation enabled: $with_netfilter_conntrack])
+        SQUID_DEFINE_BOOL(USE_LIBNETFILTERCONNTRACK,${with_netfilter_conntrack:=no},
+                      [Enable support for QOS netfilter mark preservation])
+fi
 
 if $CPPUNITCONFIG --help >/dev/null; then
   squid_cv_cppunit_version="`$CPPUNITCONFIG --version`"
@@ -2133,6 +2170,7 @@ AC_CHECK_HEADERS( \
   lber.h \
   ldap.h \
   libc.h \
+  limits \
   limits.h \
   linux/posix_types.h \
   linux/types.h \
@@ -2164,6 +2202,7 @@ AC_CHECK_HEADERS( \
   signal.h \
   sstream \
   stdarg.h \
+  stdbool.h \
   stddef.h \
   stdexcept \
   stdio.h \
diff --git a/doc/release-notes/release-3.2.sgml b/doc/release-notes/release-3.2.sgml
index 2082977749..b5b8e9e673 100644
--- a/doc/release-notes/release-3.2.sgml
+++ b/doc/release-notes/release-3.2.sgml
@@ -385,6 +385,11 @@ This section gives a thorough account of those changes in three categories:
 	<p><em>htcp-*</em> options collapsed into <em>htcp=</em> taking an optional comma-separated list of flags.
 	   The old form is deprecated but still accepted.
 
+	<tag>clientside_mark</tag>
+	<p>New configuration parameter <em>clientside_mark</em>
+	<p>Allows packets leaving Squid on the client side to be marked with a Netfilter mark value in the same way as the existing clientside_tos feature.
+	<p>This feature is only available for Netfilter environments.
+
 	<tag>deny_info</tag>
 	<p>Support URL format tags. For dynamically generated URL in denial redirect.
 
@@ -410,6 +415,25 @@ This section gives a thorough account of those changes in three categories:
 	<p>Please check and update your squid.conf to use the text <em>none</em> for no limit instead of the old 0 (zero).
 	<p>All users upgrading need to be aware that from Squid-3.3 setting this option to 0 (zero) will mean zero bytes of memory get pooled.
 
+	<tag>qos_flows</tag>
+	<p>New options <em>mark</em> and <em>tos</em> and <em>miss</em>
+	<p><em>tos</em> retains the original QOS functionality of the IP header TOS field.
+	<p><em>mark</em> offers the same functionality, but with a netfilter mark value.
+	<p>These options should be placed immediately after qos_flows.
+	<p>The <em>tos</em> value is optional in order to maintain backwards compatability.
+	<p>The preserve-miss functionality is available with the <em>mark</em> option and requires no kernel patching.
+	   It does, however, require libnetfilter_conntrack.
+	   This will be included by default if available (see the --without-netfilter-conntrack configure option for more details).
+	<p><em>miss</em> sets a value for a cache miss. It is available for both the tos and mark options and takes precedence over the preserve-miss feature.
+
+	<tag>tcp_outgoing_mark</tag>
+	<p>New configuration parameter <em>tcp_outgoing_mark</em>
+	<p>Allows packets leaving Squid on the server side to be marked with a Netfilter mark value in the same way as the existing tcp_outgoing_tos feature.
+	<p>This feature is only available for Netfilter environments.
+
+	<tag>tcp_outgoing_tos</tag>
+	<p>This parameter is now compatible with persistent server connections.
+
 	<tag>windows_ipaddrchangemonitor</tag>
 	<p>Now only available to be set in Windows builds.
 
@@ -499,6 +523,10 @@ This section gives an account of those changes in three categories:
 	   to have any effect on existing builds other than fixing some mysterious lack of core dumps.
 	   The old /var/cache location was often non-writable which blocked core dumps creation.
 
+	<tag>--without-netfiler-conntrack</tag>
+	<p>Disables the libnetfilter_conntrack library being used for the new qos_flows option <em>mark</em>.
+	   default is to auto-detect the library and use where available.
+
 </descrip>
 
 <sect1>Changes to existing options<label id="modifiedoptions">
diff --git a/include/config.h b/include/config.h
index 4ae2da08ff..386aa89e56 100644
--- a/include/config.h
+++ b/include/config.h
@@ -167,4 +167,9 @@
  */
 #include "strnstr.h"
 
+/*
+ * xstrtoul() and xstrtoui() are strtoul() and strtoui() with limits.
+ */
+#include "xstrto.h"
+
 #endif /* SQUID_CONFIG_H */
diff --git a/include/xstrto.h b/include/xstrto.h
new file mode 100644
index 0000000000..2a8b6d735f
--- /dev/null
+++ b/include/xstrto.h
@@ -0,0 +1,32 @@
+#ifndef _SQUID_XSTRTO_H
+#define _SQUID_XSTRTO_H
+
+#include "config.h"
+
+#ifdef HAVE_STDBOOL_H
+#include <stdbool.h>
+#endif
+
+/**
+ * xstrtou{i,l} - string to number conversion
+ * \param s     input string
+ * \param end   like strtoul's "end" pointer
+ * \param value pointer for result. Undefined on failure
+ * \param min   minimum accepted value
+ * \param max   maximum accepted value
+ *
+ * If @end is NULL, we assume the caller wants a "strict strtoul", and hence
+ * "15a" is rejected.
+ * In either case, the value obtained is compared for min-max compliance.
+ * Base is always 0, i.e. autodetect depending on @s.
+ *
+ * \return true/false whether number was accepted. On failure, *value has
+ * undefined contents.
+ */
+bool xstrtoul(const char *s, char **end, unsigned long *value,
+              unsigned long min, unsigned long max);
+
+bool xstrtoui(const char *s, char **end, unsigned int *value,
+              unsigned int min, unsigned int max);
+
+#endif /* _SQUID_XSTRTO_H */
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 1feae7b30c..e8d71bf882 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -71,6 +71,7 @@ libmiscutil_a_SOURCES = \
 	stub_memaccount.c \
 	util.c \
 	uudecode.c \
+	xstrto.cc \
 	xusleep.c \
 	$(XPROF_STATS_SOURCE) \
 	$(WIN32SRC)
diff --git a/lib/xstrto.cc b/lib/xstrto.cc
new file mode 100644
index 0000000000..bac8342d4f
--- /dev/null
+++ b/lib/xstrto.cc
@@ -0,0 +1,95 @@
+#ifndef _SQUID_XSTRTO_C_
+#define _SQUID_XSTRTO_C_
+
+/*
+ *  Shamelessly duplicated from the netfilter iptables sources
+ *  for use by the Squid Project under GNU Public License.
+ *
+ * Reason for use as explained by Luciano Coelho:
+ * "I found that there is a bug in strtoul (and strtoull for
+ * that matter) that causes the long to overflow if there are valid digits
+ * after the maximum possible digits for the base.  For example if you try
+ * to strtoul 0xfffffffff (with 9 f's) the strtoul will overflow and come
+ * up with a bogus result.  I can't easily truncate the string to avoid
+ * this problem, because with decimal or octal, the same valid value would
+ * take more spaces.  I could do some magic here, checking whether it's a
+ * hex, dec or oct and truncate appropriately, but that would be very ugly.
+ * So the simplest way I came up with was to use strtoull and return
+ * -EINVAL if the value exceeds 32 bits."
+ *
+ * Update/Maintenance History:
+ *
+ *    12-Sep-2010 : Copied from iptables xtables.c
+ * 			- xtables_strtoui renamed to xstrtoui
+ * 			- xtables_strtoul renamed to xstrtoul
+ *
+ * Squid VCS $Id$
+ *
+ *  Original License and code follows.
+ */
+
+#include "config.h"
+#include "xstrto.h"
+
+/* 
+ * (C) 2000-2006 by the netfilter coreteam <coreteam@netfilter.org>: 
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.   
+ */
+
+#if HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+bool
+xstrtoul(const char *s, char **end, unsigned long *value,
+         unsigned long min, unsigned long max)
+{
+    unsigned long v;
+    char *my_end;   
+
+    errno = 0;
+    v = strtoul(s, &my_end, 0);
+
+    if (my_end == s)
+        return false;
+    if (end != NULL)
+        *end = my_end;
+
+    if (errno != ERANGE && min <= v && (max == 0 || v <= max)) {
+        if (value != NULL)
+            *value = v;
+        if (end == NULL)   
+            return *my_end == '\0';
+        return true;
+    }
+
+    return false;
+}
+
+bool
+xstrtoui(const char *s, char **end, unsigned int *value,
+         unsigned int min, unsigned int max)
+{
+    unsigned long v;
+    bool ret;
+
+    ret = xstrtoul(s, end, &v, min, max);
+    if (value != NULL)
+        *value = v;
+    return ret;
+}
+
+#endif /* _SQUID_XSTRTO_C_ */
diff --git a/src/ClientRequestContext.h b/src/ClientRequestContext.h
index 0b4dad40a1..98acd4ec7a 100644
--- a/src/ClientRequestContext.h
+++ b/src/ClientRequestContext.h
@@ -50,7 +50,8 @@ public:
     bool redirect_done;
     bool no_cache_done;
     bool interpreted_req_hdrs;
-    bool clientside_tos_done;
+    bool tosToClientDone;
+    bool nfmarkToClientDone;
 
 private:
     CBDATA_CLASS(ClientRequestContext);
diff --git a/src/cache_cf.cc b/src/cache_cf.cc
index 9283849ab1..8aa5138d4b 100644
--- a/src/cache_cf.cc
+++ b/src/cache_cf.cc
@@ -78,6 +78,10 @@
 #include <glob.h>
 #endif
 
+#if HAVE_LIMITS_H
+#include <limits>
+#endif
+
 #if USE_ADAPTATION
 static void parse_adaptation_service_set_type();
 static void parse_adaptation_service_chain_type();
@@ -1104,7 +1108,7 @@ free_acl(ACL ** ae)
     aclDestroyAcls(ae);
 }
 
-static void
+void
 dump_acl_list(StoreEntry * entry, ACLList * head)
 {
     ACLList *l;
@@ -1259,8 +1263,7 @@ parse_acl_tos(acl_tos ** head)
 {
     acl_tos *l;
     acl_tos **tail = head;	/* sane name below */
-    int tos;
-    char junk;
+    unsigned int tos;           /* Initially uint for strtoui. Casted to tos_t before return */
     char *token = strtok(NULL, w_space);
 
     if (!token) {
@@ -1268,12 +1271,7 @@ parse_acl_tos(acl_tos ** head)
         return;
     }
 
-    if (sscanf(token, "0x%x%c", &tos, &junk) != 1) {
-        self_destruct();
-        return;
-    }
-
-    if (tos < 0 || tos > 255) {
+    if (!xstrtoui(token, NULL, &tos, 0, std::numeric_limits<tos_t>::max())) {
         self_destruct();
         return;
     }
@@ -1282,7 +1280,7 @@ parse_acl_tos(acl_tos ** head)
 
     l = cbdataAlloc(acl_tos);
 
-    l->tos = tos;
+    l->tos = (tos_t)tos;
 
     aclParseAclList(LegacyParser, &l->aclList);
 
@@ -1303,6 +1301,75 @@ free_acl_tos(acl_tos ** head)
     }
 }
 
+CBDATA_TYPE(acl_nfmark);
+
+static void
+dump_acl_nfmark(StoreEntry * entry, const char *name, acl_nfmark * head)
+{
+    acl_nfmark *l;
+
+    for (l = head; l; l = l->next) {
+        if (l->nfmark > 0)
+            storeAppendPrintf(entry, "%s 0x%02X", name, l->nfmark);
+        else
+            storeAppendPrintf(entry, "%s none", name);
+
+        dump_acl_list(entry, l->aclList);
+
+        storeAppendPrintf(entry, "\n");
+    }
+}
+
+static void
+freed_acl_nfmark(void *data)
+{
+    acl_nfmark *l = static_cast<acl_nfmark *>(data);
+    aclDestroyAclList(&l->aclList);
+}
+
+static void
+parse_acl_nfmark(acl_nfmark ** head)
+{
+    acl_nfmark *l;
+    acl_nfmark **tail = head;	/* sane name below */
+    nfmark_t mark;
+    char *token = strtok(NULL, w_space);
+
+    if (!token) {
+        self_destruct();
+        return;
+    }
+
+    if (!xstrtoui(token, NULL, &mark, 0, std::numeric_limits<nfmark_t>::max())) {
+        self_destruct();
+        return;
+    }
+
+    CBDATA_INIT_TYPE_FREECB(acl_nfmark, freed_acl_nfmark);
+
+    l = cbdataAlloc(acl_nfmark);
+
+    l->nfmark = mark;
+
+    aclParseAclList(LegacyParser, &l->aclList);
+
+    while (*tail)
+        tail = &(*tail)->next;
+
+    *tail = l;
+}
+
+static void
+free_acl_nfmark(acl_nfmark ** head)
+{
+    while (*head) {
+        acl_nfmark *l = *head;
+        *head = l->next;
+        l->next = NULL;
+        cbdataFree(l);
+    }
+}
+
 CBDATA_TYPE(acl_size_t);
 
 static void
diff --git a/src/cf.data.depend b/src/cf.data.depend
index 25e48df6ff..951ec03297 100644
--- a/src/cf.data.depend
+++ b/src/cf.data.depend
@@ -5,12 +5,14 @@ acl_access		acl
 acl_address		acl
 acl_b_size_t		acl
 acl_tos			acl
+acl_nfmark		acl
 address
 authparam
 b_int64_t
 b_size_t
 cachedir		cache_replacement_policy
 cachemgrpasswd
+ConfigAclTos
 CpuAffinityMap
 debug
 delay_pool_access	acl	delay_class
diff --git a/src/cf.data.pre b/src/cf.data.pre
index 723b9d79dd..b9de3dbce2 100644
--- a/src/cf.data.pre
+++ b/src/cf.data.pre
@@ -1475,11 +1475,10 @@ DOC_END
 NAME: tcp_outgoing_tos tcp_outgoing_ds tcp_outgoing_dscp
 TYPE: acl_tos
 DEFAULT: none
-LOC: Config.accessList.outgoing_tos
+LOC: Ip::Qos::TheConfig.tosToServer
 DOC_START
-	Allows you to select a TOS/Diffserv value to mark outgoing
-	connections with, based on the username or source address
-	making the request.
+	Allows you to select a TOS/Diffserv value for packets outgoing
+	on the server side, based on an ACL.
 
 	tcp_outgoing_tos ds-field [!]aclname ...
 
@@ -1502,42 +1501,97 @@ DOC_START
 
 	Processing proceeds in the order specified, and stops at first fully
 	matching line.
-
-	Note: The use of this directive using client dependent ACLs is
-	incompatible with the use of server side persistent connections. To
-	ensure correct results it is best to set server_persisten_connections
-	to off when using this directive in such configurations.
 DOC_END
 
 NAME: clientside_tos
 TYPE: acl_tos
 DEFAULT: none
-LOC: Config.accessList.clientside_tos
+LOC: Ip::Qos::TheConfig.tosToClient
 DOC_START
-	Allows you to select a TOS/Diffserv value to mark client-side
-	connections with, based on the username or source address
-	making the request.
+	Allows you to select a TOS/Diffserv value for packets being transmitted
+	on the client-side, based on an ACL.
+
+	clientside_tos ds-field [!]aclname ...
+
+	Example where normal_service_net uses the TOS value 0x00
+	and good_service_net uses 0x20
+
+	acl normal_service_net src 10.0.0.0/24
+	acl good_service_net src 10.0.1.0/24
+	clientside_tos 0x00 normal_service_net
+	clientside_tos 0x20 good_service_net
+
+	Note: This feature is incompatible with qos_flows. Any TOS values set here
+	will be overwritten by TOS values in qos_flows.
+DOC_END
+
+NAME: tcp_outgoing_mark
+TYPE: acl_nfmark
+IFDEF: SO_MARK
+DEFAULT: none
+LOC: Ip::Qos::TheConfig.nfmarkToServer
+DOC_START
+	Allows you to apply a Netfilter mark value to outgoing packets
+	on the server side, based on an ACL.
+
+	tcp_outgoing_mark mark-value [!]aclname ...
+
+	Example where normal_service_net uses the mark value 0x00
+	and good_service_net uses 0x20
+
+	acl normal_service_net src 10.0.0.0/24
+	acl good_service_net src 10.0.1.0/24
+	tcp_outgoing_mark 0x00 normal_service_net
+	tcp_outgoing_mark 0x20 good_service_net
+DOC_END
+
+NAME: clientside_mark
+TYPE: acl_nfmark
+IFDEF: SO_MARK
+DEFAULT: none
+LOC: Ip::Qos::TheConfig.nfmarkToClient
+DOC_START
+	Allows you to apply a Netfilter mark value to packets being transmitted
+	on the client-side, based on an ACL.
+
+	clientside_mark mark-value [!]aclname ...
+
+	Example where normal_service_net uses the mark value 0x00
+	and good_service_net uses 0x20
+
+	acl normal_service_net src 10.0.0.0/24
+	acl good_service_net src 10.0.1.0/24
+	clientside_mark 0x00 normal_service_net
+	clientside_mark 0x20 good_service_net
+
+	Note: This feature is incompatible with qos_flows. Any mark values set here
+	will be overwritten by mark values in qos_flows.
 DOC_END
 
 NAME: qos_flows
 TYPE: QosConfig
-IFDEF: USE_ZPH_QOS
+IFDEF: USE_QOS_TOS
 DEFAULT: none
 LOC: Ip::Qos::TheConfig
 DOC_START
 	Allows you to select a TOS/DSCP value to mark outgoing
-	connections with, based on where the reply was sourced.
+	connections with, based on where the reply was sourced.	For
+	platforms using netfilter, allows you to set a netfilter mark
+	value instead of, or in addition to, a TOS value.
 
 	TOS values really only have local significance - so you should
 	know what you're specifying. For more information, see RFC2474,
 	RFC2475, and RFC3260.
 
-	The TOS/DSCP byte must be exactly that - octet value 0x00-0xFF.
-	Note that in practice often only values up to 0x3F are usable
-	as the two highest bits have been redefined for use by ECN
-	(RFC3168).
+	The TOS/DSCP byte must be exactly that - a octet value  0 - 255. Note that
+	in practice often only multiples of 4 is usable as the two rightmost bits
+	have been redefined for use by ECN (RFC 3168 section 23.1).
+
+	Mark values can be any unsigned 32-bit integer value.
 
-	This setting is configured by setting the source TOS values:
+	This setting is configured by setting the following values:
+
+	tos|mark                Whether to set TOS or netfilter mark values
 
 	local-hit=0xFF		Value to mark local cache hits.
 
@@ -1545,23 +1599,35 @@ DOC_START
 
 	parent-hit=0xFF		Value to mark hits from parent peers.
 
+	miss=0xFF		Value to mark cache misses. Takes precedence
+				over the preserve-miss feature (see below).
 
-	NOTE: 'miss' preserve feature is only possible on Linux at this time.
-
-	For the following to work correctly, you will need to patch your
-	linux kernel with the TOS preserving ZPH patch.
-	The kernel patch can be downloaded from http://zph.bratcheda.org
+	The TOS variant of the following features are only possible on Linux
+	and require your kernel to be patched with the TOS preserving ZPH
+	patch, available from http://zph.bratcheda.org
+	No patch is needed to preserve the netfilter mark, which will work
+	with all variants of netfilter.
 
 	disable-preserve-miss
-		By default, the existing TOS value of the response coming
-		from the remote server will be retained and masked with
-		miss-mark. This option disables that feature.
+		This option disables the preservation of the TOS or netfilter
+		mark. By default, the existing TOS or netfilter mark value of
+		the response coming from the remote server will be retained
+		and masked with miss-mark.
+		NOTE: in the case of a netfilter mark, the mark must be set on
+		the connection (using the CONNMARK target) not on the packet
+		(MARK target).
 
 	miss-mask=0xFF
-		Allows you to mask certain bits in the TOS received from the
-		remote server, before copying the value to the TOS sent
-		towards clients.
-		Default: 0xFF (TOS from server is not changed).
+		Allows you to mask certain bits in the TOS or mark value
+		received from the remote server, before copying the value to
+		the TOS sent towards clients.
+		Default for tos: 0xFF (TOS from server is not changed).
+		Default for mark: 0xFFFFFFFF (mark from server is not changed).
+
+	All of these features require the --enable-zph-qos compilation flag
+	(enabled by default). Netfilter marking also requires the
+	libnetfilter_conntrack libraries (--with-netfilter-conntrack) and
+	libcap 2.09+ (--with-libcap).
 
 DOC_END
 
diff --git a/src/cf_gen_defines b/src/cf_gen_defines
index 7b4361e09a..50ce262939 100644
--- a/src/cf_gen_defines
+++ b/src/cf_gen_defines
@@ -30,7 +30,7 @@ BEGIN {
 	define["USE_USERAGENT_LOG"]="--enable-useragent-log"
 	define["USE_WCCP"]="--enable-wccp"
 	define["USE_WCCPv2"]="--enable-wccpv2"
-	define["USE_ZPH_QOS"]="--enable-zph-qos"
+	define["USE_QOS_TOS"]="--enable-zph-qos"
 	define["_SQUID_MSWIN_"]="MS Windows"
 }
 /^IFDEF:/ {
diff --git a/src/client_side_reply.cc b/src/client_side_reply.cc
index 34e3e29e7c..14d5860f5d 100644
--- a/src/client_side_reply.cc
+++ b/src/client_side_reply.cc
@@ -1689,12 +1689,15 @@ clientReplyContext::doGetMoreData()
         /* guarantee nothing has been sent yet! */
         assert(http->out.size == 0);
         assert(http->out.offset == 0);
-#if USE_ZPH_QOS
-        if (Ip::Qos::TheConfig.tos_local_hit) {
-            debugs(33, 2, "ZPH Local hit, TOS=" << Ip::Qos::TheConfig.tos_local_hit);
-            comm_set_tos(http->getConn()->fd, Ip::Qos::TheConfig.tos_local_hit);
+
+        if (Ip::Qos::TheConfig.isHitTosActive()) {
+            Ip::Qos::doTosLocalHit(http->getConn()->fd);
+        }
+
+        if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
+            Ip::Qos::doNfmarkLocalHit(http->getConn()->fd);
         }
-#endif /* USE_ZPH_QOS */
+
         localTempBuffer.offset = reqofs;
         localTempBuffer.length = getNextNode()->readBuffer.length;
         localTempBuffer.data = getNextNode()->readBuffer.data;
@@ -1974,23 +1977,15 @@ clientReplyContext::sendMoreData (StoreIOBuffer result)
         body_buf = buf;
     }
 
-#if USE_ZPH_QOS
     if (reqofs==0 && !logTypeIsATcpHit(http->logType)) {
         assert(fd >= 0); // the beginning of this method implies fd may be -1
-        int tos = 0;
-        if (Ip::Qos::TheConfig.tos_sibling_hit && http->request->hier.code==SIBLING_HIT ) {
-            tos = Ip::Qos::TheConfig.tos_sibling_hit;
-            debugs(33, 2, "ZPH: Sibling Peer hit with hier.code=" << http->request->hier.code << ", TOS=" << tos);
-        } else if (Ip::Qos::TheConfig.tos_parent_hit && http->request->hier.code==PARENT_HIT) {
-            tos = Ip::Qos::TheConfig.tos_parent_hit;
-            debugs(33, 2, "ZPH: Parent Peer hit with hier.code=" << http->request->hier.code << ", TOS=" << tos);
-        } else if (Ip::Qos::TheConfig.preserve_miss_tos && Ip::Qos::TheConfig.preserve_miss_tos_mask) {
-            tos = fd_table[fd].upstreamTOS & Ip::Qos::TheConfig.preserve_miss_tos_mask;
-            debugs(33, 2, "ZPH: Preserving TOS on miss, TOS="<<tos);
+        if (Ip::Qos::TheConfig.isHitTosActive()) {
+            Ip::Qos::doTosLocalMiss(fd, http->request->hier.code);
+        }
+        if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
+            Ip::Qos::doNfmarkLocalMiss(fd, http->request->hier.code);
         }
-        comm_set_tos(fd,tos);
     }
-#endif
 
     /* We've got the final data to start pushing... */
     flags.storelogiccomplete = 1;
diff --git a/src/client_side_request.cc b/src/client_side_request.cc
index 696084f738..9c939491d2 100644
--- a/src/client_side_request.cc
+++ b/src/client_side_request.cc
@@ -63,6 +63,7 @@
 #include "fde.h"
 #include "HttpReply.h"
 #include "HttpRequest.h"
+#include "ip/QosConfig.h"
 #include "MemObject.h"
 #include "ProtoPort.h"
 #include "Store.h"
@@ -1272,7 +1273,8 @@ ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
  * the callout.  This is strictly for convenience.
  */
 
-extern int aclMapTOS (acl_tos * head, ACLChecklist * ch);
+extern tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
+extern nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
 
 void
 ClientHttpRequest::doCallouts()
@@ -1335,15 +1337,27 @@ ClientHttpRequest::doCallouts()
         }
     }
 
-    if (!calloutContext->clientside_tos_done) {
-        calloutContext->clientside_tos_done = true;
+    if (!calloutContext->tosToClientDone) {
+        calloutContext->tosToClientDone = true;
         if (getConn() != NULL) {
             ACLFilledChecklist ch(NULL, request, NULL);
             ch.src_addr = request->client_addr;
             ch.my_addr = request->my_addr;
-            int tos = aclMapTOS(Config.accessList.clientside_tos, &ch);
+            tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
             if (tos)
-                comm_set_tos(getConn()->fd, tos);
+                Ip::Qos::setSockTos(getConn()->fd, tos);
+        }
+    }
+
+    if (!calloutContext->nfmarkToClientDone) {
+        calloutContext->nfmarkToClientDone = true;
+        if (getConn() != NULL) {
+            ACLFilledChecklist ch(NULL, request, NULL);
+            ch.src_addr = request->client_addr;
+            ch.my_addr = request->my_addr;
+            nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
+            if (mark)
+                Ip::Qos::setSockNfmark(getConn()->fd, mark);
         }
     }
 
diff --git a/src/comm.cc b/src/comm.cc
index d29db3e7a2..a5a4a7002a 100644
--- a/src/comm.cc
+++ b/src/comm.cc
@@ -51,6 +51,7 @@
 #include "icmp/net_db.h"
 #include "ip/Address.h"
 #include "ip/Intercept.h"
+#include "ip/QosConfig.h"
 #include "ip/tools.h"
 
 #if defined(_SQUID_CYGWIN_)
@@ -72,7 +73,7 @@ typedef enum {
 
 static void commStopHalfClosedMonitor(int fd);
 static IOCB commHalfClosedReader;
-static void comm_init_opened(int new_socket, Ip::Address &addr, unsigned char TOS, const char *note, struct addrinfo *AI);
+static void comm_init_opened(int new_socket, Ip::Address &addr, tos_t tos, nfmark_t nfmark, const char *note, struct addrinfo *AI);
 static int comm_apply_flags(int new_socket, Ip::Address &addr, int flags, struct addrinfo *AI);
 
 
@@ -593,7 +594,7 @@ comm_open(int sock_type,
           int flags,
           const char *note)
 {
-    return comm_openex(sock_type, proto, addr, flags, 0, note);
+    return comm_openex(sock_type, proto, addr, flags, 0, 0, note);
 }
 
 int
@@ -609,7 +610,7 @@ comm_open_listener(int sock_type,
     flags |= COMM_DOBIND;
 
     /* attempt native enabled port. */
-    sock = comm_openex(sock_type, proto, addr, flags, 0, note);
+    sock = comm_openex(sock_type, proto, addr, flags, 0, 0, note);
 
     return sock;
 }
@@ -620,20 +621,6 @@ limitError(int const anErrno)
     return anErrno == ENFILE || anErrno == EMFILE;
 }
 
-int
-comm_set_tos(int fd, int tos)
-{
-#ifdef IP_TOS
-    int x = setsockopt(fd, IPPROTO_IP, IP_TOS, (char *) &tos, sizeof(int));
-    if (x < 0)
-        debugs(50, 1, "comm_set_tos: setsockopt(IP_TOS) on FD " << fd << ": " << xstrerror());
-    return x;
-#else
-    debugs(50, 0, "WARNING: setsockopt(IP_TOS) not supported on this platform");
-    return -1;
-#endif
-}
-
 void
 comm_set_v6only(int fd, int tos)
 {
@@ -674,11 +661,11 @@ comm_openex(int sock_type,
             int proto,
             Ip::Address &addr,
             int flags,
-            unsigned char TOS,
+            tos_t tos,
+            nfmark_t nfmark,
             const char *note)
 {
     int new_socket;
-    int tos = 0;
     struct addrinfo *AI = NULL;
 
     PROF_start(comm_open);
@@ -729,9 +716,12 @@ comm_openex(int sock_type,
     debugs(50, 3, "comm_openex: Opened socket FD " << new_socket << " : family=" << AI->ai_family << ", type=" << AI->ai_socktype << ", protocol=" << AI->ai_protocol );
 
     /* set TOS if needed */
-    if (TOS && comm_set_tos(new_socket, TOS) ) {
-        tos = TOS;
-    }
+    if (tos)
+        Ip::Qos::setSockTos(new_socket, tos);
+
+    /* set netfilter mark if needed */
+    if (nfmark)
+        Ip::Qos::setSockNfmark(new_socket, nfmark);
 
     if ( Ip::EnableIpv6&IPV6_SPECIAL_SPLITSTACK && addr.IsIPv6() )
         comm_set_v6only(new_socket, 1);
@@ -741,7 +731,7 @@ comm_openex(int sock_type,
     if ( Ip::EnableIpv6&IPV6_SPECIAL_V4MAPPING && addr.IsIPv6() )
         comm_set_v6only(new_socket, 0);
 
-    comm_init_opened(new_socket, addr, TOS, note, AI);
+    comm_init_opened(new_socket, addr, tos, nfmark, note, AI);
     new_socket = comm_apply_flags(new_socket, addr, flags, AI);
 
     addr.FreeAddrInfo(AI);
@@ -755,7 +745,8 @@ comm_openex(int sock_type,
 void
 comm_init_opened(int new_socket,
                  Ip::Address &addr,
-                 unsigned char TOS,
+                 tos_t tos,
+                 nfmark_t nfmark,
                  const char *note,
                  struct addrinfo *AI)
 {
@@ -778,7 +769,9 @@ comm_init_opened(int new_socket,
 
     F->local_addr = addr;
 
-    F->tos = TOS;
+    F->tosToServer = tos;
+
+    F->nfmarkToServer = nfmark;
 
     F->sock_family = AI->ai_family;
 }
@@ -857,7 +850,7 @@ comm_import_opened(int fd,
     assert(fd >= 0);
     assert(AI);
 
-    comm_init_opened(fd, addr, 0, note, AI);
+    comm_init_opened(fd, addr, 0, 0, note, AI);
 
     if (!(flags & COMM_NOCLOEXEC))
         fd_table[fd].flags.close_on_exec = 1;
@@ -1088,8 +1081,11 @@ ConnectStateData::commResetFD()
     }
     F->local_addr.FreeAddrInfo(AI);
 
-    if (F->tos)
-        comm_set_tos(fd, F->tos);
+    if (F->tosToServer)
+        Ip::Qos::setSockTos(fd, F->tosToServer);
+
+    if (F->nfmarkToServer)
+        Ip::Qos::setSockNfmark(fd, F->nfmarkToServer);
 
     if ( Ip::EnableIpv6&IPV6_SPECIAL_SPLITSTACK && F->local_addr.IsIPv6() )
         comm_set_v6only(fd, 1);
diff --git a/src/comm.h b/src/comm.h
index 5db4996a40..19e0f0c06b 100644
--- a/src/comm.h
+++ b/src/comm.h
@@ -74,9 +74,8 @@ SQUIDCEXTERN void comm_import_opened(int fd, Ip::Address &addr, int flags, const
  */
 SQUIDCEXTERN int comm_open_listener(int sock_type, int proto, Ip::Address &addr, int flags, const char *note);
 
-SQUIDCEXTERN int comm_openex(int, int, Ip::Address &, int, unsigned char TOS, const char *);
+SQUIDCEXTERN int comm_openex(int, int, Ip::Address &, int, tos_t tos, nfmark_t nfmark, const char *);
 SQUIDCEXTERN u_short comm_local_port(int fd);
-SQUIDCEXTERN int comm_set_tos(int fd, int tos);
 
 SQUIDCEXTERN void commSetSelect(int, unsigned int, PF *, void *, time_t);
 SQUIDCEXTERN void commResetSelect(int);
diff --git a/src/fde.h b/src/fde.h
index a80a78bd60..2fc1d4076f 100644
--- a/src/fde.h
+++ b/src/fde.h
@@ -57,7 +57,10 @@ public:
     u_short remote_port;
 
     Ip::Address local_addr;
-    unsigned char tos;
+    tos_t tosToServer;          /**< The TOS value for packets going towards the server.
+                                        See also tosFromServer. */
+    nfmark_t nfmarkToServer;    /**< The netfilter mark for packets going towards the server.
+                                        See also nfmarkFromServer. */
     int sock_family;
     char ipaddr[MAX_IPSTRLEN];            /* dotted decimal address of peer */
     char desc[FD_DESC_SZ];
@@ -110,9 +113,16 @@ public:
         long handle;
     } win32;
 #endif
-#if USE_ZPH_QOS
-    unsigned char upstreamTOS;			/* see FwdState::dispatch()  */
-#endif
+    tos_t tosFromServer;                /**< Stores the TOS flags of the packets from the remote server.
+                                            See FwdState::dispatch(). Note that this differs to
+                                            tosToServer in that this is the value we *receive* from the,
+                                            connection, whereas tosToServer is the value to set on packets
+                                            *leaving* Squid.  */
+    unsigned int nfmarkFromServer;      /**< Stores the Netfilter mark value of the connection from the remote
+                                            server. See FwdState::dispatch(). Note that this differs to
+                                            nfmarkToServer in that this is the value we *receive* from the,
+                                            connection, whereas nfmarkToServer is the value to set on packets
+                                            *leaving* Squid.   */
 
 private:
     /** Clear the fde class back to NULL equivalent. */
diff --git a/src/forward.cc b/src/forward.cc
index 5cbed32ff5..331ece4cdc 100644
--- a/src/forward.cc
+++ b/src/forward.cc
@@ -42,6 +42,7 @@
 #include "hier_code.h"
 #include "HttpReply.h"
 #include "HttpRequest.h"
+#include "ip/QosConfig.h"
 #include "MemObject.h"
 #include "pconn.h"
 #include "SquidTime.h"
@@ -793,7 +794,6 @@ FwdState::connectStart()
     int ftimeout = Config.Timeout.forward - (squid_curtime - start_t);
 
     Ip::Address outgoing;
-    unsigned short tos;
     Ip::Address client_addr;
     assert(fs);
     assert(server_fd == -1);
@@ -905,9 +905,16 @@ FwdState::connectStart()
         outgoing.SetIPv4();
     }
 
-    tos = getOutgoingTOS(request);
+    tos_t tos = GetTosToServer(request);
 
-    debugs(17, 3, "fwdConnectStart: got outgoing addr " << outgoing << ", tos " << tos);
+#if SO_MARK
+    nfmark_t mark = GetNfmarkToServer(request);
+    debugs(17, 3, "fwdConnectStart: got outgoing addr " << outgoing << ", tos " << int(tos)
+                    << ", netfilter mark " << mark);
+#else
+    nfmark_t mark = 0;
+    debugs(17, 3, "fwdConnectStart: got outgoing addr " << outgoing << ", tos " << int(tos));
+#endif
 
     int commFlags = COMM_NONBLOCKING;
     if (request->flags.spoof_client_ip) {
@@ -916,7 +923,7 @@ FwdState::connectStart()
         // else no tproxy today ...
     }
 
-    fd = comm_openex(SOCK_STREAM, IPPROTO_TCP, outgoing, commFlags, tos, url);
+    fd = comm_openex(SOCK_STREAM, IPPROTO_TCP, outgoing, commFlags, tos, mark, url);
 
     debugs(17, 3, "fwdConnectStart: got TCP FD " << fd);
 
@@ -1005,44 +1012,40 @@ FwdState::dispatch()
 
     netdbPingSite(request->GetHost());
 
-#if USE_ZPH_QOS && defined(_SQUID_LINUX_)
-    /* Bug 2537: This part of ZPH only applies to patched Linux kernels. */
+    /* Update server side TOS and Netfilter mark if using persistent connections. */
+    if (Config.onoff.server_pconns) {
+        if (Ip::Qos::TheConfig.isAclTosActive()) {
+            tos_t tos = GetTosToServer(request);
+            Ip::Qos::setSockTos(server_fd, tos);
+        }
+#if SO_MARK
+        if (Ip::Qos::TheConfig.isAclNfmarkActive()) {
+            nfmark_t mark = GetNfmarkToServer(request);
+            Ip::Qos::setSockNfmark(server_fd, mark);
+        }
+#endif
+    }
 
-    /* Retrieves remote server TOS value, and stores it as part of the
+    /* Retrieves remote server TOS or MARK value, and stores it as part of the
      * original client request FD object. It is later used to forward
-     * remote server's TOS in the response to the client in case of a MISS.
+     * remote server's TOS/MARK in the response to the client in case of a MISS.
      */
-    fde * clientFde = &fd_table[client_fd];
-    if (clientFde) {
-        int tos = 1;
-        int tos_len = sizeof(tos);
-        clientFde->upstreamTOS = 0;
-        if (setsockopt(server_fd,SOL_IP,IP_RECVTOS,&tos,tos_len)==0) {
-            unsigned char buf[512];
-            int len = 512;
-            if (getsockopt(server_fd,SOL_IP,IP_PKTOPTIONS,buf,(socklen_t*)&len) == 0) {
-                /* Parse the PKTOPTIONS structure to locate the TOS data message
-                 * prepared in the kernel by the ZPH incoming TCP TOS preserving
-                 * patch.
-                 */
-                unsigned char * pbuf = buf;
-                while (pbuf-buf < len) {
-                    struct cmsghdr *o = (struct cmsghdr*)pbuf;
-                    if (o->cmsg_len<=0)
-                        break;
-
-                    if (o->cmsg_level == SOL_IP && o->cmsg_type == IP_TOS) {
-                        int *tmp = (int*)CMSG_DATA(o);
-                        clientFde->upstreamTOS = (unsigned char)*tmp;
-                        break;
-                    }
-                    pbuf += CMSG_LEN(o->cmsg_len);
-                }
-            } else {
-                debugs(33, 1, "ZPH: error in getsockopt(IP_PKTOPTIONS) on FD "<<server_fd<<" "<<xstrerror());
-            }
-        } else {
-            debugs(33, 1, "ZPH: error in setsockopt(IP_RECVTOS) on FD "<<server_fd<<" "<<xstrerror());
+    if (Ip::Qos::TheConfig.isHitNfmarkActive()) {
+        fde * clientFde = &fd_table[client_fd];
+        fde * servFde = &fd_table[server_fd];
+        if (clientFde && servFde) {
+            /* Get the netfilter mark for the connection */
+            Ip::Qos::getNfmarkFromServer(server_fd, servFde, clientFde);
+        }
+    }
+
+#if _SQUID_LINUX_
+    /* Bug 2537: The TOS forward part of QOS only applies to patched Linux kernels. */
+    if (Ip::Qos::TheConfig.isHitTosActive()) {
+        fde * clientFde = &fd_table[client_fd];
+        if (clientFde) {
+            /* Get the TOS value for the packet */
+            Ip::Qos::getTosFromServer(server_fd, clientFde);
         }
     }
 #endif
@@ -1376,7 +1379,8 @@ aclMapAddr(acl_address * head, ACLChecklist * ch)
  * DPW 2007-05-19
  * Formerly static, but now used by client_side_request.cc
  */
-int
+/// Checks for a TOS value to apply depending on the ACL
+tos_t
 aclMapTOS(acl_tos * head, ACLChecklist * ch)
 {
     acl_tos *l;
@@ -1389,6 +1393,20 @@ aclMapTOS(acl_tos * head, ACLChecklist * ch)
     return 0;
 }
 
+/// Checks for a netfilter mark value to apply depending on the ACL
+nfmark_t
+aclMapNfmark(acl_nfmark * head, ACLChecklist * ch)
+{
+    acl_nfmark *l;
+
+    for (l = head; l; l = l->next) {
+        if (!l->aclList || ch->matchAclListFast(l->aclList))
+            return l->nfmark;
+    }
+
+    return 0;
+}
+
 Ip::Address
 getOutgoingAddr(HttpRequest * request, struct peer *dst_peer)
 {
@@ -1424,8 +1442,21 @@ getOutgoingAddr(HttpRequest * request, struct peer *dst_peer)
     return aclMapAddr(Config.accessList.outgoing_address, &ch);
 }
 
-unsigned long
-getOutgoingTOS(HttpRequest * request)
+tos_t
+GetTosToServer(HttpRequest * request)
+{
+    ACLFilledChecklist ch(NULL, request, NULL);
+
+    if (request) {
+        ch.src_addr = request->client_addr;
+        ch.my_addr = request->my_addr;
+    }
+
+    return aclMapTOS(Ip::Qos::TheConfig.tosToServer, &ch);
+}
+
+nfmark_t
+GetNfmarkToServer(HttpRequest * request)
 {
     ACLFilledChecklist ch(NULL, request, NULL);
 
@@ -1434,7 +1465,7 @@ getOutgoingTOS(HttpRequest * request)
         ch.my_addr = request->my_addr;
     }
 
-    return aclMapTOS(Config.accessList.outgoing_tos, &ch);
+    return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch);
 }
 
 
diff --git a/src/forward.h b/src/forward.h
index 126e021f6b..5cf645bd41 100644
--- a/src/forward.h
+++ b/src/forward.h
@@ -10,6 +10,19 @@ class HttpRequest;
 #include "hier_code.h"
 #include "ip/Address.h"
 
+/**
+ * Returns the TOS value that we should be setting on the connection
+ * to the server, based on the ACL.
+ */
+tos_t GetTosToServer(HttpRequest * request);
+
+/**
+ * Returns the Netfilter mark value that we should be setting on the
+ * connection to the server, based on the ACL.
+ */
+nfmark_t GetNfmarkToServer(HttpRequest * request);
+
+
 class FwdServer
 {
 public:
diff --git a/src/ip/Makefile.am b/src/ip/Makefile.am
index 439bc7b25c..ce15112d04 100644
--- a/src/ip/Makefile.am
+++ b/src/ip/Makefile.am
@@ -9,8 +9,9 @@ libip_la_SOURCES = \
 	Address.cc \
 	Intercept.h \
 	Intercept.cc \
-	QosConfig.h \
-	QosConfig.cc \
+	Qos.h \
+	Qos.cc \
+	Qos.cci \
 	tools.cc \
 	tools.h
 
diff --git a/src/ip/Qos.cci b/src/ip/Qos.cci
new file mode 100644
index 0000000000..f2a53bc632
--- /dev/null
+++ b/src/ip/Qos.cci
@@ -0,0 +1,75 @@
+/* Inline QOS functions */
+
+int
+Ip::Qos::setSockTos(int fd, tos_t tos)
+{
+#ifdef IP_TOS
+    int x = setsockopt(fd, IPPROTO_IP, IP_TOS, &tos, sizeof(tos_t));
+    if (x < 0)
+        debugs(50, 2, "Ip::Qos::setSockTos: setsockopt(IP_TOS) on FD " << fd << ": " << xstrerror());
+    return x;
+#else
+    debugs(50, DBG_IMPORTANT, "WARNING: setsockopt(IP_TOS) not supported on this platform");
+    return -1;
+#endif
+}
+
+int
+Ip::Qos::setSockNfmark(int fd, nfmark_t mark)
+{
+#if SO_MARK
+    int x = setsockopt(fd, SOL_SOCKET, SO_MARK, &mark, sizeof(nfmark_t));
+    if (x < 0)
+        debugs(50, 2, "setSockNfmark: setsockopt(SO_MARK) on FD " << fd << ": " << xstrerror());
+    return x;
+#else
+    debugs(50, DBG_IMPORTANT, "WARNING: setsockopt(SO_MARK) not supported on this platform");
+    return -1;
+#endif
+}
+
+bool
+Ip::Qos::Config::isHitTosActive() const
+{
+    return (tosLocalHit || tosSiblingHit || tosParentHit || tosMiss || preserveMissTos);
+}
+
+bool
+Ip::Qos::Config::isHitNfmarkActive() const
+{
+    return (markLocalHit || markSiblingHit || markParentHit || markMiss || preserveMissMark);
+}
+
+bool
+Ip::Qos::Config::isAclNfmarkActive() const
+{
+    acl_nfmark * nfmarkAcls [] = { nfmarkToServer, nfmarkToClient };
+ 
+    for (int i=0; i<2; i++) {
+        while (nfmarkAcls[i]) {
+            acl_nfmark *l = nfmarkAcls[i];
+            if (l->nfmark > 0)
+                return true; 
+            nfmarkAcls[i] = l->next;
+        }
+    }
+
+    return false;
+}
+
+bool
+Ip::Qos::Config::isAclTosActive() const
+{
+    acl_tos * tosAcls [] = { tosToServer, tosToClient };
+
+    for (int i=0; i<2; i++) {
+        while (tosAcls[i]) {
+            acl_tos *l = tosAcls[i];
+            if (l->tos > 0)
+                return true;
+            tosAcls[i] = l->next;
+        }
+    }
+
+    return false;
+}
diff --git a/src/ip/QosConfig.cc b/src/ip/QosConfig.cc
index 194d0243cb..948e23aa47 100644
--- a/src/ip/QosConfig.cc
+++ b/src/ip/QosConfig.cc
@@ -1,40 +1,345 @@
+#include "acl/Gadgets.h"
+#include "ConfigParser.h"
+#include "fde.h"
+#include "hier_code.h"
+#include "ip/tools.h"
+#include "Qos.h"
+#include "Parsing.h"
 #include "squid.h"
 
-#if USE_ZPH_QOS
+/* Qos namespace */
 
-#include "QosConfig.h"
+void
+Ip::Qos::getTosFromServer(const int server_fd, fde *clientFde)
+{
+#if USE_QOS_TOS 
+    tos_t tos = 1;
+    int tos_len = sizeof(tos); 
+    clientFde->tosFromServer = 0;
+    if (setsockopt(server_fd,SOL_IP,IP_RECVTOS,&tos,tos_len)==0) {
+        unsigned char buf[512];
+        int len = 512;
+        if (getsockopt(server_fd,SOL_IP,IP_PKTOPTIONS,buf,(socklen_t*)&len) == 0) {
+            /* Parse the PKTOPTIONS structure to locate the TOS data message
+             * prepared in the kernel by the ZPH incoming TCP TOS preserving
+             * patch.
+             */
+            unsigned char * pbuf = buf;
+            while (pbuf-buf < len) {
+                struct cmsghdr *o = (struct cmsghdr*)pbuf;
+                if (o->cmsg_len<=0)
+                    break;
 
-Ip::Qos::QosConfig Ip::Qos::TheConfig;
+                if (o->cmsg_level == SOL_IP && o->cmsg_type == IP_TOS) {
+                    int *tmp = (int*)CMSG_DATA(o);
+                    clientFde->tosFromServer = (tos_t)*tmp;
+                    break;
+                }
+                pbuf += CMSG_LEN(o->cmsg_len);
+            }
+        } else {
+            debugs(33, 1, "QOS: error in getsockopt(IP_PKTOPTIONS) on FD " << server_fd << " " << xstrerror());
+        }
+    } else {
+        debugs(33, 1, "QOS: error in setsockopt(IP_RECVTOS) on FD " << server_fd << " " << xstrerror());
+    }
+#endif
+}
 
-Ip::Qos::QosConfig::QosConfig() :
-        tos_local_hit(0),
-        tos_sibling_hit(0),
-        tos_parent_hit(0),
-        preserve_miss_tos(1),
-        preserve_miss_tos_mask(255)
+void Ip::Qos::getNfmarkFromServer(const int server_fd, const fde *servFde, const fde *clientFde)
 {
-    ;
+#if USE_LIBNETFILTERCONNTRACK
+    /* Allocate a new conntrack */
+    if (struct nf_conntrack *ct = nfct_new()) {
+
+        /* Prepare data needed to find the connection in the conntrack table.
+         * We need the local and remote IP address, and the local and remote
+         * port numbers.
+         */
+
+        Ip::Address serv_fde_local_conn;
+        struct addrinfo *addr = NULL;
+        serv_fde_local_conn.InitAddrInfo(addr);
+        getsockname(server_fd, addr->ai_addr, &(addr->ai_addrlen));
+        serv_fde_local_conn = *addr;
+        serv_fde_local_conn.GetAddrInfo(addr);
+
+        unsigned short serv_fde_local_port = ((struct sockaddr_in*)addr->ai_addr)->sin_port;
+        struct in6_addr serv_fde_local_ip6;
+        struct in_addr serv_fde_local_ip;
+
+        if (Ip::EnableIpv6 && serv_fde_local_conn.IsIPv6()) {
+            serv_fde_local_ip6 = ((struct sockaddr_in6*)addr->ai_addr)->sin6_addr;
+            nfct_set_attr_u8(ct, ATTR_L3PROTO, AF_INET6);
+            struct in6_addr serv_fde_remote_ip6;
+            inet_pton(AF_INET6,servFde->ipaddr,(struct in6_addr*)&serv_fde_remote_ip6);
+            nfct_set_attr(ct, ATTR_IPV6_DST, serv_fde_remote_ip6.s6_addr);
+            nfct_set_attr(ct, ATTR_IPV6_SRC, serv_fde_local_ip6.s6_addr); 
+        } else {
+            serv_fde_local_ip = ((struct sockaddr_in*)addr->ai_addr)->sin_addr;
+            nfct_set_attr_u8(ct, ATTR_L3PROTO, AF_INET);
+            nfct_set_attr_u32(ct, ATTR_IPV4_DST, inet_addr(servFde->ipaddr));
+            nfct_set_attr_u32(ct, ATTR_IPV4_SRC, serv_fde_local_ip.s_addr);  
+        }
+
+        nfct_set_attr_u8(ct, ATTR_L4PROTO, IPPROTO_TCP);
+        nfct_set_attr_u16(ct, ATTR_PORT_DST, htons(servFde->remote_port));
+        nfct_set_attr_u16(ct, ATTR_PORT_SRC, serv_fde_local_port);
+
+        /* Open a handle to the conntrack */
+        if (struct nfct_handle *h = nfct_open(CONNTRACK, 0)) {
+            /* Register the callback. The callback function will record the mark value. */
+            nfct_callback_register(h, NFCT_T_ALL, getNfMarkCallback, (void *)clientFde);  
+            /* Query the conntrack table using the data previously set */
+            int x = nfct_query(h, NFCT_Q_GET, ct);
+            if (x == -1) {
+                debugs(17, 2, "QOS: Failed to retrieve connection mark: (" << x << ") " << strerror(errno)
+                  << " (Destination " << servFde->ipaddr << ":" << servFde->remote_port
+                  << ", source " << serv_fde_local_conn << ")" );
+            }
+             
+            nfct_close(h);
+        } else {
+            debugs(17, 2, "QOS: Failed to open conntrack handle for upstream netfilter mark retrieval.");
+        }
+        serv_fde_local_conn.FreeAddrInfo(addr);
+        nfct_destroy(ct);
+
+    } else {
+        debugs(17, 2, "QOS: Failed to allocate new conntrack for upstream netfilter mark retrieval.");
+    }
+#endif
+}
+
+#if USE_LIBNETFILTERCONNTRACK
+int
+Ip::Qos::getNfMarkCallback(enum nf_conntrack_msg_type type,
+              struct nf_conntrack *ct,
+              void *data)
+{
+        fde *clientFde = (fde *)data;
+        clientFde->nfmarkFromServer = nfct_get_attr_u32(ct, ATTR_MARK);
+        debugs(17, 3, "QOS: Retrieved connection mark value: " << clientFde->nfmarkFromServer);
+
+        return NFCT_CB_CONTINUE;
+}
+#endif
+
+int
+Ip::Qos::doTosLocalMiss(const int fd, const hier_code hierCode)
+{
+    tos_t tos = 0;
+    if (Ip::Qos::TheConfig.tosSiblingHit && hierCode==SIBLING_HIT) {
+        tos = Ip::Qos::TheConfig.tosSiblingHit;
+        debugs(33, 2, "QOS: Sibling Peer hit with hier code=" << hierCode << ", TOS=" << int(tos));
+    } else if (Ip::Qos::TheConfig.tosParentHit && hierCode==PARENT_HIT) {
+        tos = Ip::Qos::TheConfig.tosParentHit;
+        debugs(33, 2, "QOS: Parent Peer hit with hier code=" << hierCode << ", TOS=" << int(tos));
+    } else if (Ip::Qos::TheConfig.tosMiss) {
+        tos = Ip::Qos::TheConfig.tosMiss;
+        debugs(33, 2, "QOS: Cache miss, setting TOS=" << int(tos));
+    } else if (Ip::Qos::TheConfig.preserveMissTos && Ip::Qos::TheConfig.preserveMissTosMask) {
+        tos = fd_table[fd].tosFromServer & Ip::Qos::TheConfig.preserveMissTosMask;
+        debugs(33, 2, "QOS: Preserving TOS on miss, TOS=" << int(tos));
+    }
+    return setSockTos(fd, tos);
+}
+
+int
+Ip::Qos::doNfmarkLocalMiss(const int fd, const hier_code hierCode)
+{
+    nfmark_t mark = 0;
+    if (Ip::Qos::TheConfig.markSiblingHit && hierCode==SIBLING_HIT) {
+        mark = Ip::Qos::TheConfig.markSiblingHit;
+        debugs(33, 2, "QOS: Sibling Peer hit with hier code=" << hierCode << ", Mark=" << mark);
+    } else if (Ip::Qos::TheConfig.markParentHit && hierCode==PARENT_HIT) {
+        mark = Ip::Qos::TheConfig.markParentHit;
+        debugs(33, 2, "QOS: Parent Peer hit with hier code=" << hierCode << ", Mark=" << mark);
+    } else if (Ip::Qos::TheConfig.markMiss) {
+        mark = Ip::Qos::TheConfig.markMiss;
+        debugs(33, 2, "QOS: Cache miss, setting Mark=" << mark);
+    } else if (Ip::Qos::TheConfig.preserveMissMark) {
+        mark = fd_table[fd].nfmarkFromServer & Ip::Qos::TheConfig.preserveMissMarkMask;
+        debugs(33, 2, "QOS: Preserving mark on miss, Mark=" << mark);
+    }
+    return setSockNfmark(fd, mark);
+}
+
+int
+Ip::Qos::doTosLocalHit(const int fd)
+{
+    debugs(33, 2, "QOS: Setting TOS for local hit, TOS=" << int(Ip::Qos::TheConfig.tosLocalHit));
+    return setSockTos(fd, Ip::Qos::TheConfig.tosLocalHit);
+}
+
+int
+Ip::Qos::doNfmarkLocalHit(const int fd)
+{
+    debugs(33, 2, "QOS: Setting netfilter mark for local hit, mark=" << Ip::Qos::TheConfig.markLocalHit);
+    return setSockNfmark(fd, Ip::Qos::TheConfig.markLocalHit);
+}
+
+/* Qos::Config class */
+
+Ip::Qos::Config Ip::Qos::TheConfig;
+
+Ip::Qos::Config::Config()
+{
+    tosLocalHit = 0;
+    tosSiblingHit = 0;
+    tosParentHit = 0;
+    tosMiss = 0;
+    preserveMissTos = false;
+    preserveMissTosMask = 0xFF;
+    markLocalHit = 0;
+    markSiblingHit = 0;
+    markParentHit = 0;
+    markMiss = 0;
+    preserveMissMark = false;
+    preserveMissMarkMask = 0xFFFFFFFF;
 }
 
 void
-Ip::Qos::QosConfig::parseConfigLine()
+Ip::Qos::Config::parseConfigLine()
 {
-    // %i honors 0 and 0x prefixes, which are important for things like umask
     /* parse options ... */
     char *token;
+    /* These are set as appropriate and then used to check whether the initial loop has been done */
+    bool mark = false;
+    bool tos = false;
+    /* Assume preserve is true. We don't set at initialisation as this affects isHitTosActive().
+       We have to do this now, as we may never match the 'tos' parameter below */
+#if !USE_QOS_TOS
+    debugs(3, DBG_CRITICAL, "ERROR: Invalid option 'qos_flows'. QOS features not enabled in this build");
+    self_destruct();
+#endif
+
     while ( (token = strtok(NULL, w_space)) ) {
 
+        // Work out TOS or mark. Default to TOS for backwards compatibility
+        if (!(mark || tos)) {
+            if (strncmp(token, "mark",4) == 0) {
+#if SO_MARK
+                mark = true;
+                // Assume preserve is true. We don't set at initialisation as this affects isHitNfmarkActive()
+#if USE_LIBNETFILTERCONNTRACK
+                preserveMissMark = true;
+# else // USE_LIBNETFILTERCONNTRACK
+                preserveMissMark = false;
+                debugs(3, DBG_IMPORTANT, "WARNING: Squid not compiled with Netfilter conntrack library. "
+                                            << "Netfilter mark preservation not available.");
+#endif // USE_LIBNETFILTERCONNTRACK
+#else // SO_MARK
+                debugs(3, DBG_CRITICAL, "ERROR: Invalid parameter 'mark' in qos_flows option. "
+                                            << "Linux Netfilter marking not available.");
+                self_destruct();
+#endif // SO_MARK
+            } else if (strncmp(token, "tos",3) == 0) {
+                preserveMissTos = true;
+                tos = true;
+            } else {
+                preserveMissTos = true;
+                tos = true;
+            }
+        }
+
         if (strncmp(token, "local-hit=",10) == 0) {
-            sscanf(&token[10], "%i", &tos_local_hit);
+
+            if (mark) {
+                if (!xstrtoui(&token[10], NULL, &markLocalHit, 0, std::numeric_limits<nfmark_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad mark local-hit value " << &token[10]);
+                        self_destruct();
+                }
+            } else {
+                unsigned int v = 0;
+                if (!xstrtoui(&token[10], NULL, &v, 0, std::numeric_limits<tos_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad TOS local-hit value " << &token[10]);
+                        self_destruct();
+                }
+                tosLocalHit = (tos_t)v;
+            }
+
         } else if (strncmp(token, "sibling-hit=",12) == 0) {
-            sscanf(&token[12], "%i", &tos_sibling_hit);
+
+            if (mark) {
+                if (!xstrtoui(&token[12], NULL, &markSiblingHit, 0, std::numeric_limits<nfmark_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad mark sibling-hit value " << &token[12]);
+                        self_destruct();
+                }
+            } else {
+                unsigned int v = 0;
+                if (!xstrtoui(&token[12], NULL, &v, 0, std::numeric_limits<tos_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad TOS sibling-hit value " << &token[12]);
+                        self_destruct();
+                }
+                tosSiblingHit = (tos_t)v;
+            }
+
         } else if (strncmp(token, "parent-hit=",11) == 0) {
-            sscanf(&token[11], "%i", &tos_parent_hit);
+
+            if (mark) {
+                if (!xstrtoui(&token[11], NULL, &markParentHit, 0, std::numeric_limits<nfmark_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad mark parent-hit value " << &token[11]);
+                        self_destruct();
+                }
+            } else {
+                unsigned int v = 0;
+                if (!xstrtoui(&token[11], NULL, &v, 0, std::numeric_limits<tos_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad TOS parent-hit value " << &token[11]);
+                        self_destruct();
+                }
+                tosParentHit = (tos_t)v;
+            }
+
+        } else if (strncmp(token, "miss=",5) == 0) {
+
+            if (mark) {
+                if (!xstrtoui(&token[5], NULL, &markMiss, 0, std::numeric_limits<nfmark_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad mark miss value " << &token[5]);
+                        self_destruct();
+                }
+            } else {
+                unsigned int v = 0;
+                if (!xstrtoui(&token[5], NULL, &v, 0, std::numeric_limits<tos_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad TOS miss value " << &token[5]);
+                        self_destruct();
+                }
+                tosMiss = (tos_t)v;
+            }
+
         } else if (strcmp(token, "disable-preserve-miss") == 0) {
-            preserve_miss_tos = 0;
-            preserve_miss_tos_mask = 0;
-        } else if (preserve_miss_tos && strncmp(token, "miss-mask=",10) == 0) {
-            sscanf(&token[10], "%i", &preserve_miss_tos_mask);
+
+            if (preserveMissTosMask!=0xFFU || preserveMissMarkMask!=0xFFFFFFFFU) {
+                debugs(3, DBG_CRITICAL, "ERROR: miss-mask feature cannot be set with disable-preserve-miss");
+                self_destruct();
+            }
+            if (mark) {
+                preserveMissMark = false;
+                preserveMissMarkMask = 0;
+            } else {
+                preserveMissTos = false;
+                preserveMissTosMask = 0;
+            }
+
+        } else if (strncmp(token, "miss-mask=",10) == 0) {
+
+            if (mark && preserveMissMark) {
+                if (!xstrtoui(&token[10], NULL, &preserveMissMarkMask, 0, std::numeric_limits<nfmark_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad mark miss-mark value " << &token[10]);
+                        self_destruct();
+                }
+            } else if (preserveMissTos) {
+                unsigned int v = 0;
+                if (!xstrtoui(&token[10], NULL, &v, 0, std::numeric_limits<tos_t>::max())) {
+                        debugs(3, DBG_CRITICAL, "ERROR: Bad TOS miss-mark value " << &token[10]);
+                        self_destruct();
+                }
+                preserveMissTosMask = (tos_t)v;
+            } else {
+                debugs(3, DBG_CRITICAL, "ERROR: miss-mask feature cannot be set without miss-preservation enabled");
+                self_destruct();
+            }
+
         }
     }
 }
@@ -43,37 +348,63 @@ Ip::Qos::QosConfig::parseConfigLine()
  * NOTE: Due to the low-level nature of the library these
  * objects are part of the dump function must be self-contained.
  * which means no StoreEntry refrences. Just a basic char* buffer.
- */
+*/
 void
-Ip::Qos::QosConfig::dumpConfigLine(char *entry, const char *name) const
+Ip::Qos::Config::dumpConfigLine(char *entry, const char *name) const
 {
     char *p = entry;
-    snprintf(p, 10, "%s", name); // strlen("qos_flows ");
-    p += strlen(name);
+    if (isHitTosActive()) {
 
-    if (tos_local_hit >0) {
-        snprintf(p, 15, " local-hit=%2x", tos_local_hit);
-        p += 15;
-    }
+        p += snprintf(p, 11, "%s", name); // strlen("qos_flows ");
+        p += snprintf(p, 4, "%s", "tos");
 
-    if (tos_sibling_hit >0) {
-        snprintf(p, 17, " sibling-hit=%2x", tos_sibling_hit);
-        p += 17;
-    }
-    if (tos_parent_hit >0) {
-        snprintf(p, 16, " parent-hit=%2x", tos_parent_hit);
-        p += 16;
-    }
-    if (preserve_miss_tos != 0) {
-        snprintf(p, 22, " disable-preserve-miss");
-        p += 22;
+        if (tosLocalHit > 0) {
+            p += snprintf(p, 16, " local-hit=0x%02X", tosLocalHit);
+        }
+        if (tosSiblingHit > 0) {
+            p += snprintf(p, 18, " sibling-hit=0x%02X", tosSiblingHit);
+        }
+        if (tosParentHit > 0) {
+            p += snprintf(p, 17, " parent-hit=0x%02X", tosParentHit);
+        }
+        if (tosMiss > 0) {
+            p += snprintf(p, 11, " miss=0x%02X", tosMiss);
+        }
+        if (preserveMissTos == 0) {
+            p += snprintf(p, 23, " disable-preserve-miss");
+        }
+        if (preserveMissTos && preserveMissTosMask != 0) {
+            p += snprintf(p, 16, " miss-mask=0x%02X", preserveMissTosMask);
+        }
+        p += snprintf(p, 2, "\n");
     }
-    if (preserve_miss_tos && preserve_miss_tos_mask != 0) {
-        snprintf(p, 15, " miss-mask=%2x", preserve_miss_tos_mask);
-        p += 15;
+    
+    if (isHitNfmarkActive()) {
+        p += snprintf(p, 11, "%s", name); // strlen("qos_flows ");
+        p += snprintf(p, 5, "%s", "mark");
+
+        if (markLocalHit > 0) {
+            p += snprintf(p, 22, " local-hit=0x%02X", markLocalHit);
+        }
+        if (markSiblingHit > 0) {
+            p += snprintf(p, 24, " sibling-hit=0x%02X", markSiblingHit);
+        }
+        if (markParentHit > 0) {
+            p += snprintf(p, 23, " parent-hit=0x%02X", markParentHit);
+        }
+        if (markMiss > 0) {
+            p += snprintf(p, 17, " miss=0x%02X", markMiss);
+        }
+        if (preserveMissMark == false) {
+            p += snprintf(p, 23, " disable-preserve-miss");
+        }
+        if (preserveMissMark && preserveMissMarkMask != 0) {
+            p += snprintf(p, 22, " miss-mask=0x%02X", preserveMissMarkMask);
+        }
+        p += snprintf(p, 2, "\n");
     }
-    snprintf(p, 1, "\n");
-//    p += 1;
 }
 
-#endif /* USE_ZPH_QOS */
+#if !_USE_INLINE_
+#include "Qos.cci"
+#endif
diff --git a/src/ip/QosConfig.h b/src/ip/QosConfig.h
index 6c571edfa1..4011ffef3c 100644
--- a/src/ip/QosConfig.h
+++ b/src/ip/QosConfig.h
@@ -2,33 +2,181 @@
 #define SQUID_QOSCONFIG_H
 
 #include "config.h"
+#include "hier_code.h"
 
-#if USE_ZPH_QOS
+#if HAVE_LIBNETFILTER_CONNTRACK_LIBNETFILTER_CONNTRACK_H
+#include <libnetfilter_conntrack/libnetfilter_conntrack.h>
+#endif
+
+#if HAVE_LIBNETFILTER_CONNTRACK_LIBNETFILTER_CONNTRACK_TCP_H
+#include <libnetfilter_conntrack/libnetfilter_conntrack_tcp.h>
+#endif
+
+#if HAVE_LIMITS
+#include <limits>
+#endif
+
+// Forward-declaration
+class fde;
 
 namespace Ip
 {
 
+/**
+ * QOS namespace contains all the QOS functionality: global functions within
+ * the namespace and the configuration parameters within a config class.
+ */
 namespace Qos
 {
 
-class QosConfig
+    /**
+    * Function to retrieve the TOS value of the inbound packet.
+    * Called by FwdState::dispatch if QOS options are enabled.
+    * @param server_fd Server side descriptor of connection to get TOS for
+    * @param clientFde Pointer to client side fde instance to set tosFromServer in
+    */ 
+    void getTosFromServer(const int server_fd, fde *clientFde);
+    
+    /**
+    * Function to retrieve the netfilter mark value of the connection
+    * to the upstream server. Called by FwdState::dispatch if QOS
+    * options are enabled.
+    * @param server_fd Server side descriptor of connection to get mark for
+    * @param servFde Pointer to server side fde instance to get mark for
+    * @param clientFde Pointer to client side fde instance to set nfmarkFromServer in
+    */
+    void getNfmarkFromServer(const int server_fd, const fde *servFde, const fde *clientFde);
+
+#if USE_LIBNETFILTERCONNTRACK
+    /**          
+    * Callback function to mark connection once it's been found.
+    * This function is called by the libnetfilter_conntrack
+    * libraries, during nfct_query in Ip::Qos::getNfmarkFromServer.
+    * nfct_callback_register is used to register this function.   
+    * @param nf_conntrack_msg_type Type of conntrack message
+    * @param nf_conntrack Pointer to the conntrack structure
+    * @param clientFde Pointer to client side fde instance to set nfmarkFromServer in
+    */
+    int getNfMarkCallback(enum nf_conntrack_msg_type type, struct nf_conntrack *ct, void *clientFde);
+#endif
+
+    /**
+    * Function to work out and then apply to the socket the appropriate
+    * TOS value to set on packets when items have not been retrieved from
+    * local cache. Called by clientReplyContext::sendMoreData if QOS is  
+    * enabled for TOS.
+    * @param fd Descriptor of socket to set the TOS for
+    * @param hierCode Hier code of request
+    */
+    int doTosLocalMiss(const int fd, const hier_code hierCode);
+    
+    /**
+    * Function to work out and then apply to the socket the appropriate
+    * netfilter mark value to set on packets when items have not been  
+    * retrieved from local cache. Called by clientReplyContext::sendMoreData
+    * if QOS is enabled for TOS.
+    * @param fd Descriptor of socket to set the mark for
+    * @param hierCode Hier code of request
+    */
+    int doNfmarkLocalMiss(const int fd, const hier_code hierCode);
+    
+    /**
+    * Function to work out and then apply to the socket the appropriate
+    * TOS value to set on packets when items *have* been retrieved from
+    * local cache. Called by clientReplyContext::doGetMoreData if QOS is
+    * enabled for TOS.
+    * @param fd Descriptor of socket to set the TOS for
+    */
+    int doTosLocalHit(const int fd);
+    
+    /**
+    * Function to work out and then apply to the socket the appropriate
+    * netfilter mark value to set on packets when items *have* been
+    * retrieved from local cache. Called by clientReplyContext::doGetMoreData
+    * if QOS is enabled for TOS.
+    * @param fd Descriptor of socket to set the mark for
+    */
+    int doNfmarkLocalHit(const int fd);
+    
+    /**
+    * Function to set the TOS value of packets. Sets the value on the socket
+    * which then gets copied to the packets.
+    * @param fd Descriptor of socket to set the TOS for
+    */   
+    _SQUID_INLINE_ int setSockTos(int fd, tos_t tos);
+    
+    /**
+    * Function to set the netfilter mark value of packets. Sets the value on the
+    * socket which then gets copied to the packets. Called from Ip::Qos::doNfmarkLocalMiss
+    * @param fd Descriptor of socket to set the mark for
+    */ 
+    _SQUID_INLINE_ int setSockNfmark(int fd, nfmark_t mark);
+
+/**
+ * QOS configuration class. Contains all the parameters for QOS functions as well
+ * as functions to check whether either TOS or MARK QOS is enabled.
+ */
+class Config
 {
 public:
-    int tos_local_hit;
-    int tos_sibling_hit;
-    int tos_parent_hit;
-    int preserve_miss_tos;
-    int preserve_miss_tos_mask;
 
-public:
-    QosConfig();
-    ~QosConfig() {};
+    Config();
+    ~Config() {};
 
     void parseConfigLine();
+
+    /**
+     * Dump all the configuration values
+     *
+     * NOTE: Due to the low-level nature of the library these
+     * objects are part of the dump function must be self-contained.
+     * which means no StoreEntry references. Just a basic char* buffer.
+     */
     void dumpConfigLine(char *entry, const char *name) const;
+
+    /// Whether we should modify TOS flags based on cache hits and misses.
+    _SQUID_INLINE_ bool isHitTosActive() const;
+    
+    /// Whether we should modify netfilter marks based on cache hits and misses.
+    _SQUID_INLINE_ bool isHitNfmarkActive() const;
+
+    /**
+    * Iterates through any outgoing_nfmark or clientside_nfmark configuration parameters
+    * to find out if any Netfilter marking is required.
+    * This function is used on initialisation to define capabilities required (Netfilter
+    * marking requires CAP_NET_ADMIN).
+    */ 
+    _SQUID_INLINE_ bool isAclNfmarkActive() const;
+
+    /**
+    * Iterates through any outgoing_tos or clientside_tos configuration parameters
+    * to find out if packets should be marked with TOS flags.
+    */
+    _SQUID_INLINE_ bool isAclTosActive() const;
+
+    tos_t tosLocalHit;                  ///< TOS value to apply to local cache hits
+    tos_t tosSiblingHit;                ///< TOS value to apply to hits from siblings
+    tos_t tosParentHit;                 ///< TOS value to apply to hits from parent
+    tos_t tosMiss;                      ///< TOS value to apply to cache misses
+    bool preserveMissTos;               ///< Whether to preserve the TOS value of the inbound packet for misses
+    tos_t preserveMissTosMask;          ///< The mask to apply when preserving the TOS of misses
+
+    nfmark_t markLocalHit;              ///< Netfilter mark value to apply to local cache hits
+    nfmark_t markSiblingHit;            ///< Netfilter mark value to apply to hits from siblings
+    nfmark_t markParentHit;             ///< Netfilter mark value to apply to hits from parent
+    nfmark_t markMiss;                  ///< Netfilter mark value to apply to cache misses
+    bool preserveMissMark;              ///< Whether to preserve netfilter mark value of inbound connection
+    nfmark_t preserveMissMarkMask;      ///< The mask to apply when preserving the netfilter mark of misses
+
+    acl_tos *tosToServer;               ///< The TOS that packets to the web server should be marked with, based on ACL
+    acl_tos *tosToClient;               ///< The TOS that packets to the client should be marked with, based on ACL
+    acl_nfmark *nfmarkToServer;         ///< The MARK that packets to the web server should be marked with, based on ACL
+    acl_nfmark *nfmarkToClient;         ///< The MARK that packets to the client should be marked with, based on ACL
+
 };
 
-extern QosConfig TheConfig;
+/// Globally available instance of Qos::Config
+extern Config TheConfig;
 
 /* legacy parser access wrappers */
 #define parse_QosConfig(X)	(X)->parseConfigLine()
@@ -40,7 +188,11 @@ extern QosConfig TheConfig;
 	} while(0);
 
 }; // namespace Qos
+
 }; // namespace Ip
 
-#endif /* USE_ZPH_QOS */
+#if _USE_INLINE_
+#include "Qos.cci"
+#endif
+
 #endif /* SQUID_QOSCONFIG_H */
diff --git a/src/ip/stubQosConfig.cc b/src/ip/stubQosConfig.cc
index 3af90bf1e1..4ae3d2afdb 100644
--- a/src/ip/stubQosConfig.cc
+++ b/src/ip/stubQosConfig.cc
@@ -1,47 +1,80 @@
 #include "squid.h"
 
-#if USE_ZPH_QOS
-
 #include "ip/QosConfig.h"
 #include "Store.h"
 
-Ip::QosConfig::QosConfig() :
-        tos_local_hit(0),
-        tos_sibling_hit(0),
-        tos_parent_hit(0),
-        preserve_miss_tos(1),
-        preserve_miss_tos_mask(255)
+
+void
+Ip::Qos::getTosFromServer(fde *clientFde, const int server_fd)
+{
+#if USE_QOS_TOS
+    fatal ("Not implemented");
+#endif
+}
+
+void Ip::Qos::getNfmarkFromServer(const fde *clientFde, const fde *servFde, const int server_fd)
+{
+#if USE_QOS_NFMARK
+    fatal ("Not implemented");
+#endif
+}
+
+#if USE_QOS_NFMARK
+int
+Ip::Qos::GetNfMarkCallback(enum nf_conntrack_msg_type type,
+              struct nf_conntrack *ct,
+              void *data)
 {
-    ;
+    fatal ("Not implemented");
+}
+#endif
+
+tos_t
+Ip::Qos::doTosLocalMiss(const int fd, const hier_code hierCode) const
+{
+    fatal ("Not implemented");
+}
+
+int
+Ip::Qos::doNfmarkLocalMiss(const int fd, const hier_code hierCode) const
+{
+    fatal ("Not implemented");
+}
+
+int
+Ip::Qos::doTosLocalHit(const int fd) const
+{
+    fatal ("Not implemented");
+}
+ 
+int
+Ip::Qos::doNfmarkLocalHit(const int fd) const
+{
+    fatal ("Not implemented");
+}
+
+Ip::Qos::Config()
+{
+    fatal ("Not implemented");
+}
+
+Ip::Qos::~Config()
+{
+    fatal ("Not implemented");
 }
 
 void
-Ip::QosConfig::parseConfigLine()
-{
-    // %i honors 0 and 0x prefixes, which are important for things like umask
-    /* parse options ... */
-    char *token;
-    while ( (token = strtok(NULL, w_space)) ) {
-
-        if (strncmp(token, "local-hit=",10) == 0) {
-            sscanf(&token[10], "%i", &tos_local_hit);
-        } else if (strncmp(token, "sibling-hit=",12) == 0) {
-            sscanf(&token[12], "%i", &tos_sibling_hit);
-        } else if (strncmp(token, "parent-hit=",11) == 0) {
-            sscanf(&token[11], "%i", &tos_parent_hit);
-        } else if (strcmp(token, "disable-preserve-miss") == 0) {
-            preserve_miss_tos = 0;
-            preserve_miss_tos_mask = 0;
-        } else if (preserve_miss_tos && strncmp(token, "miss-mask=",10) == 0) {
-            sscanf(&token[10], "%i", &preserve_miss_tos_mask);
-        }
-    }
+Ip::Qos::parseConfigLine()
+{
+    fatal ("Not implemented");
 }
 
 void
-Ip::QosConfig::dumpConfigLine(char *entry, const char *name) const
+Ip::Qos::dumpConfigLine(char *entry, const char *name)
 {
-    ; /* Not needed in stub */
+    fatal ("Not implemented");
 }
 
-#endif /* USE_ZPH_QOS */
+#if !_USE_INLINE_
+#include "Qos.cci"
+#endif
diff --git a/src/protos.h b/src/protos.h
index b5bea9e330..2d284779dc 100644
--- a/src/protos.h
+++ b/src/protos.h
@@ -411,7 +411,6 @@ SQUIDCEXTERN void peerDigestNotePeerGone(PeerDigest * pd);
 SQUIDCEXTERN void peerDigestStatsReport(const PeerDigest * pd, StoreEntry * e);
 
 extern Ip::Address getOutgoingAddr(HttpRequest * request, struct peer *dst_peer);
-unsigned long getOutgoingTOS(HttpRequest * request);
 
 SQUIDCEXTERN void urnStart(HttpRequest *, StoreEntry *);
 
diff --git a/src/structs.h b/src/structs.h
index 5385b998ae..106b8e185c 100644
--- a/src/structs.h
+++ b/src/structs.h
@@ -98,7 +98,13 @@ struct acl_address {
 struct acl_tos {
     acl_tos *next;
     ACLList *aclList;
-    int tos;
+    tos_t tos;
+};
+
+struct acl_nfmark {
+    acl_nfmark *next;
+    ACLList *aclList;
+    nfmark_t nfmark;
 };
 
 struct acl_size_t {
@@ -466,8 +472,6 @@ struct SquidConfig {
         acl_access *redirector;
         acl_access *reply;
         acl_address *outgoing_address;
-        acl_tos *outgoing_tos;
-        acl_tos *clientside_tos;
 #if USE_HTCP
 
         acl_access *htcp;
diff --git a/src/tools.cc b/src/tools.cc
index 7558c9f9e2..2ccfb99a13 100644
--- a/src/tools.cc
+++ b/src/tools.cc
@@ -39,6 +39,7 @@
 #include "compat/tempnam.h"
 #include "fde.h"
 #include "ip/Intercept.h"
+#include "ip/QosConfig.h"
 #include "MemBuf.h"
 #include "ProtoPort.h"
 #include "SquidMath.h"
@@ -1326,8 +1327,7 @@ restoreCapabilities(int keep)
         int rc = 0;
         cap_value_t cap_list[10];
         cap_list[ncaps++] = CAP_NET_BIND_SERVICE;
-
-        if (Ip::Interceptor.TransparentActive()) {
+        if (Ip::Interceptor.TransparentActive() || Ip::Qos::TheConfig.isHitNfmarkActive() || Ip::Qos::TheConfig.isAclNfmarkActive()) {
             cap_list[ncaps++] = CAP_NET_ADMIN;
         }
 
diff --git a/src/tunnel.cc b/src/tunnel.cc
index a2bee91152..b0489e0ea4 100644
--- a/src/tunnel.cc
+++ b/src/tunnel.cc
@@ -602,6 +602,9 @@ tunnelConnectDone(int fdnotused, const DnsLookupDetails &dns, comm_err_t status,
     }
 }
 
+extern tos_t GetTosToServer(HttpRequest * request);
+extern nfmark_t GetNfmarkToServer(HttpRequest * request);
+
 void
 tunnelStart(ClientHttpRequest * http, int64_t * size_ptr, int *status_ptr)
 {
@@ -668,7 +671,8 @@ tunnelStart(ClientHttpRequest * http, int64_t * size_ptr, int *status_ptr)
                        IPPROTO_TCP,
                        temp,
                        flags,
-                       getOutgoingTOS(request),
+                       GetTosToServer(request),
+                       GetNfmarkToServer(request),
                        url);
 
     if (sock == COMM_ERROR) {
diff --git a/src/typedefs.h b/src/typedefs.h
index b33594e4c7..9186321b1b 100644
--- a/src/typedefs.h
+++ b/src/typedefs.h
@@ -39,6 +39,9 @@
 typedef int32_t sfileno;
 typedef signed int sdirno;
 
+typedef uint32_t nfmark_t;
+typedef unsigned char tos_t;
+
 typedef struct {
     size_t bytes;
     size_t kb;