From 244da4ad90720f8ca2229c9adc2bbd3ce87a8e7e Mon Sep 17 00:00:00 2001 From: Alexander Gozman Date: Fri, 13 Apr 2018 02:28:57 +0000 Subject: [PATCH] Reworked packet/connection marking (#170) The handling of packet and connection marks was odd: The clientside_mark ACL worked with connection marks, but the directive of the same name supported only packet marks. Also, clients packet MARK (if set) overwrote CONNMARK and, as a result, broke ACL checking. To minimize confusion, connection and packet marks are now separated: * renamed the clientside_mark ACL to client_connection_mark * renamed the clientside_mark directive to mark_client_packet * added a mark_client_connection directive While the first two points just clarify things, the last one introduces a new functionality: It allows to set or change clients CONNMARK. Both clientside_mark ACL and directive are now deprecated. --- src/AclRegs.cc | 1 + src/ClientRequestContext.h | 3 +- src/FwdState.cc | 14 +++-- src/acl/Acl.cc | 3 + src/acl/ConnMark.cc | 44 ++------------ src/acl/ConnMark.h | 7 +-- src/cache_cf.cc | 29 ++++----- src/cf.data.pre | 56 ++++++++++++++--- src/client_side_request.cc | 38 ++++++------ src/comm/Connection.cc | 1 + src/comm/Connection.h | 12 +++- src/comm/TcpAcceptor.cc | 2 +- src/fde.h | 12 ++-- src/ip/Makefile.am | 2 + src/ip/NfMarkConfig.cc | 60 +++++++++++++++++++ src/ip/NfMarkConfig.h | 49 +++++++++++++++ src/ip/QosConfig.cc | 119 ++++++++++++++++++++++++++++--------- src/ip/QosConfig.h | 27 ++++----- 18 files changed, 331 insertions(+), 148 deletions(-) create mode 100644 src/ip/NfMarkConfig.cc create mode 100644 src/ip/NfMarkConfig.h diff --git a/src/AclRegs.cc b/src/AclRegs.cc index c9ca971168..c823e11d73 100644 --- a/src/AclRegs.cc +++ b/src/AclRegs.cc @@ -163,6 +163,7 @@ Acl::Init() #if USE_LIBNETFILTERCONNTRACK RegisterMaker("clientside_mark", [](TypeName name)->ACL* { return new Acl::ConnMark; }); + RegisterMaker("client_connection_mark", [](TypeName name)->ACL* { return new Acl::ConnMark; }); #endif #if USE_OPENSSL diff --git a/src/ClientRequestContext.h b/src/ClientRequestContext.h index 5a39186e21..06ab805d83 100644 --- a/src/ClientRequestContext.h +++ b/src/ClientRequestContext.h @@ -74,8 +74,7 @@ public: bool store_id_done; bool no_cache_done; bool interpreted_req_hdrs; - bool tosToClientDone; - bool nfmarkToClientDone; + bool toClientMarkingDone; #if USE_OPENSSL bool sslBumpCheckDone; #endif diff --git a/src/FwdState.cc b/src/FwdState.cc index 63eb5a87bd..a10272bc1d 100644 --- a/src/FwdState.cc +++ b/src/FwdState.cc @@ -37,6 +37,7 @@ #include "icmp/net_db.h" #include "internal.h" #include "ip/Intercept.h" +#include "ip/NfMarkConfig.h" #include "ip/QosConfig.h" #include "ip/tools.h" #include "MemObject.h" @@ -995,7 +996,7 @@ FwdState::dispatch() if (Comm::IsConnOpen(clientConn) && Comm::IsConnOpen(serverConnection())) { fde * clientFde = &fd_table[clientConn->fd]; // XXX: move the fd_table access into Ip::Qos /* Get the netfilter CONNMARK */ - clientFde->nfmarkFromServer = Ip::Qos::getNfmarkFromConnection(serverConnection(), Ip::Qos::dirOpened); + clientFde->nfConnmarkFromServer = Ip::Qos::getNfConnmark(serverConnection(), Ip::Qos::dirOpened); } } @@ -1278,15 +1279,15 @@ aclMapTOS(acl_tos * head, ACLChecklist * ch) } /// Checks for a netfilter mark value to apply depending on the ACL -nfmark_t -aclMapNfmark(acl_nfmark * head, ACLChecklist * ch) +Ip::NfMarkConfig +aclFindNfMarkConfig(acl_nfmark * head, ACLChecklist * ch) { for (acl_nfmark *l = head; l; l = l->next) { if (!l->aclList || ch->fastCheck(l->aclList).allowed()) - return l->nfmark; + return l->markConfig; } - return 0; + return {}; } void @@ -1352,7 +1353,8 @@ nfmark_t GetNfmarkToServer(HttpRequest * request) { ACLFilledChecklist ch(NULL, request, NULL); - return aclMapNfmark(Ip::Qos::TheConfig.nfmarkToServer, &ch); + const auto mc = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfmarkToServer, &ch); + return mc.mark; } void diff --git a/src/acl/Acl.cc b/src/acl/Acl.cc index a5de02a2e2..309514fb44 100644 --- a/src/acl/Acl.cc +++ b/src/acl/Acl.cc @@ -224,6 +224,9 @@ ACL::ParseAclLine(ConfigParser &parser, ACL ** head) // ACL manager is now a built-in and has a different type. debugs(28, DBG_PARSE_NOTE(DBG_IMPORTANT), "UPGRADE: ACL 'manager' is now a built-in ACL. Remove it from your config file."); return; // ignore the line + } else if (strcmp(theType, "clientside_mark") == 0) { + debugs(28, DBG_IMPORTANT, "UPGRADE: ACL 'clientside_mark' type has been renamed to 'client_connection_mark'."); + theType = "client_connection_mark"; } if ((A = FindByName(aclname)) == NULL) { diff --git a/src/acl/ConnMark.cc b/src/acl/ConnMark.cc index 5a62e6f36a..68e1d3c89f 100644 --- a/src/acl/ConnMark.cc +++ b/src/acl/ConnMark.cc @@ -22,47 +22,15 @@ Acl::ConnMark::empty() const return false; } -static std::ostream & -operator <<(std::ostream &os, const Acl::ConnMark::ConnMarkQuery connmark) -{ - os << asHex(connmark.first); - if (connmark.second != 0xffffffff) { - os << '/' << asHex(connmark.second); - } - return os; -} - -nfmark_t -Acl::ConnMark::getNumber(Parser::Tokenizer &tokenizer, const SBuf &token) const -{ - int64_t number; - if (!tokenizer.int64(number, 0, false)) { - throw TexcHere(ToSBuf("acl ", typeString(), ": invalid value '", tokenizer.buf(), "' in ", token)); - } - - if (number > std::numeric_limits::max()) { - throw TexcHere(ToSBuf("acl ", typeString(), ": number ", number, " in ", token, " is too big")); - } - return static_cast(number); -} - void Acl::ConnMark::parse() { while (const char *t = ConfigParser::strtokFile()) { SBuf token(t); Parser::Tokenizer tokenizer(token); - - const auto mark = getNumber(tokenizer, token); - const auto mask = tokenizer.skip('/') ? getNumber(tokenizer, token) : 0xffffffff; - - if (!tokenizer.atEnd()) { - throw TexcHere(ToSBuf("acl ", typeString(), ": trailing garbage in ", token)); - } - - const ConnMarkQuery connmark(mark, mask); - marks.push_back(connmark); - debugs(28, 7, "added " << connmark); + const auto mc = Ip::NfMarkConfig::Parse(token); + marks.push_back(mc); + debugs(28, 7, "added " << mc); } if (marks.empty()) { @@ -74,10 +42,10 @@ int Acl::ConnMark::match(ACLChecklist *cl) { const auto *checklist = Filled(cl); - const auto connmark = checklist->conn()->clientConnection->nfmark; + const auto connmark = checklist->conn()->clientConnection->nfConnmark; for (const auto &m : marks) { - if ((connmark & m.second) == m.first) { + if (m.matches(connmark)) { debugs(28, 5, "found " << m << " matching " << asHex(connmark)); return 1; } @@ -99,6 +67,6 @@ Acl::ConnMark::dump() const char const * Acl::ConnMark::typeString() const { - return "clientside_mark"; + return "client_connection_mark"; } diff --git a/src/acl/ConnMark.h b/src/acl/ConnMark.h index 29ffbf8fb1..25df71c905 100644 --- a/src/acl/ConnMark.h +++ b/src/acl/ConnMark.h @@ -11,6 +11,7 @@ #include "acl/Acl.h" #include "ip/forward.h" +#include "ip/NfMarkConfig.h" #include "parser/Tokenizer.h" #include @@ -29,12 +30,8 @@ public: virtual SBufList dump() const override; virtual bool empty() const override; - /// a mark/mask pair for matching CONNMARKs - typedef std::pair ConnMarkQuery; - private: - nfmark_t getNumber(Parser::Tokenizer &tokenizer, const SBuf &token) const; - std::vector marks; ///< mark/mask pairs in configured order + std::vector marks; ///< marks/masks in configured order }; } // namespace Acl diff --git a/src/cache_cf.cc b/src/cache_cf.cc index 5e17f04441..ac08e6b89f 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -36,6 +36,7 @@ #include "icmp/IcmpConfig.h" #include "ident/Config.h" #include "ip/Intercept.h" +#include "ip/NfMarkConfig.h" #include "ip/QosConfig.h" #include "ip/tools.h" #include "ipc/Kids.h" @@ -55,6 +56,7 @@ #include "RefreshPattern.h" #include "rfc1738.h" #include "sbuf/List.h" +#include "sbuf/Stream.h" #include "SquidConfig.h" #include "SquidString.h" #include "ssl/ProxyCerts.h" @@ -1542,10 +1544,7 @@ static void dump_acl_nfmark(StoreEntry * entry, const char *name, acl_nfmark * head) { for (acl_nfmark *l = head; l; l = l->next) { - if (l->nfmark > 0) - storeAppendPrintf(entry, "%s 0x%02X", name, l->nfmark); - else - storeAppendPrintf(entry, "%s none", name); + storeAppendPrintf(entry, "%s %s", name, ToSBuf(l->markConfig).c_str()); dump_acl_list(entry, l->aclList); @@ -1556,24 +1555,18 @@ dump_acl_nfmark(StoreEntry * entry, const char *name, acl_nfmark * head) static void parse_acl_nfmark(acl_nfmark ** head) { - nfmark_t mark; - char *token = ConfigParser::NextToken(); + SBuf token(ConfigParser::NextToken()); + const auto mc = Ip::NfMarkConfig::Parse(token); - if (!token) { - self_destruct(); - return; - } - - if (!xstrtoui(token, NULL, &mark, 0, std::numeric_limits::max())) { - self_destruct(); - return; - } + // Packet marking directives should not allow to use masks. + const auto pkt_dirs = {"mark_client_packet", "clientside_mark", "tcp_outgoing_mark"}; + if (mc.hasMask() && std::find(pkt_dirs.begin(), pkt_dirs.end(), cfg_directive) != pkt_dirs.end()) + throw TexcHere(ToSBuf("'", cfg_directive, "' does not support masked marks")); acl_nfmark *l = new acl_nfmark; + l->markConfig = mc; - l->nfmark = mark; - - aclParseAclList(LegacyParser, &l->aclList, token); + aclParseAclList(LegacyParser, &l->aclList, token.c_str()); acl_nfmark **tail = head; /* sane name below */ while (*tail) diff --git a/src/cf.data.pre b/src/cf.data.pre index 9fb2229203..9a79a705f2 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1044,6 +1044,10 @@ DOC_START # encoded directly in the IPv6 address or not available. acl aclname clientside_mark mark[/mask] ... + # matches CONNMARK of an accepted connection [fast] + # DEPRECATED. Use the 'client_connection_mark' instead. + + acl aclname client_connection_mark mark[/mask] ... # matches CONNMARK of an accepted connection [fast] # # mark and mask are unsigned integers (hex, octal, or decimal). @@ -2561,24 +2565,24 @@ DOC_START Only fast ACLs are supported. DOC_END -NAME: clientside_mark +NAME: mark_client_packet clientside_mark TYPE: acl_nfmark IFDEF: SO_MARK&&USE_LIBCAP DEFAULT: none LOC: Ip::Qos::TheConfig.nfmarkToClient DOC_START - Allows you to apply a Netfilter mark value to packets being transmitted + Allows you to apply a Netfilter MARK value to packets being transmitted on the client-side, based on an ACL. - clientside_mark mark-value [!]aclname ... + mark_client_packet mark-value [!]aclname ... - Example where normal_service_net uses the mark value 0x00 + Example where normal_service_net uses the MARK value 0x00 and good_service_net uses 0x20 acl normal_service_net src 10.0.0.0/24 acl good_service_net src 10.0.1.0/24 - clientside_mark 0x00 normal_service_net - clientside_mark 0x20 good_service_net + mark_client_packet 0x00 normal_service_net + mark_client_packet 0x20 good_service_net Note: This feature is incompatible with qos_flows. Any mark values set here will be overwritten by mark values in qos_flows. @@ -2587,6 +2591,42 @@ DOC_START See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. DOC_END +NAME: mark_client_connection +TYPE: acl_nfmark +IFDEF: SO_MARK&&USE_LIBCAP +DEFAULT: none +LOC: Ip::Qos::TheConfig.nfConnmarkToClient +DOC_START + Allows you to apply a Netfilter CONNMARK value to a connection + on the client-side, based on an ACL. + + mark_client_connection mark-value[/mask] [!]aclname ... + + The mark-value and mask are unsigned integers (hex, octal, or decimal). + The mask may be used to preserve marking previously set by other agents + (e.g., iptables). + + A matching rule replaces the CONNMARK value. If a mask is also + specified, then the masked bits of the original value are zeroed, and + the configured mark-value is ORed with that adjusted value. + For example, applying a mark-value 0xAB/0xF to 0x5F CONNMARK, results + in a 0xFB marking (rather than a 0xAB or 0x5B). + + This directive semantics is similar to iptables --set-mark rather than + --set-xmark functionality. + + The directive does not interfere with qos_flows (which uses packet MARKs, + not CONNMARKs). + + Example where squid marks intercepted FTP connections: + + acl proto_ftp proto FTP + mark_client_connection 0x200/0xff00 proto_ftp + + This clause only supports fast acl types. + See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. +DOC_END + NAME: qos_flows TYPE: QosConfig IFDEF: USE_QOS_TOS @@ -4420,7 +4460,7 @@ DOC_START >la Local IP address the client connected to >lp Local port number the client connected to >qos Client connection TOS/DSCP value set by Squid - >nfmark Client connection netfilter mark set by Squid + >nfmark Client connection netfilter packet MARK set by Squid la Local listening IP address the client connection was connected to. lp Local listening port number the client connection was connected to. @@ -4431,7 +4471,7 @@ DOC_START error - if (!calloutContext->tosToClientDone) { - calloutContext->tosToClientDone = true; - if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) { - ACLFilledChecklist ch(NULL, request, NULL); - ch.src_addr = request->client_addr; - ch.my_addr = request->my_addr; + // Set appropriate MARKs and CONNMARKs if needed. + if (getConn() && Comm::IsConnOpen(getConn()->clientConnection)) { + ACLFilledChecklist ch(nullptr, request, nullptr); + ch.src_addr = request->client_addr; + ch.my_addr = request->my_addr; + + if (!calloutContext->toClientMarkingDone) { + calloutContext->toClientMarkingDone = true; tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch); if (tos) Ip::Qos::setSockTos(getConn()->clientConnection, tos); - } - } - if (!calloutContext->nfmarkToClientDone) { - calloutContext->nfmarkToClientDone = true; - if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) { - ACLFilledChecklist ch(NULL, request, NULL); - ch.src_addr = request->client_addr; - ch.my_addr = request->my_addr; - nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch); - if (mark) - Ip::Qos::setSockNfmark(getConn()->clientConnection, mark); + const auto packetMark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfmarkToClient, &ch); + if (!packetMark.isEmpty()) + Ip::Qos::setSockNfmark(getConn()->clientConnection, packetMark.mark); + + const auto connmark = aclFindNfMarkConfig(Ip::Qos::TheConfig.nfConnmarkToClient, &ch); + if (!connmark.isEmpty()) + Ip::Qos::setNfConnmark(getConn()->clientConnection, Ip::Qos::dirAccepted, connmark); } } diff --git a/src/comm/Connection.cc b/src/comm/Connection.cc index 4401dab5bd..44c183b527 100644 --- a/src/comm/Connection.cc +++ b/src/comm/Connection.cc @@ -62,6 +62,7 @@ Comm::Connection::copyDetails() const c->peerType = peerType; c->tos = tos; c->nfmark = nfmark; + c->nfConnmark = nfConnmark; c->flags = flags; c->startTime_ = startTime_; diff --git a/src/comm/Connection.h b/src/comm/Connection.h index 732576a402..a5aca487dc 100644 --- a/src/comm/Connection.h +++ b/src/comm/Connection.h @@ -146,9 +146,19 @@ public: /** Quality of Service TOS values currently sent on this connection */ tos_t tos; - /** Netfilter MARK values currently sent on this connection */ + /** Netfilter MARK values currently sent on this connection + * In case of FTP, the MARK will be sent on data connections as well. + */ nfmark_t nfmark; + /** Netfilter CONNMARK value previously retrieved from this connection + * In case of FTP, the CONNMARK will NOT be applied to data connections, for one main reason: + * the CONNMARK could be set by a third party like iptables and overwriting it in squid may + * cause side effects and break CONNMARK-based policy. In other words, data connection is + * related to control connection, but it's not the same. + */ + nfmark_t nfConnmark = 0; + /** COMM flags set on this connection */ int flags; diff --git a/src/comm/TcpAcceptor.cc b/src/comm/TcpAcceptor.cc index bfef5cb153..fcd99ddbb6 100644 --- a/src/comm/TcpAcceptor.cc +++ b/src/comm/TcpAcceptor.cc @@ -302,7 +302,7 @@ Comm::TcpAcceptor::acceptOne() return; } - newConnDetails->nfmark = Ip::Qos::getNfmarkFromConnection(newConnDetails, Ip::Qos::dirAccepted); + newConnDetails->nfConnmark = Ip::Qos::getNfConnmark(newConnDetails, Ip::Qos::dirAccepted); debugs(5, 5, HERE << "Listener: " << conn << " accepted new connection " << newConnDetails << diff --git a/src/fde.h b/src/fde.h index 3e7073027a..0bace15f9c 100644 --- a/src/fde.h +++ b/src/fde.h @@ -87,7 +87,7 @@ public: tos_t tosToServer = '\0'; /**< The TOS value for packets going towards the server. See also tosFromServer. */ nfmark_t nfmarkToServer = 0; /**< The netfilter mark for packets going towards the server. - See also nfmarkFromServer. */ + See also nfConnmarkFromServer. */ int sock_family = 0; char ipaddr[MAX_IPSTRLEN]; /* dotted decimal address of peer */ char desc[FD_DESC_SZ]; @@ -147,11 +147,11 @@ public: tosToServer in that this is the value we *receive* from the, connection, whereas tosToServer is the value to set on packets *leaving* Squid. */ - unsigned int nfmarkFromServer = 0; /**< Stores the Netfilter mark value of the connection from the remote - server. See FwdState::dispatch(). Note that this differs to - nfmarkToServer in that this is the value we *receive* from the, - connection, whereas nfmarkToServer is the value to set on packets - *leaving* Squid. */ + unsigned int nfConnmarkFromServer = 0; /**< Stores the Netfilter mark value of the connection from the remote + server. See FwdState::dispatch(). Note that this differs to + nfmarkToServer in that this is the value we *receive* from the, + connection, whereas nfmarkToServer is the value to set on packets + *leaving* Squid. */ }; #define fd_table fde::Table diff --git a/src/ip/Makefile.am b/src/ip/Makefile.am index 3fbf36a07b..939cd220fe 100644 --- a/src/ip/Makefile.am +++ b/src/ip/Makefile.am @@ -16,6 +16,8 @@ libip_la_SOURCES = \ Address.cc \ Intercept.h \ Intercept.cc \ + NfMarkConfig.h \ + NfMarkConfig.cc \ QosConfig.h \ QosConfig.cc \ tools.cc \ diff --git a/src/ip/NfMarkConfig.cc b/src/ip/NfMarkConfig.cc new file mode 100644 index 0000000000..5c7bdf68f2 --- /dev/null +++ b/src/ip/NfMarkConfig.cc @@ -0,0 +1,60 @@ +/* + * Copyright (C) 1996-2018 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" + +#include "ConfigParser.h" +#include "ip/NfMarkConfig.h" +#include "parser/Tokenizer.h" +#include "sbuf/Stream.h" + +#include + +static nfmark_t +getNfmark(Parser::Tokenizer &tokenizer, const SBuf &token) +{ + int64_t number; + if (!tokenizer.int64(number, 0, false)) + throw TexcHere(ToSBuf("NfMarkConfig: invalid value '", tokenizer.buf(), "' in '", token, "'")); + + if (number > std::numeric_limits::max()) + throw TexcHere(ToSBuf("NfMarkConfig: number, ", number, "in '", token, "' is too big")); + + return static_cast(number); +} + +Ip::NfMarkConfig +Ip::NfMarkConfig::Parse(const SBuf &token) +{ + Parser::Tokenizer tokenizer(token); + + const nfmark_t mark = getNfmark(tokenizer, token); + const nfmark_t mask = tokenizer.skip('/') ? getNfmark(tokenizer, token) : 0xffffffff; + + if (!tokenizer.atEnd()) + throw TexcHere(ToSBuf("NfMarkConfig: trailing garbage in '", token, "'")); + + return Ip::NfMarkConfig(mark, mask); +} + +nfmark_t +Ip::NfMarkConfig::applyToMark(nfmark_t m) const +{ + return (m & ~mask) | mark; +} + +std::ostream & +operator <<(std::ostream &os, const Ip::NfMarkConfig c) +{ + os << asHex(c.mark); + + if (c.mask != 0xffffffff) + os << '/' << asHex(c.mask); + + return os; +} diff --git a/src/ip/NfMarkConfig.h b/src/ip/NfMarkConfig.h new file mode 100644 index 0000000000..83d7d1c55b --- /dev/null +++ b/src/ip/NfMarkConfig.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 1996-2018 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#ifndef SQUID_NFMARKCONFIG_H +#define SQUID_NFMARKCONFIG_H + +#include "ip/forward.h" + +class SBuf; + +namespace Ip +{ + +/// a netfilter mark/mask pair +class NfMarkConfig +{ +public: + /// creates an empty object + NfMarkConfig() {} + /// creates an object with specified mark and mask + NfMarkConfig(nfmark_t mark_val, nfmark_t mask_val): mark(mark_val), mask(mask_val) {} + + /// parses a token and returns an object, expects a "mark[/mask]" format + static NfMarkConfig Parse(const SBuf &token); + /// whether the 'm' matches the configured mark/mask + bool matches(const nfmark_t m) const { return (m & mask) == mark; } + /// whether the netfilter mark is unset + bool isEmpty() const { return mark == 0; } + /// whether the mask is set + bool hasMask() const { return mask != 0xffffffff; } + /// Applies configured mark/mask to previously set mark (m). + /// m is ANDed with the negated mask and then ORed with the configured mark. + /// \returns new mark. This is similar to what iptables --set-mark does. + nfmark_t applyToMark(nfmark_t m) const; + + nfmark_t mark = 0; + nfmark_t mask = 0xffffffff; +}; + +} // namespace Ip + +std::ostream &operator <<(std::ostream &os, const Ip::NfMarkConfig connmark); + +#endif // SQUID_NFMARKCONFIG_H diff --git a/src/ip/QosConfig.cc b/src/ip/QosConfig.cc index b0d1c07f44..4f9eda33eb 100644 --- a/src/ip/QosConfig.cc +++ b/src/ip/QosConfig.cc @@ -77,20 +77,37 @@ Ip::Qos::getTosFromServer(const Comm::ConnectionPointer &server, fde *clientFde) #endif } -nfmark_t -Ip::Qos::getNfmarkFromConnection(const Comm::ConnectionPointer &conn, const Ip::Qos::ConnectionDirection connDir) -{ - nfmark_t mark = 0; #if USE_LIBNETFILTERCONNTRACK - /* Allocate a new conntrack */ - if (struct nf_conntrack *ct = nfct_new()) { - /* Prepare data needed to find the connection in the conntrack table. - * We need the local and remote IP address, and the local and remote - * port numbers. - */ - const auto src = (connDir == Ip::Qos::dirAccepted) ? conn->remote : conn->local; - const auto dst = (connDir == Ip::Qos::dirAccepted) ? conn->local : conn->remote; +/** +* Callback function to mark connection once it's been found. +* This function is called by the libnetfilter_conntrack +* libraries, during nfct_query in Ip::Qos::getNfConnmark. +* nfct_callback_register is used to register this function. +* @param nf_conntrack_msg_type Type of conntrack message +* @param nf_conntrack Pointer to the conntrack structure +* @param mark Pointer to nfmark_t mark +*/ +static int +getNfmarkCallback(enum nf_conntrack_msg_type, struct nf_conntrack *ct, void *connmark) +{ + auto *mark = static_cast(connmark); + *mark = nfct_get_attr_u32(ct, ATTR_MARK); + debugs(17, 3, asHex(*mark)); + return NFCT_CB_CONTINUE; +} +/** +* Prepares a conntrack query for specified source and destination. +* This can be used for querying or modifying attributes. +*/ +static nf_conntrack * +prepareConntrackQuery(const Ip::Address &src, const Ip::Address &dst) +{ + /* Allocate a new conntrack */ + if (auto ct = nfct_new()) { + // Prepare data needed to find the connection in the conntrack table. + // We need the local and remote IP address, and the local and remote + // port numbers. if (Ip::EnableIpv6 && src.isIPv6()) { nfct_set_attr_u8(ct, ATTR_L3PROTO, AF_INET6); struct in6_addr conn_fde_dst_ip6; @@ -113,11 +130,27 @@ Ip::Qos::getNfmarkFromConnection(const Comm::ConnectionPointer &conn, const Ip:: nfct_set_attr_u16(ct, ATTR_ORIG_PORT_DST, htons(dst.port())); nfct_set_attr_u16(ct, ATTR_ORIG_PORT_SRC, htons(src.port())); - /* Open a handle to the conntrack */ + return ct; + } + + return nullptr; +} +#endif + +nfmark_t +Ip::Qos::getNfConnmark(const Comm::ConnectionPointer &conn, const Ip::Qos::ConnectionDirection connDir) +{ + nfmark_t mark = 0; +#if USE_LIBNETFILTERCONNTRACK + const auto src = (connDir == Ip::Qos::dirAccepted) ? conn->remote : conn->local; + const auto dst = (connDir == Ip::Qos::dirAccepted) ? conn->local : conn->remote; + + if (const auto ct = prepareConntrackQuery(src, dst)) { + // Open a handle to the conntrack if (struct nfct_handle *h = nfct_open(CONNTRACK, 0)) { - /* Register the callback. The callback function will record the mark value. */ + // Register the callback. The callback function will record the mark value. nfct_callback_register(h, NFCT_T_ALL, getNfmarkCallback, static_cast(&mark)); - /* Query the conntrack table using the data previously set */ + // Query the conntrack table using the data previously set int x = nfct_query(h, NFCT_Q_GET, ct); if (x == -1) { const int xerrno = errno; @@ -126,28 +159,56 @@ Ip::Qos::getNfmarkFromConnection(const Comm::ConnectionPointer &conn, const Ip:: } nfct_close(h); } else { - debugs(17, 2, "QOS: Failed to open conntrack handle for netfilter mark retrieval."); + debugs(17, 2, "QOS: Failed to open conntrack handle for netfilter CONNMARK retrieval."); } nfct_destroy(ct); } else { - debugs(17, 2, "QOS: Failed to allocate new conntrack for netfilter mark retrieval."); + debugs(17, 2, "QOS: Failed to allocate new conntrack for netfilter CONNMARK retrieval."); } #endif return mark; } -#if USE_LIBNETFILTERCONNTRACK -int -Ip::Qos::getNfmarkCallback(enum nf_conntrack_msg_type, - struct nf_conntrack *ct, - void *connmark) +bool +Ip::Qos::setNfConnmark(Comm::ConnectionPointer &conn, const Ip::Qos::ConnectionDirection connDir, const Ip::NfMarkConfig &cm) { - auto *mark = static_cast(connmark); - *mark = nfct_get_attr_u32(ct, ATTR_MARK); - debugs(17, 3, asHex(*mark)); - return NFCT_CB_CONTINUE; -} + bool ret = false; + +#if USE_LIBNETFILTERCONNTRACK + const auto src = (connDir == Ip::Qos::dirAccepted) ? conn->remote : conn->local; + const auto dst = (connDir == Ip::Qos::dirAccepted) ? conn->local : conn->remote; + + const nfmark_t newMark = cm.applyToMark(conn->nfConnmark); + + // No need to do anything if a CONNMARK has not changed. + if (newMark == conn->nfConnmark) + return true; + + if (const auto ct = prepareConntrackQuery(src, dst)) { + // Open a handle to the conntrack + if (struct nfct_handle *h = nfct_open(CONNTRACK, 0)) { + nfct_set_attr_u32(ct, ATTR_MARK, newMark); + // Update the conntrack table using the new mark. We do not need a callback here. + const int queryResult = nfct_query(h, NFCT_Q_UPDATE, ct); + if (queryResult == 0) { + conn->nfConnmark = newMark; + ret = true; + } else { + const int xerrno = errno; + debugs(17, 2, "QOS: Failed to modify connection mark: (" << queryResult << ") " << xstrerr(xerrno) + << " (Destination " << dst << ", source " << src << ")" ); + } + nfct_close(h); + } else { + debugs(17, 2, "QOS: Failed to open conntrack handle for netfilter CONNMARK modification."); + } + nfct_destroy(ct); + } else { + debugs(17, 2, "QOS: Failed to allocate new conntrack for netfilter CONNMARK modification."); + } #endif + return ret; +} int Ip::Qos::doTosLocalMiss(const Comm::ConnectionPointer &conn, const hier_code hierCode) @@ -181,7 +242,7 @@ Ip::Qos::doNfmarkLocalMiss(const Comm::ConnectionPointer &conn, const hier_code mark = Ip::Qos::TheConfig.markParentHit; debugs(33, 2, "QOS: Parent Peer hit with hier code=" << hierCode << ", Mark=" << mark); } else if (Ip::Qos::TheConfig.preserveMissMark) { - mark = fd_table[conn->fd].nfmarkFromServer & Ip::Qos::TheConfig.preserveMissMarkMask; + mark = fd_table[conn->fd].nfConnmarkFromServer & Ip::Qos::TheConfig.preserveMissMarkMask; mark = (mark & ~Ip::Qos::TheConfig.markMissMask) | (Ip::Qos::TheConfig.markMiss & Ip::Qos::TheConfig.markMissMask); debugs(33, 2, "QOS: Preserving mark on miss, Mark=" << mark); } else if (Ip::Qos::TheConfig.markMiss) { @@ -537,7 +598,7 @@ Ip::Qos::Config::isAclNfmarkActive() const for (int i=0; i<2; ++i) { while (nfmarkAcls[i]) { acl_nfmark *l = nfmarkAcls[i]; - if (l->nfmark > 0) + if (!l->markConfig.isEmpty()) return true; nfmarkAcls[i] = l->next; } diff --git a/src/ip/QosConfig.h b/src/ip/QosConfig.h index cab364147b..285513a0ad 100644 --- a/src/ip/QosConfig.h +++ b/src/ip/QosConfig.h @@ -12,6 +12,7 @@ #include "acl/forward.h" #include "hier_code.h" #include "ip/forward.h" +#include "ip/NfMarkConfig.h" #if HAVE_LIBNETFILTER_CONNTRACK_LIBNETFILTER_CONNTRACK_H #include @@ -43,12 +44,12 @@ class acl_nfmark CBDATA_CLASS(acl_nfmark); public: - acl_nfmark() : next(NULL), aclList(NULL), nfmark(0) {} + acl_nfmark() : next(NULL), aclList(NULL) {} ~acl_nfmark(); acl_nfmark *next; ACLList *aclList; - nfmark_t nfmark; + Ip::NfMarkConfig markConfig; }; namespace Ip @@ -77,27 +78,24 @@ enum ConnectionDirection { void getTosFromServer(const Comm::ConnectionPointer &server, fde *clientFde); /** -* Function to retrieve the netfilter mark value of the connection. +* Function to retrieve the netfilter CONNMARK value of the connection. * Called by FwdState::dispatch if QOS options are enabled or by * Comm::TcpAcceptor::acceptOne * * @param conn Pointer to connection to get mark for * @param connDir Specifies connection type (incoming or outgoing) */ -nfmark_t getNfmarkFromConnection(const Comm::ConnectionPointer &conn, const ConnectionDirection connDir); +nfmark_t getNfConnmark(const Comm::ConnectionPointer &conn, const ConnectionDirection connDir); -#if USE_LIBNETFILTERCONNTRACK /** -* Callback function to mark connection once it's been found. -* This function is called by the libnetfilter_conntrack -* libraries, during nfct_query in Ip::Qos::getNfmarkFromServer. -* nfct_callback_register is used to register this function. -* @param nf_conntrack_msg_type Type of conntrack message -* @param nf_conntrack Pointer to the conntrack structure -* @param mark Pointer to nfmark_t mark +* Function to set the netfilter CONNMARK value on the connection. +* Called by ClientHttpRequest::doCallouts. +* +* @param conn Pointer to connection to set mark on +* @param connDir Specifies connection type (incoming or outgoing) +* @cm Netfilter mark configuration (mark and mask) */ -int getNfmarkCallback(enum nf_conntrack_msg_type type, struct nf_conntrack *ct, void *mark); -#endif +bool setNfConnmark(Comm::ConnectionPointer &conn, const ConnectionDirection connDir, const NfMarkConfig &cm); /** * Function to work out and then apply to the socket the appropriate @@ -233,6 +231,7 @@ public: acl_tos *tosToClient; ///< The TOS that packets to the client should be marked with, based on ACL acl_nfmark *nfmarkToServer; ///< The MARK that packets to the web server should be marked with, based on ACL acl_nfmark *nfmarkToClient; ///< The MARK that packets to the client should be marked with, based on ACL + acl_nfmark *nfConnmarkToClient = nullptr; ///< The CONNMARK that the client connection should be marked with, based on ACL }; -- 2.39.2