From 88df846be1405a88271bb2d29bd33708438ddac8 Mon Sep 17 00:00:00 2001 From: Christos Tsantilas Date: Wed, 13 Jan 2016 12:10:20 +0200 Subject: [PATCH] Add connections_encrypted ACL The new connections_encrypted ACL matches transactions where all HTTP messages were received over TLS transport connections, including messages received from ICAP servers. Some ICAP/eCAP services receive data from unencrypted sources. Some ICAP/eCAP services are "secure". By default we assume that all eCAP services and all ICAP services on TLS transport connections are "secure" unless the user uses the "connection_encryption" option in service configuration line. This is a Measurement Factory project. --- src/AclRegs.cc | 3 +++ src/HttpMsg.cc | 3 ++- src/HttpMsg.h | 22 ++++++++++++++++ src/HttpReply.cc | 1 + src/HttpRequest.cc | 2 ++ src/acl/Makefile.am | 2 ++ src/adaptation/Config.cc | 17 ++++++++++-- src/adaptation/ServiceConfig.cc | 4 +++ src/adaptation/ServiceConfig.h | 2 ++ src/adaptation/ecap/ServiceRep.cc | 2 ++ src/adaptation/ecap/XactionRep.cc | 6 +++++ src/adaptation/ecap/XactionRep.h | 1 + src/adaptation/icap/ModXact.cc | 16 +++++++++++ src/adaptation/icap/ModXact.h | 1 + src/adaptation/icap/ServiceRep.cc | 6 ++++- src/cf.data.pre | 44 +++++++++++++++++++++++++++++++ src/client_side.cc | 2 ++ src/clients/FtpGateway.cc | 1 + src/clients/FtpRelay.cc | 3 +++ src/gopher.cc | 7 ++++- src/http.cc | 2 ++ src/whois.cc | 1 + 22 files changed, 143 insertions(+), 5 deletions(-) diff --git a/src/AclRegs.cc b/src/AclRegs.cc index e4ad9c0dc5..583acfa3c9 100644 --- a/src/AclRegs.cc +++ b/src/AclRegs.cc @@ -30,6 +30,7 @@ #include "acl/Asn.h" #include "acl/Browser.h" #include "acl/Checklist.h" +#include "acl/ConnectionsEncrypted.h" #include "acl/Data.h" #include "acl/DestinationAsn.h" #include "acl/DestinationDomain.h" @@ -231,3 +232,5 @@ ACLStrategised ACLAdaptationService::RegistryEntry_(new ACLAdaptat ACL::Prototype ACLSquidError::RegistryProtoype(&ACLSquidError::RegistryEntry_, "squid_error"); ACLStrategised ACLSquidError::RegistryEntry_(new ACLSquidErrorData, ACLSquidErrorStrategy::Instance(), "squid_error"); +ACL::Prototype Acl::ConnectionsEncrypted::RegistryProtoype(&Acl::ConnectionsEncrypted::RegistryEntry_, "connections_encrypted"); +Acl::ConnectionsEncrypted Acl::ConnectionsEncrypted::RegistryEntry_("connections_encrypted"); diff --git a/src/HttpMsg.cc b/src/HttpMsg.cc index 5ba16e9f80..fea8bb9ebd 100644 --- a/src/HttpMsg.cc +++ b/src/HttpMsg.cc @@ -24,7 +24,8 @@ HttpMsg::HttpMsg(http_hdr_owner_type owner): cache_control(NULL), hdr_sz(0), content_length(0), - pstate(psReadyToParseStartLine) + pstate(psReadyToParseStartLine), + sources(0) {} HttpMsg::~HttpMsg() diff --git a/src/HttpMsg.h b/src/HttpMsg.h index 2a67db9b61..42c313642f 100644 --- a/src/HttpMsg.h +++ b/src/HttpMsg.h @@ -23,6 +23,26 @@ class HttpMsg : public RefCountable public: typedef RefCount Pointer; + /// Who may have created or modified this message? + enum Sources { + srcUnknown = 0, + + /* flags in 0xFFFF zone are for "secure" or "encrypted" sources */ + srcHttps = 1 << 0, ///< https_port or bumped http_port tunnel; HTTPS server + srcFtps = 1 << 1, ///< ftps_port or SFTP server; currently unused + srcIcaps = 1 << 2, ///< Secure ICAP service + srcEcaps = 1 << 3, ///< eCAP service that is considered secure; currently unused + + /* these flags "taint" the message: it may have been observed or mangled outside Squid */ + srcHttp = 1 << (16 + 0), ///< http_port or HTTP server + srcFtp = 1 << (16 + 1), ///< ftp_port or FTP server + srcIcap = 1 << (16 + 2), ///< traditional ICAP service without encryption + srcEcap = 1 << (16 + 3), ///< eCAP service that uses insecure libraries/daemons + srcGopher = 1 << (16 + 14), ///< Gopher server + srcWhois = 1 << (16 + 15), ///< Whois server + srcUnsafe = 0xFFFF0000, ///< Unsafe sources mask + srcSafe = 0x0000FFFF ///< Safe sources mask + }; HttpMsg(http_hdr_owner_type owner); virtual ~HttpMsg(); @@ -65,6 +85,8 @@ public: BodyPipe::Pointer body_pipe; // optional pipeline to receive message body + uint32_t sources; ///< The message sources + // returns true and sets hdr_sz on success // returns false and sets *error to zero when needs more data // returns false and sets *error to a positive Http::StatusCode on error diff --git a/src/HttpReply.cc b/src/HttpReply.cc index 69962f0110..f2409cac49 100644 --- a/src/HttpReply.cc +++ b/src/HttpReply.cc @@ -588,6 +588,7 @@ bool HttpReply::inheritProperties(const HttpMsg *aMsg) if (!aRep) return false; keep_alive = aRep->keep_alive; + sources = aRep->sources; return true; } diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index eca0de2d8b..252a314da6 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -250,6 +250,8 @@ HttpRequest::inheritProperties(const HttpMsg *aMsg) clientConnectionManager = aReq->clientConnectionManager; notes = aReq->notes; + + sources = aReq->sources; return true; } diff --git a/src/acl/Makefile.am b/src/acl/Makefile.am index 7e97930a37..f8ac12531d 100644 --- a/src/acl/Makefile.am +++ b/src/acl/Makefile.am @@ -153,6 +153,8 @@ SSL_ACLS = \ CertificateData.h \ Certificate.cc \ Certificate.h \ + ConnectionsEncrypted.cc \ + ConnectionsEncrypted.h \ ServerCertificate.cc \ ServerCertificate.h \ ServerName.cc \ diff --git a/src/adaptation/Config.cc b/src/adaptation/Config.cc index b16b9791d2..721db4b18e 100644 --- a/src/adaptation/Config.cc +++ b/src/adaptation/Config.cc @@ -157,11 +157,24 @@ Adaptation::Config::dumpService(StoreEntry *entry, const char *name) const typedef Services::iterator SCI; for (SCI i = AllServices().begin(); i != AllServices().end(); ++i) { const ServiceConfig &cfg = (*i)->cfg(); - storeAppendPrintf(entry, "%s " SQUIDSTRINGPH "_%s %s %d " SQUIDSTRINGPH "\n", + bool isEcap = cfg.protocol.caseCmp("ecap") == 0; + bool isIcap = !isEcap; + const char *optConnectionEncryption = ""; + // Print connections_encrypted option if no default value is used + if (cfg.secure.encryptTransport && !cfg.connectionEncryption) + optConnectionEncryption = " connection-encryption=off"; + else if (isEcap && !cfg.connectionEncryption) + optConnectionEncryption = " connection-encryption=off"; + else if (isIcap && !cfg.secure.encryptTransport && cfg.connectionEncryption) + optConnectionEncryption = " connection-encryption=on"; + + storeAppendPrintf(entry, "%s " SQUIDSTRINGPH " %s_%s %d " SQUIDSTRINGPH "%s\n", name, SQUIDSTRINGPRINT(cfg.key), cfg.methodStr(), cfg.vectPointStr(), cfg.bypass, - SQUIDSTRINGPRINT(cfg.uri)); + SQUIDSTRINGPRINT(cfg.uri), + + optConnectionEncryption); } } diff --git a/src/adaptation/ServiceConfig.cc b/src/adaptation/ServiceConfig.cc index 7721c71d3a..1aa63eefbe 100644 --- a/src/adaptation/ServiceConfig.cc +++ b/src/adaptation/ServiceConfig.cc @@ -127,6 +127,10 @@ Adaptation::ServiceConfig::parse() else if (strcmp(name, "on-overload") == 0) { grokked = grokOnOverload(onOverload, value); onOverloadSet = true; + } else if (strcmp(name, "connection-encryption") == 0) { + bool encrypt; + grokked = grokBool(encrypt, name, value); + connectionEncryption.configure(encrypt); } else if (strncmp(name, "ssl", 3) == 0 || strncmp(name, "tls-", 4) == 0) { #if !USE_OPENSSL debugs(3, DBG_PARSE_NOTE(DBG_IMPORTANT), "WARNING: adaptation option '" << name << "' requires --with-openssl. ICAP service option ignored."); diff --git a/src/adaptation/ServiceConfig.h b/src/adaptation/ServiceConfig.h index 23bd26b6c3..a926354e20 100644 --- a/src/adaptation/ServiceConfig.h +++ b/src/adaptation/ServiceConfig.h @@ -13,6 +13,7 @@ #include "base/RefCount.h" #include "security/PeerOptions.h" #include "SquidString.h" +#include "YesNoNone.h" namespace Adaptation { @@ -50,6 +51,7 @@ public: // security settings for adaptation service Security::PeerOptions secure; + YesNoNone connectionEncryption; ///< whether this service uses only secure connections protected: Method parseMethod(const char *buf) const; diff --git a/src/adaptation/ecap/ServiceRep.cc b/src/adaptation/ecap/ServiceRep.cc index 973cc555f9..f0cd833995 100644 --- a/src/adaptation/ecap/ServiceRep.cc +++ b/src/adaptation/ecap/ServiceRep.cc @@ -170,6 +170,8 @@ void Adaptation::Ecap::ServiceRep::finalize() { Adaptation::Service::finalize(); + if (!cfg().connectionEncryption.configured()) + writeableCfg().connectionEncryption.configure(true); theService = FindAdapterService(cfg().uri); if (theService) { try { diff --git a/src/adaptation/ecap/XactionRep.cc b/src/adaptation/ecap/XactionRep.cc index 2a5d1b95d0..46351c2841 100644 --- a/src/adaptation/ecap/XactionRep.cc +++ b/src/adaptation/ecap/XactionRep.cc @@ -420,6 +420,7 @@ Adaptation::Ecap::XactionRep::useAdapted(const libecap::shared_ptrbody()) { // final, bodyless answer proxyingAb = opNever; updateHistory(msg); @@ -733,3 +734,8 @@ Adaptation::Ecap::XactionRep::status() const return buf.content(); } +void +Adaptation::Ecap::XactionRep::updateSources(HttpMsg *adapted) +{ + adapted->sources |= service().cfg().connectionEncryption ? HttpMsg::srcEcaps : HttpMsg::srcEcap; +} diff --git a/src/adaptation/ecap/XactionRep.h b/src/adaptation/ecap/XactionRep.h index 7efb947e22..778ebbd67a 100644 --- a/src/adaptation/ecap/XactionRep.h +++ b/src/adaptation/ecap/XactionRep.h @@ -94,6 +94,7 @@ protected: void updateHistory(HttpMsg *adapted); void terminateMaster(); void scheduleStop(const char *reason); + void updateSources(HttpMsg *adapted); const libecap::Area clientIpValue() const; const libecap::Area usernameValue() const; diff --git a/src/adaptation/icap/ModXact.cc b/src/adaptation/icap/ModXact.cc index 220b1a0d78..875f8e7c36 100644 --- a/src/adaptation/icap/ModXact.cc +++ b/src/adaptation/icap/ModXact.cc @@ -771,6 +771,16 @@ void Adaptation::Icap::ModXact::startSending() if (state.sending == State::sendingVirgin) echoMore(); + else { + // If we are not using the virgin HTTP object update the + // HttpMsg::sources flag. + // The state.sending may set to State::sendingVirgin in the case + // of 206 responses too, where we do not want to update HttpMsg::sources + // flag. However even for 206 responses the state.sending is + // not set yet to sendingVirgin. This is done in later step + // after the parseBody method called. + updateSources(); + } } void Adaptation::Icap::ModXact::parseIcapHead() @@ -1949,6 +1959,12 @@ void Adaptation::Icap::ModXact::clearError() request->clearError(); } +void Adaptation::Icap::ModXact::updateSources() +{ + Must(adapted.header); + adapted.header->sources |= (service().cfg().connectionEncryption ? HttpMsg::srcIcaps : HttpMsg::srcIcap); +} + /* Adaptation::Icap::ModXactLauncher */ Adaptation::Icap::ModXactLauncher::ModXactLauncher(HttpMsg *virginHeader, HttpRequest *virginCause, AccessLogEntry::Pointer &alp, Adaptation::ServicePointer aService): diff --git a/src/adaptation/icap/ModXact.h b/src/adaptation/icap/ModXact.h index e223d67889..fc051c82cc 100644 --- a/src/adaptation/icap/ModXact.h +++ b/src/adaptation/icap/ModXact.h @@ -221,6 +221,7 @@ private: void prepEchoing(); void prepPartialBodyEchoing(uint64_t pos); void echoMore(); + void updateSources(); ///< Update the HttpMsg sources virtual bool doneAll() const; virtual void swanSong(); diff --git a/src/adaptation/icap/ServiceRep.cc b/src/adaptation/icap/ServiceRep.cc index 06fae2056c..c9fdf35a65 100644 --- a/src/adaptation/icap/ServiceRep.cc +++ b/src/adaptation/icap/ServiceRep.cc @@ -85,7 +85,11 @@ Adaptation::Icap::ServiceRep::finalize() if (cfg().secure.encryptTransport) { debugs(3, DBG_IMPORTANT, "Initializing service " << cfg().resource << " SSL context"); sslContext = writeableCfg().secure.createClientContext(true); - } + if (!cfg().connectionEncryption.configured()) + writeableCfg().connectionEncryption.configure(true); + } else if (!cfg().connectionEncryption.configured()) + writeableCfg().connectionEncryption.configure(false); + theSessionFailures.configure(TheConfig.oldest_service_failure > 0 ? TheConfig.oldest_service_failure : -1); diff --git a/src/cf.data.pre b/src/cf.data.pre index 78df090c5f..661021d41f 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1218,6 +1218,30 @@ IF USE_OPENSSL acl aclname ssl::server_name_regex [-i] \.foo\.com ... # regex matches server name obtained from various sources [fast] + + acl aclname connections_encrypted + # matches transactions with all HTTP messages received over TLS + # transport connections. [fast] + # + # The master transaction deals with HTTP messages received from + # various sources. All sources used by the master transaction in the + # past are considered by the ACL. The following rules define whether + # a given message source taints the entire master transaction, + # resulting in ACL mismatches: + # + # * The HTTP client transport connection is not TLS. + # * An adaptation service connection-encryption flag is off. + # * The peer or origin server transport connection is not TLS. + # + # Caching currently does not affect these rules. This cache ignorance + # implies that only the current HTTP client transport and REQMOD + # services status determine whether this ACL matches a from-cache + # transaction. The source of the cached response does not have any + # effect on future transaction that use the cached response without + # revalidation. This may change. + # + # DNS, ICP, and HTCP exchanges during the master transaction do not + # affect these rules. ENDIF acl aclname any-of acl1 acl2 ... # match any one of the acls [fast or slow] @@ -8445,6 +8469,17 @@ DOC_START Use the given number as the Max-Connections limit, regardless of the Max-Connections value given by the service, if any. + connection-encryption=on|off + Determines the ICAP service effect on the connections_encrypted + ACL. + + The default is "on" for Secure ICAP services (i.e., those + with the icaps:// service URIs scheme) and "off" for plain ICAP + services. + + Does not affect ICAP connections (e.g., does not turn Secure + ICAP on or off). + ==== ICAPS / TLS OPTIONS ==== These options are used for Secure ICAP (icaps://....) services only. @@ -8621,6 +8656,15 @@ DOC_START Routing is not allowed by default. + connection-encryption=on|off + Determines the eCAP service effect on the connections_encrypted + ACL. + + Defaults to "on", which does not taint the master transaction + w.r.t. that ACL. + + Does not affect eCAP API calls. + Older ecap_service format without optional named parameters is deprecated but supported for backward compatibility. diff --git a/src/client_side.cc b/src/client_side.cc index 2ede121d18..4b5505e1b7 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -2342,6 +2342,8 @@ clientProcessRequest(ConnStateData *conn, const Http1::RequestParserPointer &hp, // TODO: decouple http->flags.accel from request->flags.sslBumped request->flags.noDirect = (request->flags.accelerated && !request->flags.sslBumped) ? !conn->port->allow_direct : 0; + request->sources |= isFtp ? HttpMsg::srcFtp : + ((request->flags.sslBumped || conn->port->transport.protocol == AnyP::PROTO_HTTPS) ? HttpMsg::srcHttps : HttpMsg::srcHttp); #if USE_AUTH if (request->flags.sslBumped) { if (conn->getAuth() != NULL) diff --git a/src/clients/FtpGateway.cc b/src/clients/FtpGateway.cc index 2a836ae1fc..0d7ecc2729 100644 --- a/src/clients/FtpGateway.cc +++ b/src/clients/FtpGateway.cc @@ -2589,6 +2589,7 @@ Ftp::Gateway::appendSuccessHeader() if (mime_enc) reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc); + reply->sources |= HttpMsg::srcFtp; setVirginReply(reply); adaptOrFinalizeReply(); } diff --git a/src/clients/FtpRelay.cc b/src/clients/FtpRelay.cc index b5483ef4f4..f715c95dd9 100644 --- a/src/clients/FtpRelay.cc +++ b/src/clients/FtpRelay.cc @@ -346,6 +346,7 @@ Ftp::Relay::forwardReply() EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); HttpReply *const reply = createHttpReply(Http::scNoContent); + reply->sources |= HttpMsg::srcFtp; setVirginReply(reply); adaptOrFinalizeReply(); @@ -418,6 +419,8 @@ Ftp::Relay::startDataDownload() " (" << data.conn->local << ")"); HttpReply *const reply = createHttpReply(Http::scOkay, -1); + reply->sources |= HttpMsg::srcFtp; + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); setVirginReply(reply); adaptOrFinalizeReply(); diff --git a/src/gopher.cc b/src/gopher.cc index bc8226a78c..35d8c41ca8 100644 --- a/src/gopher.cc +++ b/src/gopher.cc @@ -122,6 +122,7 @@ public: char *buf; /* pts to a 4k page */ Comm::ConnectionPointer serverConn; FwdState::Pointer fwd; + HttpReply::Pointer reply_; char replybuf[BUFSIZ]; }; @@ -249,6 +250,7 @@ gopherMimeCreate(GopherStateData * gopherState) reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc); entry->replaceHttpReply(reply); + gopherState->reply_ = reply; } /** @@ -772,8 +774,11 @@ gopherReadReply(const Comm::ConnectionPointer &conn, char *buf, size_t len, Comm ++IOStats.Gopher.read_hist[bin]; HttpRequest *req = gopherState->fwd->request; - if (req->hier.bodyBytesRead < 0) + if (req->hier.bodyBytesRead < 0) { req->hier.bodyBytesRead = 0; + // first bytes read, update Reply flags: + gopherState->reply_->sources |= HttpMsg::srcGopher; + } req->hier.bodyBytesRead += len; } diff --git a/src/http.cc b/src/http.cc index 0e62471be0..5b39187bf4 100644 --- a/src/http.cc +++ b/src/http.cc @@ -769,6 +769,8 @@ HttpStateData::processReplyHeader() // done with Parser, now process using the HttpReply hp = NULL; + newrep->sources |= request->url.getScheme() == AnyP::PROTO_HTTPS ? HttpMsg::srcHttps : HttpMsg::srcHttp; + newrep->removeStaleWarnings(); if (newrep->sline.protocol == AnyP::PROTO_HTTP && newrep->sline.status() >= 100 && newrep->sline.status() < 200) { diff --git a/src/whois.cc b/src/whois.cc index 392400f71c..8fe78987bd 100644 --- a/src/whois.cc +++ b/src/whois.cc @@ -105,6 +105,7 @@ WhoisState::setReplyToOK(StoreEntry *sentry) HttpReply *reply = new HttpReply; sentry->buffer(); reply->setHeaders(Http::scOkay, "Gatewaying", "text/plain", -1, -1, -2); + reply->sources |= HttpMsg::srcWhois; sentry->replaceHttpReply(reply); } -- 2.47.2