From: Christos Tsantilas Date: Mon, 23 May 2016 17:03:20 +0000 (+0300) Subject: Sync with trunk-r14686 X-Git-Tag: SQUID_4_0_13~5^2~7 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0bffe3cec1400f4d69724552e77e77878defd8a3;p=thirdparty%2Fsquid.git Sync with trunk-r14686 - Replaces with newer versions of BinaryTokenizer and HandshakeParser classes - Modifications to use latest Handshake parser from trunk. - Get HandshakeParser::serverCertificates, HandshakeParser::parseServerCertificates and HandshakeParser::ParseCertificate implementation from lp:fast-sni branch --- 0bffe3cec1400f4d69724552e77e77878defd8a3 diff --cc src/client_side.cc index f855e8d4cb,74ca44c855..fe01dbd522 --- a/src/client_side.cc +++ b/src/client_side.cc @@@ -3269,18 -3259,12 +3276,11 @@@ boo ConnStateData::splice() { // normally we can splice here, because we just got client hello message - auto ssl = fd_table[clientConnection->fd].ssl.get(); - - //retrieve received TLS client information - clientConnection->tlsNegotiations()->fillWith(ssl); -- - BIO *b = SSL_get_rbio(ssl); - Ssl::ClientBio *bio = static_cast(b->ptr); - SBuf const &rbuf = bio->rBufData(); - debugs(83,5, "Bio for " << clientConnection << " read " << rbuf.length() << " helo bytes"); - // Do splice: - fd_table[clientConnection->fd].read_method = &default_read_method; - fd_table[clientConnection->fd].write_method = &default_write_method; + if (fd_table[clientConnection->fd].ssl.get()) { + // Restore default read methods + fd_table[clientConnection->fd].read_method = &default_read_method; + fd_table[clientConnection->fd].write_method = &default_write_method; + } if (transparent()) { // set the current protocol to something sensible (was "HTTPS" for the bumping process) diff --cc src/security/Handshake.cc index 08c4033a4d,f5744f1eb9..cd8f418d7f --- a/src/security/Handshake.cc +++ b/src/security/Handshake.cc @@@ -151,27 -313,30 +313,31 @@@ Security::HandshakeParser::parseHandsha const Handshake message(tkMessages); switch (message.msg_type) { - case HandshakeType::hskServerHello: - Must(state < atHelloReceived); - // TODO: Parse ServerHello in message.body; extract version/session - // If the server is resuming a session, stop parsing w/o certificates - // because all subsequent [Finished] messages will be encrypted, right? - state = atHelloReceived; - return; - case HandshakeType::hskCertificate: - Must(state < atCertificatesReceived); - parseServerCertificates(message.body); - state = atCertificatesReceived; - return; - case HandshakeType::hskServerHelloDone: - Must(state < atHelloDoneReceived); - // zero-length - state = atHelloDoneReceived; - parseDone = true; - return; + case HandshakeType::hskClientHello: + Must(state < atHelloReceived); + Security::HandshakeParser::parseClientHelloHandshakeMessage(message.msg_body); + state = atHelloReceived; + done = "ClientHello"; + return; + case HandshakeType::hskServerHello: + Must(state < atHelloReceived); + parseServerHelloHandshakeMessage(message.msg_body); + state = atHelloReceived; + return; + case HandshakeType::hskCertificate: + Must(state < atCertificatesReceived); ++ parseServerCertificates(message.msg_body); + state = atCertificatesReceived; + return; + case HandshakeType::hskServerHelloDone: + Must(state < atHelloDoneReceived); + // zero-length + state = atHelloDoneReceived; + done = "ServerHelloDone"; + return; } - debugs(83, 5, "ignoring " << - DebugFrame("handshake msg", message.msg_type, message.length)); + debugs(83, 5, "ignoring " << message.msg_body.length() << "-byte type-" << + message.msg_type << " handshake message"); } void @@@ -215,34 -535,104 +536,138 @@@ Security::HandshakeParser::parseHello(c } #if USE_OPENSSL +X509 * +Security::HandshakeParser::ParseCertificate(const SBuf &raw) +{ + typedef const unsigned char *x509Data; + const x509Data x509Start = reinterpret_cast(raw.rawContent()); + x509Data x509Pos = x509Start; + X509 *x509 = d2i_X509(nullptr, &x509Pos, raw.length()); + Must(x509); // successfully parsed + Must(x509Pos == x509Start + raw.length()); // no leftovers + return x509; +} + +void +Security::HandshakeParser::parseServerCertificates(const SBuf &raw) +{ - BinaryTokenizer tkList(raw); - const P24String list(tkList, "CertificateList"); ++ Parser::BinaryTokenizer tkList(raw); ++ const SBuf clist = tkList.pstring24("CertificateList"); + Must(tkList.atEnd()); // no leftovers after all certificates + - BinaryTokenizer tkItems(list.body); ++ Parser::BinaryTokenizer tkItems(clist); + while (!tkItems.atEnd()) { - const P24String item(tkItems, "Certificate"); - X509 *cert = ParseCertificate(item.body); ++ X509 *cert = ParseCertificate(tkItems.pstring24("Certificate")); + if (!serverCertificates.get()) + serverCertificates.reset(sk_X509_new_null()); + sk_X509_push(serverCertificates.get(), cert); + debugs(83, 7, "parsed " << sk_X509_num(serverCertificates.get()) << " certificates so far"); + } + +} + + /// A helper function to create a set of all supported TLS extensions + static + Security::Extensions + Security::SupportedExtensions() + { + // optimize lookup speed by reserving the number of values x3, approximately + Security::Extensions extensions(64); + + // Keep this list ordered and up to date by running something like + // egrep '# *define TLSEXT_TYPE_' /usr/include/openssl/tls1.h + // TODO: Teach OpenSSL to return the list of extensions it supports. + #if defined(TLSEXT_TYPE_server_name) // 0 + extensions.insert(TLSEXT_TYPE_server_name); + #endif + #if defined(TLSEXT_TYPE_max_fragment_length) // 1 + extensions.insert(TLSEXT_TYPE_max_fragment_length); + #endif + #if defined(TLSEXT_TYPE_client_certificate_url) // 2 + extensions.insert(TLSEXT_TYPE_client_certificate_url); + #endif + #if defined(TLSEXT_TYPE_trusted_ca_keys) // 3 + extensions.insert(TLSEXT_TYPE_trusted_ca_keys); + #endif + #if defined(TLSEXT_TYPE_truncated_hmac) // 4 + extensions.insert(TLSEXT_TYPE_truncated_hmac); + #endif + #if defined(TLSEXT_TYPE_status_request) // 5 + extensions.insert(TLSEXT_TYPE_status_request); + #endif + #if defined(TLSEXT_TYPE_user_mapping) // 6 + extensions.insert(TLSEXT_TYPE_user_mapping); + #endif + #if defined(TLSEXT_TYPE_client_authz) // 7 + extensions.insert(TLSEXT_TYPE_client_authz); + #endif + #if defined(TLSEXT_TYPE_server_authz) // 8 + extensions.insert(TLSEXT_TYPE_server_authz); + #endif + #if defined(TLSEXT_TYPE_cert_type) // 9 + extensions.insert(TLSEXT_TYPE_cert_type); + #endif + #if defined(TLSEXT_TYPE_elliptic_curves) // 10 + extensions.insert(TLSEXT_TYPE_elliptic_curves); + #endif + #if defined(TLSEXT_TYPE_ec_point_formats) // 11 + extensions.insert(TLSEXT_TYPE_ec_point_formats); + #endif + #if defined(TLSEXT_TYPE_srp) // 12 + extensions.insert(TLSEXT_TYPE_srp); + #endif + #if defined(TLSEXT_TYPE_signature_algorithms) // 13 + extensions.insert(TLSEXT_TYPE_signature_algorithms); + #endif + #if defined(TLSEXT_TYPE_use_srtp) // 14 + extensions.insert(TLSEXT_TYPE_use_srtp); + #endif + #if defined(TLSEXT_TYPE_heartbeat) // 15 + extensions.insert(TLSEXT_TYPE_heartbeat); + #endif + #if defined(TLSEXT_TYPE_session_ticket) // 35 + extensions.insert(TLSEXT_TYPE_session_ticket); + #endif + #if defined(TLSEXT_TYPE_renegotiate) // 0xff01 + extensions.insert(TLSEXT_TYPE_renegotiate); + #endif + #if defined(TLSEXT_TYPE_next_proto_neg) // 13172 + extensions.insert(TLSEXT_TYPE_next_proto_neg); + #endif + + /* + * OpenSSL does not support these last extensions by default, but those + * building the OpenSSL libraries and/or Squid might define them. + */ + + // OpenSSL may be built to support draft-rescorla-tls-opaque-prf-input-00, + // with the extension type value configured at build time. OpenSSL, Squid, + // and TLS agents must all be built with the same extension type value. + #if defined(TLSEXT_TYPE_opaque_prf_input) + extensions.insert(TLSEXT_TYPE_opaque_prf_input); + #endif + + // Define this to add extensions supported by your OpenSSL but unknown to + // your Squid version. Use {list-initialization} to add multiple extensions. + #if defined(TLSEXT_TYPE_SUPPORTED_BY_MY_SQUID) + extensions.insert(TLSEXT_TYPE_SUPPORTED_BY_MY_SQUID); + #endif + + return extensions; // might be empty + } + + #else + ++void ++Security::HandshakeParser::parseServerCertificates(const SBuf &raw) ++{ ++} ++ + static + Security::Extensions + Security::SupportedExtensions() + { + return Extensions(); // no extensions are supported without OpenSSL + } #endif + diff --cc src/security/Handshake.h index 3ed2261c66,36f9f6ba46..747519f5fb --- a/src/security/Handshake.h +++ b/src/security/Handshake.h @@@ -9,13 -9,12 +9,15 @@@ #ifndef SQUID_SECURITY_HANDSHAKE_H #define SQUID_SECURITY_HANDSHAKE_H - #include "fd.h" + #include "anyp/ProtocolVersion.h" + #include "base/YesNoNone.h" #include "parser/BinaryTokenizer.h" - #include "sbuf/SBuf.h" +#if USE_OPENSSL +#include "ssl/gadgets.h" +#endif + #include + namespace Security { @@@ -100,38 -58,25 +61,29 @@@ public /// The parsing states typedef enum {atHelloNone = 0, atHelloStarted, atHelloReceived, atCertificatesReceived, atHelloDoneReceived, atNstReceived, atCcsReceived, atFinishReceived} ParserState; - HandshakeParser(): state(atHelloNone), ressumingSession(false), parseDone(false), parseError(false), currentContentType(0), unParsedContent(0), parsingPos(0), currentMsg(0), currentMsgSize(0), certificatesMsgPos(0), certificatesMsgSize(0) {} + HandshakeParser(); - /// Parses the initial sequence of raw bytes sent by the SSL server. - /// Returns true upon successful completion (HelloDone or Finished received). - /// Otherwise, returns false (and sets parseError to true on errors). - bool parseServerHello(const SBuf &data); + /// Parses the initial sequence of raw bytes sent by the TLS/SSL agent. + /// Returns true upon successful completion (e.g., got HelloDone). + /// Returns false if more data is needed. + /// Throws on errors. + bool parseHello(const SBuf &data); + + TlsDetails::Pointer details; ///< TLS handshake meta info or nil. +#if USE_OPENSSL + Ssl::X509_STACK_Pointer serverCertificates; ///< parsed certificates chain +#endif + ParserState state; ///< current parsing state. - bool ressumingSession; ///< True if this is a resumming session - - bool parseDone; ///< The parser finishes its job - bool parseError; ///< Set to tru by parse on parse error. - - private: - unsigned int currentContentType; ///< The current SSL record content type - size_t unParsedContent; ///< The size of current SSL record, which is not parsed yet - size_t parsingPos; ///< The parsing position from the beginning of parsed data - size_t currentMsg; ///< The current handshake message possition from the beginning of parsed data - size_t currentMsgSize; ///< The current handshake message size. - - size_t certificatesMsgPos; ///< The possition of certificates message from the beggining of parsed data - size_t certificatesMsgSize; ///< The size of certificates message + bool resumingSession; ///< True if this is a resuming session private: - void parseServerHelloTry(); - + bool isSslv2Record(const SBuf &raw) const; void parseRecord(); + void parseModernRecord(); + void parseVersion2Record(); void parseMessages(); void parseChangeCipherCpecMessage(); @@@ -140,11 -85,22 +92,27 @@@ void parseApplicationDataMessage(); void skipMessage(const char *msgType); + bool parseRecordVersion2Try(); + void parseVersion2HandshakeMessage(const SBuf &raw); + void parseClientHelloHandshakeMessage(const SBuf &raw); + void parseServerHelloHandshakeMessage(const SBuf &raw); + + bool parseCompressionMethods(const SBuf &raw); + void parseExtensions(const SBuf &raw); + SBuf parseSniExtension(const SBuf &extensionData) const; + + void parseCiphers(const SBuf &raw); + void parseV23Ciphers(const SBuf &raw); + + void parseServerCertificates(const SBuf &raw); +#if USE_OPENSSL + static X509 *ParseCertificate(const SBuf &raw); +#endif + + unsigned int currentContentType; ///< The current TLS/SSL record content type + + const char *done; ///< not nil if we got what we were looking for + /// concatenated TLSPlaintext.fragments of TLSPlaintext.type SBuf fragments; diff --cc src/ssl/PeerConnector.cc index d571ca5691,0e22b8ce70..cf1cb16e07 --- a/src/ssl/PeerConnector.cc +++ b/src/ssl/PeerConnector.cc @@@ -11,11 -11,10 +11,12 @@@ #include "squid.h" #include "acl/FilledChecklist.h" #include "comm/Loops.h" +#include "Downloader.h" #include "errorpage.h" #include "fde.h" +#include "http/Stream.h" #include "HttpRequest.h" + #include "security/NegotiationHistory.h" #include "SquidConfig.h" #include "ssl/bio.h" #include "ssl/cert_validate_message.h" diff --cc src/ssl/bio.cc index 9c2b22ad79,fb68f86471..c048d07f54 --- a/src/ssl/bio.cc +++ b/src/ssl/bio.cc @@@ -245,6 -196,20 +196,22 @@@ Ssl::ClientBio::read(char *buf, int siz return -1; } + Ssl::ServerBio::ServerBio(const int anFd): + Bio(anFd), + helloMsgSize(0), + helloBuild(false), + allowSplice(false), + allowBump(false), + holdWrite_(false), ++ holdRead_(true), + record_(false), + parsedHandshake(false), ++ parseError(false), + bumpMode_(bumpNone), + rbufConsumePos(0) + { + } + void Ssl::ServerBio::stateChanged(const SSL *ssl, int where, int ret) { @@@ -258,50 -224,86 +226,93 @@@ Ssl::ServerBio::setClientFeatures(Secur }; int - Ssl::ServerBio::readAndBufferServerHelloMsg(BIO *table, const char *description) + Ssl::ServerBio::read(char *buf, int size, BIO *table) { + if (parsedHandshake) // done parsing TLS Hello + return readAndGive(buf, size, table); + else + return readAndParse(buf, size, table); + } - int ret = readAndBuffer(table, description); - if (ret <= 0) - return ret; + /// Read and give everything to OpenSSL. + int + Ssl::ServerBio::readAndGive(char *buf, const int size, BIO *table) + { + // If we have unused buffered bytes, give those bytes to OpenSSL now, + // before reading more. TODO: Read if we have buffered less than size? + if (rbufConsumePos < rbuf.length()) + return giveBuffered(buf, size); - if (!parser_.parseServerHello(rbuf)) { - if (!parser_.parseError) - BIO_set_retry_read(table); - return -1; + if (record_) { + const int result = readAndBuffer(table); + if (result <= 0) + return result; + return giveBuffered(buf, size); } - return 1; + return Ssl::Bio::read(buf, size, table); } + /// Read and give everything to our parser. + /// When/if parsing is finished (successfully or not), start giving to OpenSSL. int - Ssl::ServerBio::read(char *buf, int size, BIO *table) + Ssl::ServerBio::readAndParse(char *buf, const int size, BIO *table) { - if (!parser_.parseDone || record_) { - int ret = readAndBufferServerHelloMsg(table, "TLS server Hello"); - if (!rbuf.length() && parser_.parseDone && ret <= 0) - return ret; + const int result = readAndBuffer(table); + if (result <= 0) + return result; + + try { + if (!parser_.parseHello(rbuf)) { + // need more data to finish parsing + BIO_set_retry_read(table); + return -1; + } + parsedHandshake = true; // done parsing (successfully) + } + catch (const std::exception &ex) { + debugs(83, 2, "parsing error on FD " << fd_ << ": " << ex.what()); + parsedHandshake = true; // done parsing (due to an error) ++ parseError = true; + } + + if (holdRead_) { - debugs(83, 7, "Hold flag is set on ServerBio, retry latter. (Hold " << size << "bytes)"); - BIO_set_retry_read(table); - return -1; ++ debugs(83, 7, "Hold flag is set, retry latter. (Hold " << size << "bytes)"); ++ BIO_set_retry_read(table); ++ return -1; } - if (parser_.parseDone && !parser_.parseError) { - int unsent = rbuf.length() - rbufConsumePos; - if (unsent > 0) { - int bytes = (size <= unsent ? size : unsent); - memcpy(buf, rbuf.rawContent() + rbufConsumePos, bytes); - rbufConsumePos += bytes; - debugs(83, 7, "Pass " << bytes << " bytes to openSSL as read"); - return bytes; - } else - return Ssl::Bio::read(buf, size, table); - } + return giveBuffered(buf, size); + } - return -1; + /// Reads more data into the read buffer. Returns either the number of bytes + /// read or, on errors (including "try again" errors), a negative number. + int + Ssl::ServerBio::readAndBuffer(BIO *table) + { + char *space = rbuf.rawSpace(SQUID_TCP_SO_RCVBUF); + const int result = Ssl::Bio::read(space, rbuf.spaceSize(), table); + if (result <= 0) + return result; + + rbuf.forceSize(rbuf.length() + result); + return result; + } + + /// give previously buffered bytes to OpenSSL + /// returns the number of bytes given + int + Ssl::ServerBio::giveBuffered(char *buf, const int size) + { + if (rbuf.length() <= rbufConsumePos) + return -1; // buffered nothing yet + + const int unsent = rbuf.length() - rbufConsumePos; + const int bytes = (size <= unsent ? size : unsent); + memcpy(buf, rbuf.rawContent() + rbufConsumePos, bytes); + rbufConsumePos += bytes; + debugs(83, 7, bytes << "<=" << size << " bytes to OpenSSL"); + return bytes; } // This function makes the required checks to examine if the client hello diff --cc src/ssl/bio.h index 1399f0908d,32b4e1ad4e..75f8665ff0 --- a/src/ssl/bio.h +++ b/src/ssl/bio.h @@@ -217,38 -148,38 +152,48 @@@ public void mode(Ssl::BumpMode m) {bumpMode_ = m;} Ssl::BumpMode bumpMode() {return bumpMode_;} ///< return the bumping mode + /// Return true if the Server hello message received - bool gotHello() const { return (parser_.parseDone && !parser_.parseError); } ++ bool gotHello() const { return (parsedHandshake && !parseError); } + + /// Return true if the Server Hello parsing failed - bool gotHelloFailed() const { return (parser_.parseDone && parser_.parseError); } ++ bool gotHelloFailed() const { return (parsedHandshake && parseError); } + + const Ssl::X509_STACK_Pointer &serverCertificatesIfAny() { return parser_.serverCertificates; } /* XXX: may be nil */ + + /// \return the TLS Details advertised by TLS server. + const Security::TlsDetails::Pointer &receivedHelloDetails() const {return parser_.details;} + private: - sslFeatures clientFeatures; ///< SSL client features extracted from ClientHello message or SSL object + int readAndGive(char *buf, const int size, BIO *table); + int readAndParse(char *buf, const int size, BIO *table); + int readAndBuffer(BIO *table); + int giveBuffered(char *buf, const int size); + + /// SSL client features extracted from ClientHello message or SSL object + Security::TlsDetails::Pointer clientTlsDetails; + /// TLS client hello message, used to adapt our tls Hello message to the server + SBuf clientHelloMessage; SBuf helloMsg; ///< Used to buffer output data. mb_size_t helloMsgSize; bool helloBuild; ///< True if the client hello message sent to the server bool allowSplice; ///< True if the SSL stream can be spliced bool allowBump; ///< True if the SSL stream can be bumped bool holdWrite_; ///< The write hold state of the bio. + bool holdRead_; ///< The read hold state of the bio. bool record_; ///< If true the input data recorded to rbuf for internal use + bool parsedHandshake; ///< whether we are done parsing TLS Hello ++ bool parseError; ///< error while parsing server hello message Ssl::BumpMode bumpMode_; - ///< The size of data stored in rbuf which passed to the openSSL + /// The size of data stored in rbuf which passed to the openSSL size_t rbufConsumePos; - Security::HandshakeParser parser_; ///< The SSL messages parser. + Security::HandshakeParser parser_; ///< The TLS/SSL messages parser. }; - inline - std::ostream &operator <<(std::ostream &os, Ssl::Bio::sslFeatures const &f) - { - return f.print(os); - } - } // namespace Ssl + void + applyTlsDetailsToSSL(SSL *ssl, Security::TlsDetails::Pointer const &details, Ssl::BumpMode bumpMode); + #endif /* SQUID_SSL_BIO_H */