From: Remi Gacogne Date: Fri, 17 Sep 2021 15:31:22 +0000 (+0200) Subject: dnsdist: Add experimental support for TLS asynchronous engines X-Git-Tag: auth-4.7.0-alpha1~120^2~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=489caa9fad6d609a5bce16c2136184c57496cbf4;p=thirdparty%2Fpdns.git dnsdist: Add experimental support for TLS asynchronous engines --- diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc index a052779e38..12feb6bf4c 100644 --- a/pdns/dnsdist-console.cc +++ b/pdns/dnsdist-console.cc @@ -490,6 +490,7 @@ const std::vector g_consoleKeywords{ { "KeyValueStoreLookupRule", true, "kvs, lookupKey", "matches queries if the key is found in the specified Key Value store" }, { "KeyValueStoreRangeLookupRule", true, "kvs, lookupKey", "matches queries if the key is found in the specified Key Value store" }, { "leastOutstanding", false, "", "Send traffic to downstream server with least outstanding queries, with the lowest 'order', and within that the lowest recent latency"}, + { "loadTLSEngine", true, "engineName [, defaultString]", "Load the OpenSSL engine named 'engineName', setting the engine default string to 'defaultString' if supplied"}, { "LogAction", true, "[filename], [binary], [append], [buffered]", "Log a line for each query, to the specified file if any, to the console (require verbose) otherwise. When logging to a file, the `binary` optional parameter specifies whether we log in binary form (default) or in textual form, the `append` optional parameter specifies whether we open the file for appending or truncate each time (default), and the `buffered` optional parameter specifies whether writes to the file are buffered (default) or not." }, { "LogResponseAction", true, "[filename], [append], [buffered]", "Log a line for each response, to the specified file if any, to the console (require verbose) otherwise. The `append` optional parameter specifies whether we open the file for appending or truncate each time (default), and the `buffered` optional parameter specifies whether writes to the file are buffered (default) or not." }, { "LuaAction", true, "function", "Invoke a Lua function that accepts a DNSQuestion" }, diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index 782b5fed70..03b8e826e2 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -236,6 +236,10 @@ static void parseTLSConfig(TLSConfig& config, const std::string& context, boost: if (vars->count("enableRenegotiation")) { config.d_enableRenegotiation = boost::get((*vars)["enableRenegotiation"]); } + + if (vars->count("tlsAsyncMode")) { + config.d_asyncMode = boost::get((*vars).at("tlsAsyncMode")); + } } #endif // defined(HAVE_DNS_OVER_TLS) || defined(HAVE_DNS_OVER_HTTPS) @@ -2807,6 +2811,20 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) g_socketUDPSendBuffer = snd; g_socketUDPRecvBuffer = recv; }); + +#if defined(HAVE_LIBSSL) + luaCtx.writeFunction("loadTLSEngine", [client](const std::string& engineName, boost::optional defaultString) { + if (client) { + return; + } + + auto [success, error] = libssl_load_engine(engineName, defaultString ? std::optional(*defaultString) : std::nullopt); + if (!success) { + g_outputBuffer = "Error while trying to load TLS engine '" + engineName + "': " + error + "\n"; + errlog("Error while trying to load TLS engine '%s': %s", engineName, error); + } + }); +#endif /* HAVE_LIBSSL */ } vector> setupLua(LuaContext& luaCtx, bool client, bool configCheck, const std::string& config) diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc index f181d751a6..99180826fb 100644 --- a/pdns/dnsdist-tcp.cc +++ b/pdns/dnsdist-tcp.cc @@ -383,6 +383,17 @@ void IncomingTCPConnectionState::terminateClientConnection() } } d_ownedConnectionsToBackend.clear(); + + /* let's make sure we don't have any remaining async descriptors laying around */ + auto afds = d_handler.getAsyncFDs(); + for (const auto afd : afds) { + try { + d_threadData.mplexer->removeReadFD(afd); + } + catch (...) { + } + } + /* meaning we will no longer be 'active' when the backend response or timeout comes in */ d_ioState.reset(); @@ -416,8 +427,43 @@ void IncomingTCPConnectionState::queueResponse(std::shared_ptractive()) { - state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedWrite ? state->getClientWriteTTD(now) : state->getClientReadTTD(now)); + updateIO(state, iostate, now); + } + } +} + +void IncomingTCPConnectionState::handleAsyncReady(int fd, FDMultiplexer::funcparam_t& param) +{ + auto state = boost::any_cast>(param); + + /* If we are here, the async jobs for this SSL* are finished + so we should be able to remove all FDs */ + auto afds = state->d_handler.getAsyncFDs(); + for (const auto afd : afds) { + try { + state->d_threadData.mplexer->removeReadFD(afd); } + catch (...) { + } + } + + /* and now we restart our own I/O state machine */ + struct timeval now; + gettimeofday(&now, nullptr); + handleIO(state, now); +} + +void IncomingTCPConnectionState::updateIO(std::shared_ptr& state, IOState newState, const struct timeval& now) +{ + if (newState == IOState::Async) { + auto fds = state->d_handler.getAsyncFDs(); + for (const auto fd : fds) { + state->d_threadData.mplexer->addReadFD(fd, handleAsyncReady, state); + } + state->d_ioState->update(IOState::Done, handleIOCallback, state); + } + else { + state->d_ioState->update(newState, handleIOCallback, state, newState == IOState::NeedWrite ? state->getClientWriteTTD(now) : state->getClientReadTTD(now)); } } @@ -1004,7 +1050,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptrd_ioState->update(iostate, handleIOCallback, state); } else { - state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now)); + updateIO(state, iostate, now); } ioGuard.release(); } @@ -1028,7 +1074,7 @@ void IncomingTCPConnectionState::notifyIOError(IDState&& query, const struct tim if (state->active() && iostate != IOState::Done) { // we need to update the state right away, nobody will do that for us - state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedWrite ? state->getClientWriteTTD(now) : state->getClientReadTTD(now)); + updateIO(state, iostate, now); } } catch (const std::exception& e) { diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh index 9ed8b6b3fa..6fa76ab6a1 100644 --- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh +++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh @@ -113,6 +113,8 @@ public: static void handleIO(std::shared_ptr& conn, const struct timeval& now); static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param); + static void handleAsyncReady(int fd, FDMultiplexer::funcparam_t& param); + static void updateIO(std::shared_ptr& state, IOState newState, const struct timeval& now); static IOState sendResponse(std::shared_ptr& state, const struct timeval& now, TCPResponse&& response); static void queueResponse(std::shared_ptr& state, const struct timeval& now, TCPResponse&& response); diff --git a/pdns/dnsdistdist/docs/advanced/tuning.rst b/pdns/dnsdistdist/docs/advanced/tuning.rst index 6d85961bf4..20aebc7bd9 100644 --- a/pdns/dnsdistdist/docs/advanced/tuning.rst +++ b/pdns/dnsdistdist/docs/advanced/tuning.rst @@ -107,6 +107,10 @@ If the number of queued connections ("Queued" in :func:`showTCPStats`) reaches t A different possibility is that there is not enough threads accepting new connections and distributing them to worker threads. Looking at whether the ``listenOverflows`` metric in :func:`dumpStats` increase over time will tell if we are losing TCP connections because the queue is full. In that case, since a single :func:`addLocal` or :func:`addTLSLocal` directive results in only one acceptor thread, it might useful to add more of these. +For incoming and outgoing DNS over TLS support, the choice of the TLS provider (OpenSSL and GnuTLS are both supported) might yield very different results depending on the exact architecture. + +Since 1.7.0, incoming DNS over TLS might also benefits from experimental support for TLS acceleration engines, like Intel QAT. See :func:`loadTLSEngine`, and the `tlsAsyncMode` parameter of :func:`addTLSLocal` for more information. + Rules and Lua ------------- diff --git a/pdns/dnsdistdist/docs/reference/config.rst b/pdns/dnsdistdist/docs/reference/config.rst index 5b5cc0cf83..c597e80302 100644 --- a/pdns/dnsdistdist/docs/reference/config.rst +++ b/pdns/dnsdistdist/docs/reference/config.rst @@ -166,6 +166,8 @@ Listen Sockets ``sessionTimeout`` and ``tcpListenQueueSize`` options added. .. versionchanged:: 1.6.0 ``enableRenegotiation``, ``maxConcurrentTCPConnections``, ``maxInFlight`` and ``releaseBuffers`` options added. + .. versionchanged:: 1.7.0 + ``tlsAsyncMode`` option added. Listen on the specified address and TCP port for incoming DNS over TLS connections, presenting the specified X.509 certificate. @@ -199,6 +201,7 @@ Listen Sockets * ``maxConcurrentTCPConnections=0``: int - Maximum number of concurrent incoming TCP connections. The default is 0 which means unlimited. * ``releaseBuffers=true``: bool - Whether OpenSSL should release its I/O buffers when a connection goes idle, saving roughly 35 kB of memory per connection. * ``enableRenegotiation=false``: bool - Whether secure TLS renegotiation should be enabled (OpenSSL only, the GnuTLS provider does not support it). Disabled by default since it increases the attack surface and is seldom used for DNS. + * ``tlsAsyncMode=false``: bool - Whether to enable experimental asynchronous TLS I/O operations if OpenSSL is used as the TLS provider and an asynchronous capable SSL engine is loaded. See also :func:`loadTLSEngine` to load the engine. .. function:: setLocal(address[, options]) @@ -1611,6 +1614,17 @@ Other functions :param int numberOfDaysOfValidity: Number of days this OCSP response should be valid. :param int numberOfMinutesOfValidity: Number of minutes this OCSP response should be valid, in addition to the number of days. +.. function:: loadTLSEngine(engineName [, defaultString]) + + .. versionadded:: 1.7.0 + + Load the OpenSSL engine named ``engineName``, setting the engine default string to ``defaultString`` if supplied. Engines can be used to accelerate cryptographic operations, like for example Intel QAT. + At the moment up to a maximum of 32 loaded engines are supported, and that support is experimental. + Some engines might not actually decrease, and sometimes increase, the CPU usage when the TLS asynchronous mode of OpenSSL is not enabled. To enable it see the ``tlsAsyncMode`` parameter on :func:`addTLSLocal`. + + :param string engineName: The name of the engine to load. + :param string defaultString: The default string to pass to the engine. The exact value depends on the engine but represents the algorithms to register to this engine, as a list of comma-separated keywords. For example "RSA,EC,DSA,DH,PKEY,PKEY_CRYPTO,PKEY_ASN1". + DOHFrontend ~~~~~~~~~~~ diff --git a/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc b/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc index edea2c88df..87f79286b9 100644 --- a/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistnghttp2_cc.cc @@ -444,6 +444,11 @@ public: { } + std::vector getAsyncFDs() override + { + return {}; + } + /* unused in that context, don't bother */ void doHandshake() override { diff --git a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc index 1fbb00e9ce..9b81805ca3 100644 --- a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc +++ b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc @@ -254,6 +254,11 @@ public: return {}; } + std::vector getAsyncFDs() override + { + return {}; + } + void setSession(std::unique_ptr& session) override { } diff --git a/pdns/libssl.cc b/pdns/libssl.cc index 2c0e8be682..776580803b 100644 --- a/pdns/libssl.cc +++ b/pdns/libssl.cc @@ -8,9 +8,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -63,6 +65,7 @@ static void openssl_thread_cleanup() #endif /* (OPENSSL_VERSION_NUMBER < 0x1010000fL || (defined LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x2090100fL) */ static std::atomic s_users; +static LockGuarded>> s_engines; static int s_ticketsKeyIndex{-1}; static int s_countersIndex{-1}; static int s_keyLogIndex{-1}; @@ -104,6 +107,11 @@ void registerOpenSSLUser() void unregisterOpenSSLUser() { if (s_users.fetch_sub(1) == 1) { + for (auto& [name, engine] : *s_engines.lock()) { + ENGINE_finish(engine.get()); + engine.reset(); + } + s_engines.lock()->clear(); #if (OPENSSL_VERSION_NUMBER < 0x1010000fL || (defined LIBRESSL_VERSION_NUMBER && LIBRESSL_VERSION_NUMBER < 0x2090100fL)) ERR_free_strings(); @@ -119,6 +127,42 @@ void unregisterOpenSSLUser() } } +std::pair libssl_load_engine(const std::string& engineName, const std::optional& defaultString) +{ + if (s_users.load() == 0) { + /* We need to make sure that OpenSSL has been properly initialized before loading an engine. + This messes up our accounting a bit, so some memory might not be properly released when + the program exits when engines are in use. */ + registerOpenSSLUser(); + } + + auto engines = s_engines.lock(); + if (engines->count(engineName) > 0) { + return { false, "TLS engine already loaded" }; + } + + ENGINE* enginePtr = ENGINE_by_id(engineName.c_str()); + if (enginePtr == nullptr) { + return { false, "unable to load TLS engine" }; + } + + auto engine = std::unique_ptr(enginePtr, ENGINE_free); + enginePtr = nullptr; + + if (!ENGINE_init(engine.get())) { + return { false, "Unable to init TLS engine" }; + } + + if (defaultString) { + if (ENGINE_set_default_string(engine.get(), defaultString->c_str()) == 0) { + return { false, "error while setting the TLS engine default string" }; + } + } + + engines->insert({engineName, std::move(engine)}); + return { true, "" }; +} + void* libssl_get_ticket_key_callback_data(SSL* s) { SSL_CTX* sslCtx = SSL_get_SSL_CTX(s); @@ -706,12 +750,21 @@ std::unique_ptr libssl_init_server_context(const TLS SSL_CTX_sess_set_cache_size(ctx.get(), config.d_maxStoredSessions); } + long mode = 0; #ifdef SSL_MODE_RELEASE_BUFFERS if (config.d_releaseBuffers) { - SSL_CTX_set_mode(ctx.get(), SSL_MODE_RELEASE_BUFFERS); + mode |= SSL_MODE_RELEASE_BUFFERS; } #endif +#ifdef SSL_MODE_ASYNC + if (config.d_asyncMode) { + mode |= SSL_MODE_ASYNC; + } +#endif + + SSL_CTX_set_mode(ctx.get(), mode); + /* we need to set this callback to acknowledge the server name sent by the client, otherwise it will not stored in the session and will not be accessible when the session is resumed, causing SSL_get_servername to return nullptr */ diff --git a/pdns/libssl.hh b/pdns/libssl.hh index 4561b4a496..db12751221 100644 --- a/pdns/libssl.hh +++ b/pdns/libssl.hh @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,8 @@ public: bool d_releaseBuffers{true}; /* whether so-called secure renegotiation should be allowed for TLS < 1.3 */ bool d_enableRenegotiation{false}; + /* enable TLS async mode, if supported by any engine */ + bool d_asyncMode{false}; }; struct TLSErrorCounters @@ -135,4 +138,6 @@ bool libssl_set_alpn_protos(SSL_CTX* ctx, const std::vector std::string libssl_get_error_string(); +std::pair libssl_load_engine(const std::string& engineName, const std::optional& defaultString); + #endif /* HAVE_LIBSSL */ diff --git a/pdns/tcpiohandler.cc b/pdns/tcpiohandler.cc index 143d93d878..a716932f5f 100644 --- a/pdns/tcpiohandler.cc +++ b/pdns/tcpiohandler.cc @@ -144,6 +144,28 @@ public: SSL_set_ex_data(d_conn.get(), s_tlsConnIndex, this); } + std::vector getAsyncFDs() override + { + if (SSL_waiting_for_async(d_conn.get()) != 1) { + return {}; + } + + OSSL_ASYNC_FD fds[32]; + size_t numfds = sizeof(fds)/sizeof(*fds); + SSL_get_all_async_fds(d_conn.get(), nullptr, &numfds); + if (numfds == 0) { + return {}; + } + + SSL_get_all_async_fds(d_conn.get(), fds, &numfds); + std::vector results; + results.reserve(numfds); + for (size_t idx = 0; idx < numfds; idx++) { + results.push_back(fds[idx]); + } + return results; + } + IOState convertIORequestToIOState(int res) const { int error = SSL_get_error(d_conn.get(), res); @@ -164,6 +186,9 @@ public: else if (error == SSL_ERROR_ZERO_RETURN) { throw std::runtime_error("TLS connection closed by remote end"); } + else if (error == SSL_ERROR_WANT_ASYNC) { + return IOState::Async; + } else { if (g_verbose) { throw std::runtime_error("Error while processing TLS connection: (" + std::to_string(error) + ") " + libssl_get_error_string()); @@ -1450,6 +1475,11 @@ public: return gnutls_alpn_set_protocols(d_conn.get(), values.data(), values.size(), flags); } + std::vector getAsyncFDs() override + { + return {}; + } + private: std::shared_ptr d_creds; std::shared_ptr d_ticketsKey; diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh index 5b23e00dc3..87822e091b 100644 --- a/pdns/tcpiohandler.hh +++ b/pdns/tcpiohandler.hh @@ -9,7 +9,7 @@ #include "misc.hh" #include "noinitvector.hh" -enum class IOState : uint8_t { Done, NeedRead, NeedWrite }; +enum class IOState : uint8_t { Done, NeedRead, NeedWrite, Async }; class TLSSession { @@ -39,6 +39,7 @@ public: virtual std::vector> getSessions() = 0; virtual void setSession(std::unique_ptr& session) = 0; virtual bool isUsable() const = 0; + virtual std::vector getAsyncFDs() = 0; virtual void close() = 0; void setUnknownTicketKey() @@ -547,6 +548,14 @@ public: return d_conn->isUsable(); } + std::vector getAsyncFDs() + { + if (!d_conn) { + return {}; + } + return d_conn->getAsyncFDs(); + } + const static bool s_disableConnectForUnitTests; private: