Handle initialize_tls() failure gracefully in tlslisten_acceptcb()

initialize_tls() can fail due to non-memory OpenSSL errors such as
BIO_new() internal failures or SSL_set_tlsext_host_name() rejecting an
invalid hostname.  These are per-connection errors that should not
bring down the entire server.

Replace the RUNTIME_CHECK (which calls abort()) with proper error
handling: log the failure, clean up the TLS socket, and return an
error to the caller so the connection is rejected gracefully.

diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index 3a9ab1dfbf6cefcd54dcf1fbcf631a4e6520c879..143065597583b7d63e717b9cb69a6febb07d27e7 100644 (file)
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -951,8 +951,19 @@ tlslisten_acceptcb(isc_nmhandle_t *handle, isc_result_t result, void *cbarg) {
        handle->sock->tlsstream.tlssocket = tlssock;
 
        result = initialize_tls(tlssock, true);
-       RUNTIME_CHECK(result == ISC_R_SUCCESS);
-       /* TODO: catch failure code, detach tlssock, and log the error */
+       if (result != ISC_R_SUCCESS) {
+               isc__nmsocket_log(tlssock, ISC_LOG_ERROR,
+                                 "TLS initialization failed: %s",
+                                 isc_result_totext(result));
+               handle->sock->tlsstream.tlssocket = NULL;
+               isc_nmhandle_detach(&tlssock->outerhandle);
+               tlssock->closed = true;
+               isc_tlsctx_free(&tlssock->tlsstream.ctx);
+               isc__nmsocket_detach(&tlssock->listener);
+               isc__nmsocket_detach(&tlssock->server);
+               isc__nmsocket_detach(&tlssock);
+               return result;
+       }
 
        tls_try_to_enable_tcp_nodelay(tlssock);