From: Artem Boldariev <artem@boldariev.com>
Date: Mon, 5 Dec 2022 18:19:03 +0000 (+0200)
Subject: TLS: try to avoid allocating send request objects
X-Git-Tag: v9.19.9~68^2~7
X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=56732ac2a0c38e726c2beba4e8acf33226bf7739;p=thirdparty%2Fbind9.git

TLS: try to avoid allocating send request objects

This commit optimises TLS send request object allocation to enable
send request object reuse, somewhat reducing pressure on the memory
manager. It is especially helpful in the case when Stream DNS uses the
TLS implementation as the transport.
---

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index 4fe9e1c0c57..bd8d8449892 100644
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -878,6 +878,7 @@ struct isc_nmsocket {
 		} state; /*%< The order of these is significant */
 		size_t nsending;
 		bool tcp_nodelay_value;
+		isc_nmsocket_tls_send_req_t *send_req; /*%< Send req to reuse */
 	} tlsstream;
 
 #if HAVE_LIBNGHTTP2
diff --git a/lib/isc/netmgr/tlsstream.c b/lib/isc/netmgr/tlsstream.c
index 05d18e30bf5..67298cf8eb7 100644
--- a/lib/isc/netmgr/tlsstream.c
+++ b/lib/isc/netmgr/tlsstream.c
@@ -131,6 +131,9 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 		(isc_nmsocket_tls_send_req_t *)cbarg;
 	isc_nmsocket_t *tlssock = NULL;
 	bool finish = send_req->finish;
+	isc_nm_cb_t send_cb = NULL;
+	void *send_cbarg = NULL;
+	isc_nmhandle_t *send_handle = NULL;
 
 	REQUIRE(VALID_NMHANDLE(handle));
 	REQUIRE(VALID_NMSOCK(handle->sock));
@@ -138,34 +141,52 @@ tls_senddone(isc_nmhandle_t *handle, isc_result_t eresult, void *cbarg) {
 
 	tlssock = send_req->tlssock;
 	send_req->tlssock = NULL;
+	send_cb = send_req->cb;
+	send_cbarg = send_req->cbarg;
+	send_handle = send_req->handle;
+	send_req->handle = NULL;
 
 	if (finish) {
 		tls_try_shutdown(tlssock->tlsstream.tls, true);
 	}
 
-	if (send_req->cb != NULL) {
-		INSIST(VALID_NMHANDLE(tlssock->statichandle));
-		send_req->cb(send_req->handle, eresult, send_req->cbarg);
-		isc_nmhandle_detach(&send_req->handle);
-		/* The last handle has been just detached: close the underlying
-		 * socket. */
-		if (tlssock->statichandle == NULL) {
-			finish = true;
-		}
-	}
-
-	/* We are tying to avoid a memory allocation for small write
+	/*
+	 * We are tying to avoid a memory allocation for small write
 	 * requests. See the mirroring code in the tls_send_outgoing()
-	 * function. */
+	 * function. The object is attempted to be freed or put for reuse
+	 * before the call to callback because there is a chance that it
+	 * is going to be reused during the call to the callback.
+	 */
 	if (send_req->data.length > sizeof(send_req->smallbuf)) {
 		isc_mem_put(handle->sock->worker->mctx, send_req->data.base,
 			    send_req->data.length);
 	} else {
 		INSIST(&send_req->smallbuf[0] == send_req->data.base);
 	}
-	isc_mem_put(handle->sock->worker->mctx, send_req, sizeof(*send_req));
+
+	send_req->data.base = NULL;
+	send_req->data.length = 0;
+
+	/* Try to keep the object to be reused later - to avoid an allocation */
+	if (tlssock->tlsstream.send_req == NULL) {
+		tlssock->tlsstream.send_req = send_req;
+	} else {
+		isc_mem_put(handle->sock->worker->mctx, send_req,
+			    sizeof(*send_req));
+	}
 	tlssock->tlsstream.nsending--;
 
+	if (send_cb != NULL) {
+		INSIST(VALID_NMHANDLE(tlssock->statichandle));
+		send_cb(send_handle, eresult, send_cbarg);
+		isc_nmhandle_detach(&send_handle);
+		/* The last handle has been just detached: close the underlying
+		 * socket. */
+		if (tlssock->statichandle == NULL) {
+			finish = true;
+		}
+	}
+
 	if (finish && eresult == ISC_R_SUCCESS && tlssock->reading) {
 		tls_failed_read_cb(tlssock, ISC_R_EOF);
 	} else if (eresult == ISC_R_SUCCESS) {
@@ -277,7 +298,14 @@ tls_send_outgoing(isc_nmsocket_t *sock, bool finish, isc_nmhandle_t *tlshandle,
 		pending = TLS_BUF_SIZE;
 	}
 
-	send_req = isc_mem_get(sock->worker->mctx, sizeof(*send_req));
+	/* Try to reuse previously allocated object */
+	if (sock->tlsstream.send_req != NULL) {
+		send_req = sock->tlsstream.send_req;
+		sock->tlsstream.send_req = NULL;
+	} else {
+		send_req = isc_mem_get(sock->worker->mctx, sizeof(*send_req));
+	}
+
 	*send_req = (isc_nmsocket_tls_send_req_t){ .finish = finish,
 						   .data.length = pending };
 
@@ -1181,6 +1209,14 @@ isc__nm_tls_cleanup_data(isc_nmsocket_t *sock) {
 			isc_tlsctx_client_session_cache_detach(
 				&sock->tlsstream.client_sess_cache);
 		}
+
+		if (sock->tlsstream.send_req != NULL) {
+			INSIST(sock->tlsstream.send_req->data.base == NULL);
+			INSIST(sock->tlsstream.send_req->data.length == 0);
+			isc_mem_put(sock->worker->mctx,
+				    sock->tlsstream.send_req,
+				    sizeof(*sock->tlsstream.send_req));
+		}
 	} else if (sock->type == isc_nm_tcpsocket &&
 		   sock->tlsstream.tlssocket != NULL)
 	{