From: Maxim Mikityanskiy Date: Wed, 9 Nov 2022 09:26:11 +0000 (+0200) Subject: Add support for KTLS zerocopy sendfile on Linux X-Git-Tag: openssl-3.2.0-alpha1~1677 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cd715b7e7fdd2aeb0fd80220d2df5187b291f87a;p=thirdparty%2Fopenssl.git Add support for KTLS zerocopy sendfile on Linux TLS device offload allows to perform zerocopy sendfile transmissions. FreeBSD provides this feature by default, and Linux 5.19 introduced it as an opt-in. Zerocopy improves the TX rate significantly, but has a side effect: if the underlying file is changed while being transmitted, and a TCP retransmission happens, the receiver may get a TLS record containing both new and old data, which leads to an authentication failure and termination of connection. This effect is the reason Linux makes a copy on sendfile by default. This commit adds support for TLS zerocopy sendfile on Linux disabled by default to avoid any unlikely backward compatibility issues on Linux, although sacrificing consistency in OpenSSL's behavior on Linux and FreeBSD. A new option called KTLSTxZerocopySendfile is added to enable the new zerocopy behavior on Linux. This option should be used when the the application guarantees that the file is not modified during transmission, or it doesn't care about breaking the connection. The related documentation is also added in this commit. The unit test added doesn't test the actual functionality (it would require specific hardware and a non-local peer), but solely checks that it's possible to set the new option flag. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Reviewed-by: Boris Pismenny Reviewed-by: Matt Caswell Reviewed-by: Todd Short Reviewed-by: Tomas Mraz (Merged from https://github.com/openssl/openssl/pull/18650) --- diff --git a/CHANGES.md b/CHANGES.md index 2c12daf151f..ede13f7d79e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -226,6 +226,10 @@ OpenSSL 3.2 *Tianjia Zhang* + * Zerocopy KTLS sendfile() support on Linux. + + *Maxim Mikityanskiy* + OpenSSL 3.0 ----------- diff --git a/apps/s_server.c b/apps/s_server.c index f519505ade0..46af6b87dad 100644 --- a/apps/s_server.c +++ b/apps/s_server.c @@ -96,6 +96,7 @@ static int keymatexportlen = 20; static int async = 0; static int use_sendfile = 0; +static int use_zc_sendfile = 0; static const char *session_id_prefix = NULL; @@ -716,6 +717,7 @@ typedef enum OPTION_choice { OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA, OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG, OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS, + OPT_USE_ZC_SENDFILE, OPT_TFO, OPT_CERT_COMP, OPT_R_ENUM, OPT_S_ENUM, @@ -966,6 +968,7 @@ const OPTIONS s_server_options[] = { #ifndef OPENSSL_NO_KTLS {"ktls", OPT_KTLS, '-', "Enable Kernel TLS for sending and receiving"}, {"sendfile", OPT_SENDFILE, '-', "Use sendfile to response file with -WWW"}, + {"zerocopy_sendfile", OPT_USE_ZC_SENDFILE, '-', "Use zerocopy mode of KTLS sendfile"}, #endif OPT_R_OPTIONS, @@ -1080,6 +1083,7 @@ int s_server_main(int argc, char *argv[]) s_brief = 0; async = 0; use_sendfile = 0; + use_zc_sendfile = 0; port = OPENSSL_strdup(PORT); cctx = SSL_CONF_CTX_new(); @@ -1654,6 +1658,11 @@ int s_server_main(int argc, char *argv[]) case OPT_SENDFILE: #ifndef OPENSSL_NO_KTLS use_sendfile = 1; +#endif + break; + case OPT_USE_ZC_SENDFILE: +#ifndef OPENSSL_NO_KTLS + use_zc_sendfile = 1; #endif break; case OPT_IGNORE_UNEXPECTED_EOF: @@ -1728,6 +1737,11 @@ int s_server_main(int argc, char *argv[]) #endif #ifndef OPENSSL_NO_KTLS + if (use_zc_sendfile && !use_sendfile) { + BIO_printf(bio_out, "Warning: -zerocopy_sendfile depends on -sendfile, enabling -sendfile now.\n"); + use_sendfile = 1; + } + if (use_sendfile && enable_ktls == 0) { BIO_printf(bio_out, "Warning: -sendfile depends on -ktls, enabling -ktls now.\n"); enable_ktls = 1; @@ -1933,6 +1947,8 @@ int s_server_main(int argc, char *argv[]) #ifndef OPENSSL_NO_KTLS if (enable_ktls) SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS); + if (use_zc_sendfile) + SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE); #endif if (max_send_fragment > 0 diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c index 75bfe645662..f494b14000f 100644 --- a/crypto/bio/bss_conn.c +++ b/crypto/bio/bss_conn.c @@ -598,6 +598,11 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; + case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: + ret = ktls_enable_tx_zerocopy_sendfile(b->num); + if (ret) + BIO_set_ktls_zerocopy_sendfile_flag(b); + break; # endif default: ret = 0; diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c index 69dfd37bfe1..f64eb8c843a 100644 --- a/crypto/bio/bss_sock.c +++ b/crypto/bio/bss_sock.c @@ -235,6 +235,11 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr) BIO_clear_ktls_ctrl_msg_flag(b); ret = 0; break; + case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE: + ret = ktls_enable_tx_zerocopy_sendfile(b->num); + if (ret) + BIO_set_ktls_zerocopy_sendfile_flag(b); + break; # endif case BIO_CTRL_EOF: ret = (b->flags & BIO_FLAGS_IN_EOF) != 0; diff --git a/doc/man1/openssl-s_server.pod.in b/doc/man1/openssl-s_server.pod.in index 94f3b4b46ce..a1e354908c9 100644 --- a/doc/man1/openssl-s_server.pod.in +++ b/doc/man1/openssl-s_server.pod.in @@ -132,6 +132,7 @@ B B [B<-alpn> I] [B<-ktls>] [B<-sendfile>] +[B<-zerocopy_sendfile>] [B<-keylogfile> I] [B<-recv_max_early_data> I] [B<-max_early_data> I] @@ -792,6 +793,15 @@ instead of BIO_write() to send the HTTP response requested by a client. This option is only valid when B<-ktls> along with B<-WWW> or B<-HTTP> are specified. +=item B<-zerocopy_sendfile> + +If this option is set, SSL_sendfile() will use the zerocopy TX mode, which gives +a performance boost when used with KTLS hardware offload. Note that invalid +TLS records might be transmitted if the file is changed while being sent. +This option depends on B<-sendfile>; when used alone, B<-sendfile> is implied, +and a warning is shown. Note that KTLS sendfile on FreeBSD always runs in the +zerocopy mode. + =item B<-keylogfile> I Appends TLS secrets to the specified keylog file such that external programs diff --git a/doc/man3/SSL_CONF_cmd.pod b/doc/man3/SSL_CONF_cmd.pod index c20df37e3b2..3717c202bd6 100644 --- a/doc/man3/SSL_CONF_cmd.pod +++ b/doc/man3/SSL_CONF_cmd.pod @@ -561,6 +561,14 @@ B: support receiving compressed certificates, enabled default. Inverse of B: that is, B<-RxCertificateCompression> is the same as setting B. +B: use the zerocopy TX mode of sendfile(), which gives +a performance boost when used with KTLS hardware offload. Note that invalid TLS +records might be transmitted if the file is changed while being sent. This +option has no effect if B is not enabled. Equivalent to +B. This option only applies to Linux. +KTLS sendfile on FreeBSD doesn't offer an option to disable zerocopy and +always runs in this mode. + =item B The B argument is a comma separated list of flags to set. diff --git a/doc/man3/SSL_CTX_set_options.pod b/doc/man3/SSL_CTX_set_options.pod index 1da057adb83..b72973f8d06 100644 --- a/doc/man3/SSL_CTX_set_options.pod +++ b/doc/man3/SSL_CTX_set_options.pod @@ -175,6 +175,16 @@ by the kernel directly and not via any available OpenSSL Providers. This might be undesirable if, for example, the application requires all cryptographic operations to be performed by the FIPS provider. +=item SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE + +With this option, sendfile() will use the zerocopy mode, which gives a +performance boost when used with KTLS hardware offload. Note that invalid TLS +records might be transmitted if the file is changed while being sent. This +option has no effect if B is not enabled. + +This option only applies to Linux. KTLS sendfile on FreeBSD doesn't offer an +option to disable zerocopy and always runs in this mode. + =item SSL_OP_ENABLE_MIDDLEBOX_COMPAT If set then dummy Change Cipher Spec (CCS) messages are sent in TLSv1.3. This diff --git a/include/internal/bio.h b/include/internal/bio.h index 40218e1fb05..9481f4c985e 100644 --- a/include/internal/bio.h +++ b/include/internal/bio.h @@ -43,16 +43,20 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); # define BIO_CTRL_SET_KTLS 72 # define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG 74 # define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG 75 +# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 /* * This is used with socket BIOs: * BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending. * BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next. * BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving. + * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with + * this BIO for sending using sendfile. */ # define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000 # define BIO_FLAGS_KTLS_RX 0x2000 # define BIO_FLAGS_KTLS_TX 0x4000 +# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000 /* KTLS related controls and flags */ # define BIO_set_ktls_flag(b, is_tx) \ @@ -65,6 +69,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) # define BIO_clear_ktls_ctrl_msg_flag(b) \ BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG) +# define BIO_set_ktls_zerocopy_sendfile_flag(b) \ + BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE) # define BIO_set_ktls(b, keyblob, is_tx) \ BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob) @@ -72,6 +78,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read); BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL) # define BIO_clear_ktls_ctrl_msg(b) \ BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL) +# define BIO_set_ktls_tx_zerocopy_sendfile(b) \ + BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL) /* Functions to allow the core to offer the CORE_BIO type to providers */ OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio); diff --git a/include/internal/ktls.h b/include/internal/ktls.h index efa4451d7c8..af27a325697 100644 --- a/include/internal/ktls.h +++ b/include/internal/ktls.h @@ -214,6 +214,13 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off, # warning "Skipping Compilation of KTLS receive data path" # endif # endif +# if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0) +# define OPENSSL_NO_KTLS_ZC_TX +# ifndef PEDANTIC +# warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile" +# warning "Skipping Compilation of KTLS zerocopy sendfile" +# endif +# endif # define OPENSSL_KTLS_AES_GCM_128 # if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0) # define OPENSSL_KTLS_AES_GCM_256 @@ -293,6 +300,18 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info, crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1; } +static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd) +{ +#ifndef OPENSSL_NO_KTLS_ZC_TX + int enable = 1; + + return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO, + &enable, sizeof(enable)) ? 0 : 1; +#else + return 0; +#endif +} + /* * Send a TLS record using the crypto_info provided in ktls_start and use * record_type instead of the default SSL3_RT_APPLICATION_DATA. diff --git a/include/openssl/bio.h.in b/include/openssl/bio.h.in index 6105c602fd8..89c88c67e79 100644 --- a/include/openssl/bio.h.in +++ b/include/openssl/bio.h.in @@ -182,6 +182,11 @@ extern "C" { # define BIO_CTRL_DGRAM_GET_NO_TRUNC 88 # define BIO_CTRL_DGRAM_SET_NO_TRUNC 89 +/* + * internal BIO: + * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90 + */ + # define BIO_DGRAM_CAP_NONE 0U # define BIO_DGRAM_CAP_HANDLES_SRC_ADDR (1U << 0) # define BIO_DGRAM_CAP_HANDLES_DST_ADDR (1U << 1) @@ -225,7 +230,7 @@ extern "C" { # define BIO_FLAGS_NONCLEAR_RST 0x400 # define BIO_FLAGS_IN_EOF 0x800 -/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */ +/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */ typedef union bio_addr_st BIO_ADDR; typedef struct bio_addrinfo_st BIO_ADDRINFO; diff --git a/include/openssl/ssl.h.in b/include/openssl/ssl.h.in index 2224b3269b6..871ad265c54 100644 --- a/include/openssl/ssl.h.in +++ b/include/openssl/ssl.h.in @@ -420,6 +420,8 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg); */ # define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION SSL_OP_BIT(32) # define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION SSL_OP_BIT(33) + /* Enable KTLS TX zerocopy on Linux */ +# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE SSL_OP_BIT(34) /* * Option "collections." diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c index 5c94837dc0b..2f9b11a512a 100644 --- a/ssl/record/methods/ktls_meth.c +++ b/ssl/record/methods/ktls_meth.c @@ -334,6 +334,14 @@ static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level, if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction)) return OSSL_RECORD_RETURN_NON_FATAL_ERR; + if (rl->direction == OSSL_RECORD_DIRECTION_WRITE && + (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0) + /* Ignore errors. The application opts in to using the zerocopy + * optimization. If the running kernel doesn't support it, just + * continue without the optimization. + */ + BIO_set_ktls_tx_zerocopy_sendfile(rl->bio); + return OSSL_RECORD_RETURN_SUCCESS; } diff --git a/ssl/ssl_conf.c b/ssl/ssl_conf.c index bebfc501a91..0bea29fb66c 100644 --- a/ssl/ssl_conf.c +++ b/ssl/ssl_conf.c @@ -400,6 +400,7 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value) SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT), SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION), SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION), + SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE), }; if (value == NULL) return -3; diff --git a/test/sslapitest.c b/test/sslapitest.c index 8f14381b56f..a26f6286f30 100644 --- a/test/sslapitest.c +++ b/test/sslapitest.c @@ -1293,7 +1293,8 @@ end: #define SENDFILE_CHUNK (4 * 4096) #define min(a,b) ((a) > (b) ? (b) : (a)) -static int execute_test_ktls_sendfile(int tls_version, const char *cipher) +static int execute_test_ktls_sendfile(int tls_version, const char *cipher, + int zerocopy) { SSL_CTX *cctx = NULL, *sctx = NULL; SSL *clientssl = NULL, *serverssl = NULL; @@ -1350,6 +1351,12 @@ static int execute_test_ktls_sendfile(int tls_version, const char *cipher) if (!TEST_true(SSL_set_options(serverssl, SSL_OP_ENABLE_KTLS))) goto end; + if (zerocopy) { + if (!TEST_true(SSL_set_options(serverssl, + SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE))) + goto end; + } + if (!TEST_true(create_ssl_connection(serverssl, clientssl, SSL_ERROR_NONE))) goto end; @@ -1480,14 +1487,16 @@ static int test_ktls(int test) cipher->cipher); } -static int test_ktls_sendfile(int tst) +static int test_ktls_sendfile(int test) { struct ktls_test_cipher *cipher; + int tst = test >> 1; OPENSSL_assert(tst < (int)NUM_KTLS_TEST_CIPHERS); cipher = &ktls_test_ciphers[tst]; - return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher); + return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher, + test & 1); } #endif @@ -10544,7 +10553,7 @@ int setup_tests(void) #if !defined(OPENSSL_NO_KTLS) && !defined(OPENSSL_NO_SOCK) # if !defined(OPENSSL_NO_TLS1_2) || !defined(OSSL_NO_USABLE_TLS1_3) ADD_ALL_TESTS(test_ktls, NUM_KTLS_TEST_CIPHERS * 4); - ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS); + ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS * 2); # endif #endif ADD_TEST(test_large_message_tls);