From: Matt Caswell Date: Fri, 9 Sep 2022 12:26:50 +0000 (+0100) Subject: Re-enable the multiblock code and move it into the record layer X-Git-Tag: openssl-3.2.0-alpha1~2024 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=23bf52a4b40deb033de0a257b724012afe32b169;p=thirdparty%2Fopenssl.git Re-enable the multiblock code and move it into the record layer Reviewed-by: Hugo Landau Reviewed-by: Richard Levitte (Merged from https://github.com/openssl/openssl/pull/19198) --- diff --git a/ssl/record/methods/recmethod_local.h b/ssl/record/methods/recmethod_local.h index 7c094b3a161..334a093362a 100644 --- a/ssl/record/methods/recmethod_local.h +++ b/ssl/record/methods/recmethod_local.h @@ -294,7 +294,7 @@ int tls_processed_read_pending(OSSL_RECORD_LAYER *rl); size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl); int tls_write_pending(OSSL_RECORD_LAYER *rl); size_t tls_get_max_record_len(OSSL_RECORD_LAYER *rl); -size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t buflen, +size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len, size_t maxfrag, size_t *preffrag); int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, size_t numtempl); diff --git a/ssl/record/methods/tls_common.c b/ssl/record/methods/tls_common.c index 32e26008006..6e48bb3a7cf 100644 --- a/ssl/record/methods/tls_common.c +++ b/ssl/record/methods/tls_common.c @@ -18,6 +18,15 @@ #include "../record_local.h" #include "recmethod_local.h" +#if defined(OPENSSL_SMALL_FOOTPRINT) || \ + !( defined(AES_ASM) && ( \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(_M_AMD64) || defined(_M_X64) ) \ + ) +# undef EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK +# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 +#endif + static void tls_int_free(OSSL_RECORD_LAYER *rl); void ossl_rlayer_fatal(OSSL_RECORD_LAYER *rl, int al, int reason, @@ -95,6 +104,25 @@ static int tls_allow_compression(OSSL_RECORD_LAYER *rl) } #endif +static void tls_release_write_buffer_int(OSSL_RECORD_LAYER *rl, size_t start) +{ + SSL3_BUFFER *wb; + size_t pipes; + + pipes = rl->numwpipes; + + while (pipes > start) { + wb = &rl->wbuf[pipes - 1]; + + if (SSL3_BUFFER_is_app_buffer(wb)) + SSL3_BUFFER_set_app_buffer(wb, 0); + else + OPENSSL_free(wb->buf); + wb->buf = NULL; + pipes--; + } +} + static int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, size_t len) { @@ -104,8 +132,6 @@ static int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, size_t currpipe; SSL_CONNECTION *s = (SSL_CONNECTION *)rl->cbarg; - rl->numwpipes = numwpipes; - if (len == 0) { if (rl->isdtls) headerlen = DTLS1_RT_HEADER_LENGTH + 1; @@ -139,7 +165,8 @@ static int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, if (s->wbio == NULL || !BIO_get_ktls_send(s->wbio)) { p = OPENSSL_malloc(len); if (p == NULL) { - rl->numwpipes = currpipe; + if (rl->numwpipes < currpipe) + rl->numwpipes = currpipe; /* * We've got a malloc failure, and we're still initialising * buffers. We assume we're so doomed that we won't even be able @@ -157,26 +184,17 @@ static int tls_setup_write_buffer(OSSL_RECORD_LAYER *rl, size_t numwpipes, } } + /* Free any previously allocated buffers that we are no longer using */ + tls_release_write_buffer_int(rl, currpipe); + + rl->numwpipes = numwpipes; + return 1; } static void tls_release_write_buffer(OSSL_RECORD_LAYER *rl) { - SSL3_BUFFER *wb; - size_t pipes; - - pipes = rl->numwpipes; - - while (pipes > 0) { - wb = &rl->wbuf[pipes - 1]; - - if (SSL3_BUFFER_is_app_buffer(wb)) - SSL3_BUFFER_set_app_buffer(wb, 0); - else - OPENSSL_free(wb->buf); - wb->buf = NULL; - pipes--; - } + tls_release_write_buffer_int(rl, 0); rl->numwpipes = 0; } @@ -1408,21 +1426,158 @@ size_t tls_get_max_record_len(OSSL_RECORD_LAYER *rl) return 0; } -size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t buflen, +static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, int type, + size_t len, size_t fraglen) +{ +#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + /* TODO(RECLAYER): REMOVE ME */ + SSL_CONNECTION *s = rl->cbarg; + + if (type == SSL3_RT_APPLICATION_DATA + && len >= 4 * fraglen + && s->compress == NULL + && rl->msg_callback == NULL + && !rl->use_etm + && RLAYER_USE_EXPLICIT_IV(rl) + && !BIO_get_ktls_send(s->wbio) + && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx)) + & EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) != 0) + return 1; +#endif + return 0; +} + +size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len, size_t maxfrag, size_t *preffrag) { + if (tls_is_multiblock_capable(rl, type, len, *preffrag)) { + /* minimize address aliasing conflicts */ + if ((*preffrag & 0xfff) == 0) + *preffrag -= 512; + + if (len >= 8 * (*preffrag)) + return 8; + + return 4; + } return 1; } -int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, - size_t numtempl) +static int tls_write_records_multiblock(OSSL_RECORD_LAYER *rl, + OSSL_RECORD_TEMPLATE *templates, + size_t numtempl) +{ +#if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK + size_t i; + size_t totlen; + SSL3_BUFFER *wb; + /* TODO(RECLAYER): Remove me */ + SSL_CONNECTION *s = rl->cbarg; + + if (numtempl != 4 && numtempl != 8) + return 0; + + /* + * Check templates have contiguous buffers and are all the same type and + * length + */ + for (i = 1; i < numtempl; i++) { + if (templates[i - 1].type != templates[i].type + || templates[i - 1].buflen != templates[i].buflen + || templates[i - 1].buf + templates[i - 1].buflen + != templates[i].buf) + return 0; + } + + totlen = templates[0].buflen * numtempl; + if (!tls_is_multiblock_capable(rl, templates[0].type, totlen, + templates[0].buflen)) + return 0; + + /* + * If we get this far, then multiblock is suitable + * Depending on platform multi-block can deliver several *times* + * better performance. Downside is that it has to allocate + * jumbo buffer to accommodate up to 8 records, but the + * compromise is considered worthy. + */ + + unsigned char aad[13]; + EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param; + size_t packlen; + int packleni; + + /* + * Allocate jumbo buffer. This will get freed next time we do a non + * multiblock write in the call to tls_setup_write_buffer() - the different + * buffer sizes will be spotted and the buffer reallocated. + */ + packlen = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE, + (int)templates[0].buflen, NULL); + packlen *= numtempl; + if (!tls_setup_write_buffer(rl, 1, packlen)) { + /* RLAYERfatal() already called */ + return -1; + } + wb = &rl->wbuf[0]; + + mb_param.interleave = numtempl; + memcpy(aad, s->rlayer.write_sequence, 8); + aad[8] = templates[0].type; + aad[9] = (unsigned char)(templates[0].version >> 8); + aad[10] = (unsigned char)(templates[0].version); + aad[11] = 0; + aad[12] = 0; + mb_param.out = NULL; + mb_param.inp = aad; + mb_param.len = totlen; + + packleni = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_AAD, + sizeof(mb_param), &mb_param); + packlen = (size_t)packleni; + if (packleni <= 0 || packlen > wb->len) { /* never happens */ + RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + + mb_param.out = wb->buf; + mb_param.inp = templates[0].buf; + mb_param.len = totlen; + + if (EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, + EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT, + sizeof(mb_param), &mb_param) <= 0) { + RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_INTERNAL_ERROR); + return -1; + } + + s->rlayer.write_sequence[7] += mb_param.interleave; + if (s->rlayer.write_sequence[7] < mb_param.interleave) { + int j = 6; + while (j >= 0 && (++s->rlayer.write_sequence[j--]) == 0) ; + } + + wb->offset = 0; + wb->left = packlen; + + return 1; +#else /* !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK */ + return 0; +#endif +} + +static int tls_write_records_standard(OSSL_RECORD_LAYER *rl, + OSSL_RECORD_TEMPLATE *templates, + size_t numtempl) { WPACKET pkt[SSL_MAX_PIPELINES + 1]; SSL3_RECORD wr[SSL_MAX_PIPELINES + 1]; WPACKET *thispkt; SSL3_RECORD *thiswr; unsigned char *recordstart; - int mac_size, clear = 0; + int mac_size, clear = 0, ret = 0; int eivlen = 0; size_t align = 0; SSL3_BUFFER *wb; @@ -1436,13 +1591,6 @@ int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, OSSL_RECORD_TEMPLATE prefixtempl; OSSL_RECORD_TEMPLATE *thistempl; - /* Check we don't have pending data waiting to write */ - if (!ossl_assert(rl->nextwbuf >= rl->numwpipes - || SSL3_BUFFER_get_left(&rl->wbuf[rl->nextwbuf]) == 0)) { - RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); - goto err; - } - sess = s->session; if ((sess == NULL) @@ -1467,15 +1615,13 @@ int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, && !clear && templates[0].type == SSL3_RT_APPLICATION_DATA; - if (rl->numwpipes < numtempl + prefix) { - /* - * TODO(RECLAYER): In the prefix case the first buffer can be a lot - * smaller. It is wasteful to allocate a full sized buffer here - */ - if (!tls_setup_write_buffer(rl, numtempl + prefix, 0)) { - /* RLAYERfatal() already called */ - goto err; - } + /* + * TODO(RECLAYER): In the prefix case the first buffer can be a lot + * smaller. It is wasteful to allocate a full sized buffer here + */ + if (!tls_setup_write_buffer(rl, numtempl + prefix, 0)) { + /* RLAYERfatal() already called */ + goto err; } using_ktls = BIO_get_ktls_send(rl->bio); @@ -1848,13 +1994,41 @@ int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, SSL3_BUFFER_set_left(&rl->wbuf[j], SSL3_RECORD_get_length(thiswr)); } - rl->nextwbuf = 0; - /* we now just need to write the buffers */ - return tls_retry_write_records(rl); + ret = 1; err: for (j = 0; j < wpinited; j++) WPACKET_cleanup(&pkt[j]); - return OSSL_RECORD_RETURN_FATAL; + return ret; +} + +int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates, + size_t numtempl) +{ + int ret; + + /* Check we don't have pending data waiting to write */ + if (!ossl_assert(rl->nextwbuf >= rl->numwpipes + || SSL3_BUFFER_get_left(&rl->wbuf[rl->nextwbuf]) == 0)) { + RLAYERfatal(rl, SSL_AD_INTERNAL_ERROR, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED); + return OSSL_RECORD_RETURN_FATAL; + } + + ret = tls_write_records_multiblock(rl, templates, numtempl); + if (ret < 0) { + /* RLAYERfatal already called */ + return OSSL_RECORD_RETURN_FATAL; + } + if (ret == 0) { + /* Multiblock wasn't suitable so just do a standard write */ + if (!tls_write_records_standard(rl, templates, numtempl)) { + /* RLAYERfatal already called */ + return OSSL_RECORD_RETURN_FATAL; + } + } + + rl->nextwbuf = 0; + /* we now just need to write the buffers */ + return tls_retry_write_records(rl); } int tls_retry_write_records(OSSL_RECORD_LAYER *rl) diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c index 9f81a9f0466..d3c83701a51 100644 --- a/ssl/record/rec_layer_s3.c +++ b/ssl/record/rec_layer_s3.c @@ -18,15 +18,6 @@ #include "record_local.h" #include "internal/packet.h" -#if defined(OPENSSL_SMALL_FOOTPRINT) || \ - !( defined(AES_ASM) && ( \ - defined(__x86_64) || defined(__x86_64__) || \ - defined(_M_AMD64) || defined(_M_X64) ) \ - ) -# undef EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK -# define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0 -#endif - void RECORD_LAYER_init(RECORD_LAYER *rl, SSL_CONNECTION *s) { rl->s = s; @@ -285,141 +276,7 @@ int ssl3_write_bytes(SSL *ssl, int type, const void *buf_, size_t len, s->rlayer.wpend_ret = len; } -/* TODO(RECLAYER): Re-enable multiblock code */ -#if 0 && !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK - /* - * Depending on platform multi-block can deliver several *times* - * better performance. Downside is that it has to allocate - * jumbo buffer to accommodate up to 8 records, but the - * compromise is considered worthy. - */ - if (type == SSL3_RT_APPLICATION_DATA - && len >= 4 * (max_send_fragment = ssl_get_max_send_fragment(s)) - && s->compress == NULL - && s->msg_callback == NULL - && !SSL_WRITE_ETM(s) - && SSL_USE_EXPLICIT_IV(s) - && !BIO_get_ktls_send(s->wbio) - && (EVP_CIPHER_get_flags(EVP_CIPHER_CTX_get0_cipher(s->enc_write_ctx)) - & EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK) != 0) { - unsigned char aad[13]; - EVP_CTRL_TLS1_1_MULTIBLOCK_PARAM mb_param; - size_t packlen; - int packleni; - - /* minimize address aliasing conflicts */ - if ((max_send_fragment & 0xfff) == 0) - max_send_fragment -= 512; - - if (tot == 0 || wb->buf == NULL) { /* allocate jumbo buffer */ - ssl3_release_write_buffer(s); - - packlen = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE, - (int)max_send_fragment, NULL); - - if (len >= 8 * max_send_fragment) - packlen *= 8; - else - packlen *= 4; - - if (!ssl3_setup_write_buffer(s, 1, packlen)) { - /* SSLfatal() already called */ - return -1; - } - } else if (tot == len) { /* done? */ - /* free jumbo buffer */ - ssl3_release_write_buffer(s); - *written = tot; - return 1; - } - - n = (len - tot); - for (;;) { - if (n < 4 * max_send_fragment) { - /* free jumbo buffer */ - ssl3_release_write_buffer(s); - break; - } - - if (s->s3.alert_dispatch) { - i = ssl->method->ssl_dispatch_alert(ssl); - if (i <= 0) { - /* SSLfatal() already called if appropriate */ - s->rlayer.wnum = tot; - return i; - } - } - - if (n >= 8 * max_send_fragment) - nw = max_send_fragment * (mb_param.interleave = 8); - else - nw = max_send_fragment * (mb_param.interleave = 4); - - memcpy(aad, s->rlayer.write_sequence, 8); - aad[8] = type; - aad[9] = (unsigned char)(s->version >> 8); - aad[10] = (unsigned char)(s->version); - aad[11] = 0; - aad[12] = 0; - mb_param.out = NULL; - mb_param.inp = aad; - mb_param.len = nw; - - packleni = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_AAD, - sizeof(mb_param), &mb_param); - packlen = (size_t)packleni; - if (packleni <= 0 || packlen > wb->len) { /* never happens */ - /* free jumbo buffer */ - ssl3_release_write_buffer(s); - break; - } - - mb_param.out = wb->buf; - mb_param.inp = &buf[tot]; - mb_param.len = nw; - - if (EVP_CIPHER_CTX_ctrl(s->enc_write_ctx, - EVP_CTRL_TLS1_1_MULTIBLOCK_ENCRYPT, - sizeof(mb_param), &mb_param) <= 0) - return -1; - - s->rlayer.write_sequence[7] += mb_param.interleave; - if (s->rlayer.write_sequence[7] < mb_param.interleave) { - int j = 6; - while (j >= 0 && (++s->rlayer.write_sequence[j--]) == 0) ; - } - - wb->offset = 0; - wb->left = packlen; - - s->rlayer.wpend_tot = nw; - s->rlayer.wpend_buf = &buf[tot]; - s->rlayer.wpend_type = type; - s->rlayer.wpend_ret = nw; - - i = ssl3_write_pending(s, type, &buf[tot], nw, &tmpwrit); - if (i <= 0) { - /* SSLfatal() already called if appropriate */ - if (i < 0 && (!s->wbio || !BIO_should_retry(s->wbio))) { - /* free jumbo buffer */ - ssl3_release_write_buffer(s); - } - s->rlayer.wnum = tot; - return i; - } - if (tmpwrit == n) { - /* free jumbo buffer */ - ssl3_release_write_buffer(s); - *written = tot + tmpwrit; - return 1; - } - n -= tmpwrit; - tot += tmpwrit; - } - } else -#endif /* !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK */ + /* TODO(RECLAYER): Is this needed any more? */ if (tot == len) { /* done? */ *written = tot; return 1; diff --git a/ssl/record/recordmethod.h b/ssl/record/recordmethod.h index dad086ea817..43c1cee578b 100644 --- a/ssl/record/recordmethod.h +++ b/ssl/record/recordmethod.h @@ -179,7 +179,7 @@ struct ossl_record_method_st { * to process in a single call to write_records. It is the caller's * responsibility to ensure that no call to write_records exceeds this * number of records. |type| is the type of the records that the caller - * wants to write, and |buflen| is the total amount of data that it wants + * wants to write, and |len| is the total amount of data that it wants * to send. |maxfrag| is the maximum allowed fragment size based on user * configuration, or TLS parameter negotiation. |*preffrag| contains on * entry the default fragment size that will actually be used based on user @@ -187,7 +187,7 @@ struct ossl_record_method_st { * exit the record layer may update this to an alternative fragment size to * be used. This must always be less than or equal to |maxfrag|. */ - size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t buflen, + size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t len, size_t maxfrag, size_t *preffrag); /*