2 * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #include "internal/quic_txp.h"
11 #include "internal/quic_fifd.h"
12 #include "internal/quic_stream_map.h"
13 #include "internal/quic_error.h"
14 #include "internal/common.h"
15 #include <openssl/err.h>
17 #define MIN_CRYPTO_HDR_SIZE 3
19 #define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
20 #define MIN_FRAME_SIZE_MAX_DATA 2
21 #define MIN_FRAME_SIZE_ACK 5
22 #define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
23 #define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
24 #define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
25 #define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
32 /* Generate normal packets containing most frame types, subject to EL. */
33 #define TX_PACKETISER_ARCHETYPE_NORMAL 0
36 * A probe packet is different in that:
37 * - It bypasses CC, but *is* counted as in flight for purposes of CC;
38 * - It must be ACK-eliciting.
40 #define TX_PACKETISER_ARCHETYPE_PROBE 1
43 * An ACK-only packet is different in that:
44 * - It bypasses CC, and is considered a 'non-inflight' packet;
45 * - It may not contain anything other than an ACK frame, not even padding.
47 #define TX_PACKETISER_ARCHETYPE_ACK_ONLY 2
49 #define TX_PACKETISER_ARCHETYPE_NUM 3
51 struct ossl_quic_tx_packetiser_st
{
52 OSSL_QUIC_TX_PACKETISER_ARGS args
;
55 * Opaque initial token blob provided by caller. TXP frees using the
56 * callback when it is no longer needed.
58 const unsigned char *initial_token
;
59 size_t initial_token_len
;
60 ossl_quic_initial_token_free_fn
*initial_token_free_cb
;
61 void *initial_token_free_cb_arg
;
63 /* Subcomponents of the TXP that we own. */
64 QUIC_FIFD fifd
; /* QUIC Frame-in-Flight Dispatcher */
67 uint64_t next_pn
[QUIC_PN_SPACE_NUM
]; /* Next PN to use in given PN space. */
68 OSSL_TIME last_tx_time
; /* Last time a packet was generated, or 0. */
70 /* Internal state - frame (re)generation flags. */
71 unsigned int want_handshake_done
: 1;
72 unsigned int want_max_data
: 1;
73 unsigned int want_max_streams_bidi
: 1;
74 unsigned int want_max_streams_uni
: 1;
76 /* Internal state - frame (re)generation flags - per PN space. */
77 unsigned int want_ack
: QUIC_PN_SPACE_NUM
;
78 unsigned int force_ack_eliciting
: QUIC_PN_SPACE_NUM
;
81 * Internal state - connection close terminal state.
82 * Once this is set, it is not unset unlike other want_ flags - we keep
83 * sending it in every packet.
85 unsigned int want_conn_close
: 1;
87 /* Has the handshake been completed? */
88 unsigned int handshake_complete
: 1;
90 OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame
;
93 * Counts of the number of bytes received and sent while in the closing
96 uint64_t closing_bytes_recv
;
97 uint64_t closing_bytes_xmit
;
99 /* Internal state - packet assembly. */
101 unsigned char *scratch
; /* scratch buffer for packet assembly */
102 size_t scratch_len
; /* number of bytes allocated for scratch */
103 OSSL_QTX_IOVEC
*iovec
; /* scratch iovec array for use with QTX */
104 size_t alloc_iovec
; /* size of iovec array */
105 } el
[QUIC_ENC_LEVEL_NUM
];
107 /* Message callback related arguments */
108 ossl_msg_cb msg_callback
;
109 void *msg_callback_arg
;
110 SSL
*msg_callback_ssl
;
113 void (*ack_tx_cb
)(const OSSL_QUIC_FRAME_ACK
*ack
,
120 * The TX helper records state used while generating frames into packets. It
121 * enables serialization into the packet to be done "transactionally" where
122 * serialization of a frame can be rolled back if it fails midway (e.g. if it
126 OSSL_QUIC_TX_PACKETISER
*txp
;
128 * The Maximum Packet Payload Length in bytes. This is the amount of
129 * space we have to generate frames into.
133 * Number of bytes we have generated so far.
135 size_t bytes_appended
;
137 * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
138 * will reference this scratch buffer. When we need to use more of it (e.g.
139 * when we need to put frame headers somewhere), we append to the scratch
140 * buffer, resizing if necessary, and increase this accordingly.
142 size_t scratch_bytes
;
144 * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
145 * until reserve_allowed is set to 1. Currently this is always at most 1, as
146 * a PING frame takes up one byte and this mechanism is only used to ensure
147 * we can encode a PING frame if we have been asked to ensure a packet is
148 * ACK-eliciting and we are unusure if we are going to add any other
149 * ACK-eliciting frames before we reach our MaxPPL budget.
153 * Number of iovecs we have currently appended. This is the number of
154 * entries valid in txp->iovec.
157 /* The EL this TX helper is being used for. */
160 * Whether we are allowed to make use of the reserve bytes in our MaxPPL
161 * budget. This is used to ensure we have room to append a PING frame later
162 * if we need to. Once we know we will not need to append a PING frame, this
165 unsigned int reserve_allowed
: 1;
167 * Set to 1 if we have appended a STREAM frame with an implicit length. If
168 * this happens we should never append another frame after that frame as it
169 * cannot be validly encoded. This is just a safety check.
171 unsigned int done_implicit
: 1;
174 * The fields in this structure are valid if active is set, which means
175 * that a serialization transaction is currently in progress.
179 unsigned int active
: 1;
183 static void tx_helper_rollback(struct tx_helper
*h
);
184 static int txp_el_ensure_iovec(struct txp_el
*el
, size_t num
);
186 /* Initialises the TX helper. */
187 static int tx_helper_init(struct tx_helper
*h
, OSSL_QUIC_TX_PACKETISER
*txp
,
188 uint32_t enc_level
, size_t max_ppl
, size_t reserve
)
190 if (reserve
> max_ppl
)
194 h
->enc_level
= enc_level
;
195 h
->max_ppl
= max_ppl
;
196 h
->reserve
= reserve
;
198 h
->bytes_appended
= 0;
199 h
->scratch_bytes
= 0;
200 h
->reserve_allowed
= 0;
201 h
->done_implicit
= 0;
205 if (max_ppl
> h
->txp
->el
[enc_level
].scratch_len
) {
206 unsigned char *scratch
;
208 scratch
= OPENSSL_realloc(h
->txp
->el
[enc_level
].scratch
, max_ppl
);
212 h
->txp
->el
[enc_level
].scratch
= scratch
;
213 h
->txp
->el
[enc_level
].scratch_len
= max_ppl
;
219 static void tx_helper_cleanup(struct tx_helper
*h
)
222 tx_helper_rollback(h
);
227 static void tx_helper_unrestrict(struct tx_helper
*h
)
229 h
->reserve_allowed
= 1;
233 * Append an extent of memory to the iovec list. The memory must remain
234 * allocated until we finish generating the packet and call the QTX.
236 * In general, the buffers passed to this function will be from one of two
239 * - Application data contained in stream buffers managed elsewhere
240 * in the QUIC stack; or
242 * - Control frame data appended into txp->scratch using tx_helper_begin and
246 static int tx_helper_append_iovec(struct tx_helper
*h
,
247 const unsigned char *buf
,
250 struct txp_el
*el
= &h
->txp
->el
[h
->enc_level
];
255 if (!ossl_assert(!h
->done_implicit
))
258 if (!txp_el_ensure_iovec(el
, h
->num_iovec
+ 1))
261 el
->iovec
[h
->num_iovec
].buf
= buf
;
262 el
->iovec
[h
->num_iovec
].buf_len
= buf_len
;
265 h
->bytes_appended
+= buf_len
;
270 * How many more bytes of space do we have left in our plaintext packet payload?
272 static size_t tx_helper_get_space_left(struct tx_helper
*h
)
275 - (h
->reserve_allowed
? 0 : h
->reserve
) - h
->bytes_appended
;
279 * Begin a control frame serialization transaction. This allows the
280 * serialization of the control frame to be backed out if it turns out it won't
281 * fit. Write the control frame to the returned WPACKET. Ensure you always
282 * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
285 static WPACKET
*tx_helper_begin(struct tx_helper
*h
)
287 size_t space_left
, len
;
289 struct txp_el
*el
= &h
->txp
->el
[h
->enc_level
];
291 if (!ossl_assert(!h
->txn
.active
))
294 if (!ossl_assert(!h
->done_implicit
))
297 data
= (unsigned char *)el
->scratch
+ h
->scratch_bytes
;
298 len
= el
->scratch_len
- h
->scratch_bytes
;
300 space_left
= tx_helper_get_space_left(h
);
301 if (!ossl_assert(space_left
<= len
))
304 if (!WPACKET_init_static_len(&h
->txn
.wpkt
, data
, len
, 0))
307 if (!WPACKET_set_max_size(&h
->txn
.wpkt
, space_left
)) {
308 WPACKET_cleanup(&h
->txn
.wpkt
);
317 static void tx_helper_end(struct tx_helper
*h
, int success
)
320 WPACKET_finish(&h
->txn
.wpkt
);
322 WPACKET_cleanup(&h
->txn
.wpkt
);
328 /* Abort a control frame serialization transaction. */
329 static void tx_helper_rollback(struct tx_helper
*h
)
337 /* Commit a control frame. */
338 static int tx_helper_commit(struct tx_helper
*h
)
345 if (!WPACKET_get_total_written(&h
->txn
.wpkt
, &l
)) {
350 if (!tx_helper_append_iovec(h
, h
->txn
.data
, l
)) {
355 if (h
->txp
->msg_callback
!= NULL
&& l
> 0) {
357 int ctype
= SSL3_RT_QUIC_FRAME_FULL
;
360 if (!PACKET_buf_init(&pkt
, h
->txn
.data
, l
)
361 || !ossl_quic_wire_peek_frame_header(&pkt
, &ftype
, NULL
)) {
366 if (ftype
== OSSL_QUIC_FRAME_TYPE_PADDING
)
367 ctype
= SSL3_RT_QUIC_FRAME_PADDING
;
368 else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(ftype
)
369 || ftype
== OSSL_QUIC_FRAME_TYPE_CRYPTO
)
370 ctype
= SSL3_RT_QUIC_FRAME_HEADER
;
372 h
->txp
->msg_callback(1, OSSL_QUIC1_VERSION
, ctype
, h
->txn
.data
, l
,
373 h
->txp
->msg_callback_ssl
,
374 h
->txp
->msg_callback_arg
);
377 h
->scratch_bytes
+= l
;
382 struct archetype_data
{
383 unsigned int allow_ack
: 1;
384 unsigned int allow_ping
: 1;
385 unsigned int allow_crypto
: 1;
386 unsigned int allow_handshake_done
: 1;
387 unsigned int allow_path_challenge
: 1;
388 unsigned int allow_path_response
: 1;
389 unsigned int allow_new_conn_id
: 1;
390 unsigned int allow_retire_conn_id
: 1;
391 unsigned int allow_stream_rel
: 1;
392 unsigned int allow_conn_fc
: 1;
393 unsigned int allow_conn_close
: 1;
394 unsigned int allow_cfq_other
: 1;
395 unsigned int allow_new_token
: 1;
396 unsigned int allow_force_ack_eliciting
: 1;
397 unsigned int allow_padding
: 1;
398 unsigned int require_ack_eliciting
: 1;
399 unsigned int bypass_cc
: 1;
402 struct txp_pkt_geom
{
403 size_t cmpl
, cmppl
, hwm
, pkt_overhead
;
405 struct archetype_data adata
;
411 QUIC_TXPIM_PKT
*tpkt
;
412 QUIC_STREAM
*stream_head
;
414 struct txp_pkt_geom geom
;
418 static QUIC_SSTREAM
*get_sstream_by_id(uint64_t stream_id
, uint32_t pn_space
,
420 static void on_regen_notify(uint64_t frame_type
, uint64_t stream_id
,
421 QUIC_TXPIM_PKT
*pkt
, void *arg
);
422 static void on_confirm_notify(uint64_t frame_type
, uint64_t stream_id
,
423 QUIC_TXPIM_PKT
*pkt
, void *arg
);
424 static void on_sstream_updated(uint64_t stream_id
, void *arg
);
425 static int sstream_is_pending(QUIC_SSTREAM
*sstream
);
426 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER
*txp
,
430 uint32_t *conn_close_enc_level
);
431 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER
*txp
);
432 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER
*txp
,
437 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER
*txp
);
438 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER
*txp
,
440 int chosen_for_conn_close
);
441 static int txp_pkt_init(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
,
442 uint32_t enc_level
, uint32_t archetype
,
443 size_t running_total
);
444 static void txp_pkt_cleanup(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
);
445 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt
*pkt
,
446 OSSL_QUIC_TX_PACKETISER
*txp
);
447 static int txp_pkt_append_padding(struct txp_pkt
*pkt
,
448 OSSL_QUIC_TX_PACKETISER
*txp
, size_t num_bytes
);
449 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER
*txp
, struct txp_pkt
*pkt
,
450 uint32_t archetype
, int *txpim_pkt_reffed
);
451 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER
*txp
,
454 OSSL_QUIC_TX_PACKETISER
*ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS
*args
)
456 OSSL_QUIC_TX_PACKETISER
*txp
;
460 || args
->txpim
== NULL
462 || args
->ackm
== NULL
464 || args
->conn_txfc
== NULL
465 || args
->conn_rxfc
== NULL
466 || args
->max_streams_bidi_rxfc
== NULL
467 || args
->max_streams_uni_rxfc
== NULL
) {
468 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
472 txp
= OPENSSL_zalloc(sizeof(*txp
));
477 txp
->last_tx_time
= ossl_time_zero();
479 if (!ossl_quic_fifd_init(&txp
->fifd
,
480 txp
->args
.cfq
, txp
->args
.ackm
, txp
->args
.txpim
,
481 get_sstream_by_id
, txp
,
482 on_regen_notify
, txp
,
483 on_confirm_notify
, txp
,
484 on_sstream_updated
, txp
,
493 void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER
*txp
)
500 ossl_quic_tx_packetiser_set_initial_token(txp
, NULL
, 0, NULL
, NULL
);
501 ossl_quic_fifd_cleanup(&txp
->fifd
);
502 OPENSSL_free(txp
->conn_close_frame
.reason
);
504 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
505 enc_level
< QUIC_ENC_LEVEL_NUM
;
507 OPENSSL_free(txp
->el
[enc_level
].iovec
);
508 OPENSSL_free(txp
->el
[enc_level
].scratch
);
515 * Determine if an Initial packet token length is reasonable based on the
516 * current MDPL, returning 1 if it is OK.
518 * The real PMTU to the peer could differ from our (pessimistic) understanding
519 * of the PMTU, therefore it is possible we could receive an Initial token from
520 * a server in a Retry packet which is bigger than the MDPL. In this case it is
521 * impossible for us ever to make forward progress and we need to error out
522 * and fail the connection attempt.
524 * The specific boundary condition is complex: for example, after the size of
525 * the Initial token, there are the Initial packet header overheads and then
526 * encryption/AEAD tag overheads. After that, the minimum room for frame data in
527 * order to guarantee forward progress must be guaranteed. For example, a crypto
528 * stream needs to always be able to serialize at least one byte in a CRYPTO
529 * frame in order to make forward progress. Because the offset field of a CRYPTO
530 * frame uses a variable-length integer, the number of bytes needed to ensure
533 * Rather than trying to get this boundary condition check actually right,
534 * require a reasonable amount of slack to avoid pathological behaviours. (After
535 * all, transmitting a CRYPTO stream one byte at a time is probably not
538 * We choose 160 bytes as the required margin, which is double the rough
539 * estimation of the minimum we would require to guarantee forward progress
540 * under worst case packet overheads.
542 #define TXP_REQUIRED_TOKEN_MARGIN 160
544 static int txp_check_token_len(size_t token_len
, size_t mdpl
)
549 if (token_len
>= mdpl
)
552 if (TXP_REQUIRED_TOKEN_MARGIN
>= mdpl
)
553 /* (should not be possible because MDPL must be at least 1200) */
556 if (token_len
> mdpl
- TXP_REQUIRED_TOKEN_MARGIN
)
562 int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER
*txp
,
563 const unsigned char *token
,
565 ossl_quic_initial_token_free_fn
*free_cb
,
568 if (!txp_check_token_len(token_len
, txp_get_mdpl(txp
)))
571 if (txp
->initial_token
!= NULL
&& txp
->initial_token_free_cb
!= NULL
)
572 txp
->initial_token_free_cb(txp
->initial_token
, txp
->initial_token_len
,
573 txp
->initial_token_free_cb_arg
);
575 txp
->initial_token
= token
;
576 txp
->initial_token_len
= token_len
;
577 txp
->initial_token_free_cb
= free_cb
;
578 txp
->initial_token_free_cb_arg
= free_cb_arg
;
582 int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER
*txp
,
583 const QUIC_CONN_ID
*dcid
)
586 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
590 txp
->args
.cur_dcid
= *dcid
;
594 int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER
*txp
,
595 const QUIC_CONN_ID
*scid
)
598 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
602 txp
->args
.cur_scid
= *scid
;
606 /* Change the destination L4 address the TXP uses to send datagrams. */
607 int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER
*txp
,
608 const BIO_ADDR
*peer
)
611 BIO_ADDR_clear(&txp
->args
.peer
);
615 txp
->args
.peer
= *peer
;
619 void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER
*txp
,
620 void (*cb
)(const OSSL_QUIC_FRAME_ACK
*ack
,
626 txp
->ack_tx_cb_arg
= cb_arg
;
629 void ossl_quic_tx_packetiser_set_qlog(OSSL_QUIC_TX_PACKETISER
*txp
,
632 ossl_quic_fifd_set_qlog(&txp
->fifd
, qlog
);
635 int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER
*txp
,
638 if (enc_level
>= QUIC_ENC_LEVEL_NUM
) {
639 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_INVALID_ARGUMENT
);
643 if (enc_level
!= QUIC_ENC_LEVEL_0RTT
)
644 txp
->args
.crypto
[ossl_quic_enc_level_to_pn_space(enc_level
)] = NULL
;
649 void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER
*txp
)
651 txp
->handshake_complete
= 1;
654 void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER
*txp
)
656 txp
->want_handshake_done
= 1;
659 void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER
*txp
,
662 txp
->force_ack_eliciting
|= (1UL << pn_space
);
665 void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER
*txp
,
668 txp
->want_ack
|= (1UL << pn_space
);
671 #define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
672 #define TXP_ERR_SUCCESS 1 /* Success */
673 #define TXP_ERR_SPACE 2 /* Not enough room for another packet */
674 #define TXP_ERR_INPUT 3 /* Invalid/malformed input */
677 * Generates a datagram by polling the various ELs to determine if they want to
678 * generate any frames, and generating a datagram which coalesces packets for
681 int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER
*txp
,
682 QUIC_TXP_STATUS
*status
)
685 * Called to generate one or more datagrams, each containing one or more
688 * There are some tricky things to note here:
690 * - The TXP is only concerned with generating encrypted packets;
691 * other packets use a different path.
693 * - Any datagram containing an Initial packet must have a payload length
694 * (DPL) of at least 1200 bytes. This padding need not necessarily be
695 * found in the Initial packet.
697 * - It is desirable to be able to coalesce an Initial packet
698 * with a Handshake packet. Since, before generating the Handshake
699 * packet, we do not know how long it will be, we cannot know the
700 * correct amount of padding to ensure a DPL of at least 1200 bytes.
701 * Thus this padding must added to the Handshake packet (or whatever
702 * packet is the last in the datagram).
704 * - However, at the time that we generate the Initial packet,
705 * we do not actually know for sure that we will be followed
706 * in the datagram by another packet. For example, suppose we have
707 * some queued data (e.g. crypto stream data for the HANDSHAKE EL)
708 * it looks like we will want to send on the HANDSHAKE EL.
709 * We could assume padding will be placed in the Handshake packet
710 * subsequently and avoid adding any padding to the Initial packet
711 * (which would leave no room for the Handshake packet in the
714 * However, this is not actually a safe assumption. Suppose that we
715 * are using a link with a MDPL of 1200 bytes, the minimum allowed by
716 * QUIC. Suppose that the Initial packet consumes 1195 bytes in total.
717 * Since it is not possible to fit a Handshake packet in just 5 bytes,
718 * upon trying to add a Handshake packet after generating the Initial
719 * packet, we will discover we have no room to fit it! This is not a
720 * problem in itself as another datagram can be sent subsequently, but
721 * it is a problem because we were counting to use that packet to hold
722 * the essential padding. But if we have already finished encrypting
723 * the Initial packet, we cannot go and add padding to it anymore.
724 * This leaves us stuck.
726 * Because of this, we have to plan multiple packets simultaneously, such
727 * that we can start generating a Handshake (or 0-RTT or 1-RTT, or so on)
728 * packet while still having the option to go back and add padding to the
729 * Initial packet if it turns out to be needed.
731 * Trying to predict ahead of time (e.g. during Initial packet generation)
732 * whether we will successfully generate a subsequent packet is fraught with
733 * error as it relies on a large number of variables:
735 * - Do we have room to fit a packet header? (Consider that due to
736 * variable-length integer encoding this is highly variable and can even
737 * depend on payload length due to a variable-length Length field.)
739 * - Can we fit even a single one of the frames we want to put in this
740 * packet in the packet? (Each frame type has a bespoke encoding. While
741 * our encodings of some frame types are adaptive based on the available
742 * room - e.g. STREAM frames - ultimately all frame types have some
743 * absolute minimum number of bytes to be successfully encoded. For
744 * example, if after an Initial packet there is enough room to encode
745 * only one byte of frame data, it is quite likely we can't send any of
746 * the frames we wanted to send.) While this is not strictly a problem
747 * because we could just fill the packet with padding frames, this is a
748 * pointless packet and is wasteful.
750 * Thus we adopt a multi-phase architecture:
752 * 1. Archetype Selection: Determine desired packet archetype.
754 * 2. Packet Staging: Generation of packet information and packet payload
755 * data (frame data) into staging areas.
757 * 3. Packet Adjustment: Adjustment of staged packets, adding padding to
758 * the staged packets if needed.
760 * 4. Commit: The packets are sent to the QTX and recorded as having been
765 uint32_t archetype
, enc_level
;
766 uint32_t conn_close_enc_level
= QUIC_ENC_LEVEL_NUM
;
767 struct txp_pkt pkt
[QUIC_ENC_LEVEL_NUM
];
768 size_t pkts_done
= 0;
769 uint64_t cc_limit
= txp
->args
.cc_method
->get_tx_allowance(txp
->args
.cc_data
);
770 int need_padding
= 0, txpim_pkt_reffed
;
772 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
773 enc_level
< QUIC_ENC_LEVEL_NUM
;
775 pkt
[enc_level
].h_valid
= 0;
777 memset(status
, 0, sizeof(*status
));
780 * Should not be needed, but a sanity check in case anyone else has been
783 ossl_qtx_finish_dgram(txp
->args
.qtx
);
785 /* 1. Archetype Selection */
786 archetype
= txp_determine_archetype(txp
, cc_limit
);
788 /* 2. Packet Staging */
789 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
790 enc_level
< QUIC_ENC_LEVEL_NUM
;
792 size_t running_total
= (enc_level
> QUIC_ENC_LEVEL_INITIAL
)
793 ? pkt
[enc_level
- 1].geom
.hwm
: 0;
795 pkt
[enc_level
].geom
.hwm
= running_total
;
797 if (!txp_should_try_staging(txp
, enc_level
, archetype
, cc_limit
,
798 &conn_close_enc_level
))
801 if (!txp_pkt_init(&pkt
[enc_level
], txp
, enc_level
, archetype
,
804 * If this fails this is not a fatal error - it means the geometry
805 * planning determined there was not enough space for another
806 * packet. So just proceed with what we've already planned for.
810 rc
= txp_generate_for_el(txp
, &pkt
[enc_level
],
811 conn_close_enc_level
== enc_level
);
812 if (rc
!= TXP_ERR_SUCCESS
)
815 if (pkt
[enc_level
].force_pad
)
817 * txp_generate_for_el emitted a frame which forces packet padding.
821 pkt
[enc_level
].geom
.hwm
= running_total
822 + pkt
[enc_level
].h
.bytes_appended
823 + pkt
[enc_level
].geom
.pkt_overhead
;
826 /* 3. Packet Adjustment */
827 if (pkt
[QUIC_ENC_LEVEL_INITIAL
].h_valid
828 && pkt
[QUIC_ENC_LEVEL_INITIAL
].h
.bytes_appended
> 0)
830 * We have an Initial packet in this datagram, so we need to make sure
831 * the total size of the datagram is adequate.
836 size_t total_dgram_size
= 0;
837 const size_t min_dpl
= QUIC_MIN_INITIAL_DGRAM_LEN
;
838 uint32_t pad_el
= QUIC_ENC_LEVEL_NUM
;
840 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
841 enc_level
< QUIC_ENC_LEVEL_NUM
;
843 if (pkt
[enc_level
].h_valid
&& pkt
[enc_level
].h
.bytes_appended
> 0) {
844 if (pad_el
== QUIC_ENC_LEVEL_NUM
846 * We might not be able to add padding, for example if we
847 * are using the ACK_ONLY archetype.
849 && pkt
[enc_level
].geom
.adata
.allow_padding
850 && !pkt
[enc_level
].h
.done_implicit
)
853 txp_pkt_postgen_update_pkt_overhead(&pkt
[enc_level
], txp
);
854 total_dgram_size
+= pkt
[enc_level
].geom
.pkt_overhead
855 + pkt
[enc_level
].h
.bytes_appended
;
858 if (pad_el
!= QUIC_ENC_LEVEL_NUM
&& total_dgram_size
< min_dpl
) {
859 size_t deficit
= min_dpl
- total_dgram_size
;
861 if (!txp_pkt_append_padding(&pkt
[pad_el
], txp
, deficit
))
864 total_dgram_size
+= deficit
;
867 * Padding frames make a packet ineligible for being a non-inflight
870 pkt
[pad_el
].tpkt
->ackm_pkt
.is_inflight
= 1;
874 * If we have failed to make a datagram of adequate size, for example
875 * because we have a padding requirement but are using the ACK_ONLY
876 * archetype (because we are CC limited), which precludes us from
877 * sending padding, give up on generating the datagram - there is
880 if (total_dgram_size
< min_dpl
) {
887 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
888 enc_level
< QUIC_ENC_LEVEL_NUM
;
891 if (!pkt
[enc_level
].h_valid
)
892 /* Did not attempt to generate a packet for this EL. */
895 if (pkt
[enc_level
].h
.bytes_appended
== 0)
896 /* Nothing was generated for this EL, so skip. */
899 rc
= txp_pkt_commit(txp
, &pkt
[enc_level
], archetype
,
902 status
->sent_ack_eliciting
903 = status
->sent_ack_eliciting
904 || pkt
[enc_level
].tpkt
->ackm_pkt
.is_ack_eliciting
;
906 if (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
)
907 status
->sent_handshake
908 = (pkt
[enc_level
].h_valid
909 && pkt
[enc_level
].h
.bytes_appended
> 0);
912 if (txpim_pkt_reffed
)
913 pkt
[enc_level
].tpkt
= NULL
; /* don't free */
921 /* Flush & Cleanup */
924 ossl_qtx_finish_dgram(txp
->args
.qtx
);
926 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
927 enc_level
< QUIC_ENC_LEVEL_NUM
;
929 txp_pkt_cleanup(&pkt
[enc_level
], txp
);
931 status
->sent_pkt
= pkts_done
;
936 static const struct archetype_data archetypes
[QUIC_ENC_LEVEL_NUM
][TX_PACKETISER_ARCHETYPE_NUM
] = {
939 /* EL 0(INITIAL) - Archetype 0(NORMAL) */
943 /*allow_crypto =*/ 1,
944 /*allow_handshake_done =*/ 0,
945 /*allow_path_challenge =*/ 0,
946 /*allow_path_response =*/ 0,
947 /*allow_new_conn_id =*/ 0,
948 /*allow_retire_conn_id =*/ 0,
949 /*allow_stream_rel =*/ 0,
950 /*allow_conn_fc =*/ 0,
951 /*allow_conn_close =*/ 1,
952 /*allow_cfq_other =*/ 0,
953 /*allow_new_token =*/ 0,
954 /*allow_force_ack_eliciting =*/ 1,
955 /*allow_padding =*/ 1,
956 /*require_ack_eliciting =*/ 0,
959 /* EL 0(INITIAL) - Archetype 1(PROBE) */
963 /*allow_crypto =*/ 1,
964 /*allow_handshake_done =*/ 0,
965 /*allow_path_challenge =*/ 0,
966 /*allow_path_response =*/ 0,
967 /*allow_new_conn_id =*/ 0,
968 /*allow_retire_conn_id =*/ 0,
969 /*allow_stream_rel =*/ 0,
970 /*allow_conn_fc =*/ 0,
971 /*allow_conn_close =*/ 1,
972 /*allow_cfq_other =*/ 0,
973 /*allow_new_token =*/ 0,
974 /*allow_force_ack_eliciting =*/ 1,
975 /*allow_padding =*/ 1,
976 /*require_ack_eliciting =*/ 1,
979 /* EL 0(INITIAL) - Archetype 2(ACK_ONLY) */
983 /*allow_crypto =*/ 0,
984 /*allow_handshake_done =*/ 0,
985 /*allow_path_challenge =*/ 0,
986 /*allow_path_response =*/ 0,
987 /*allow_new_conn_id =*/ 0,
988 /*allow_retire_conn_id =*/ 0,
989 /*allow_stream_rel =*/ 0,
990 /*allow_conn_fc =*/ 0,
991 /*allow_conn_close =*/ 0,
992 /*allow_cfq_other =*/ 0,
993 /*allow_new_token =*/ 0,
994 /*allow_force_ack_eliciting =*/ 1,
995 /*allow_padding =*/ 0,
996 /*require_ack_eliciting =*/ 0,
1000 /* EL 1(HANDSHAKE) */
1002 /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
1006 /*allow_crypto =*/ 1,
1007 /*allow_handshake_done =*/ 0,
1008 /*allow_path_challenge =*/ 0,
1009 /*allow_path_response =*/ 0,
1010 /*allow_new_conn_id =*/ 0,
1011 /*allow_retire_conn_id =*/ 0,
1012 /*allow_stream_rel =*/ 0,
1013 /*allow_conn_fc =*/ 0,
1014 /*allow_conn_close =*/ 1,
1015 /*allow_cfq_other =*/ 0,
1016 /*allow_new_token =*/ 0,
1017 /*allow_force_ack_eliciting =*/ 1,
1018 /*allow_padding =*/ 1,
1019 /*require_ack_eliciting =*/ 0,
1022 /* EL 1(HANDSHAKE) - Archetype 1(PROBE) */
1026 /*allow_crypto =*/ 1,
1027 /*allow_handshake_done =*/ 0,
1028 /*allow_path_challenge =*/ 0,
1029 /*allow_path_response =*/ 0,
1030 /*allow_new_conn_id =*/ 0,
1031 /*allow_retire_conn_id =*/ 0,
1032 /*allow_stream_rel =*/ 0,
1033 /*allow_conn_fc =*/ 0,
1034 /*allow_conn_close =*/ 1,
1035 /*allow_cfq_other =*/ 0,
1036 /*allow_new_token =*/ 0,
1037 /*allow_force_ack_eliciting =*/ 1,
1038 /*allow_padding =*/ 1,
1039 /*require_ack_eliciting =*/ 1,
1042 /* EL 1(HANDSHAKE) - Archetype 2(ACK_ONLY) */
1046 /*allow_crypto =*/ 0,
1047 /*allow_handshake_done =*/ 0,
1048 /*allow_path_challenge =*/ 0,
1049 /*allow_path_response =*/ 0,
1050 /*allow_new_conn_id =*/ 0,
1051 /*allow_retire_conn_id =*/ 0,
1052 /*allow_stream_rel =*/ 0,
1053 /*allow_conn_fc =*/ 0,
1054 /*allow_conn_close =*/ 0,
1055 /*allow_cfq_other =*/ 0,
1056 /*allow_new_token =*/ 0,
1057 /*allow_force_ack_eliciting =*/ 1,
1058 /*allow_padding =*/ 0,
1059 /*require_ack_eliciting =*/ 0,
1065 /* EL 2(0RTT) - Archetype 0(NORMAL) */
1069 /*allow_crypto =*/ 0,
1070 /*allow_handshake_done =*/ 0,
1071 /*allow_path_challenge =*/ 0,
1072 /*allow_path_response =*/ 0,
1073 /*allow_new_conn_id =*/ 1,
1074 /*allow_retire_conn_id =*/ 1,
1075 /*allow_stream_rel =*/ 1,
1076 /*allow_conn_fc =*/ 1,
1077 /*allow_conn_close =*/ 1,
1078 /*allow_cfq_other =*/ 0,
1079 /*allow_new_token =*/ 0,
1080 /*allow_force_ack_eliciting =*/ 0,
1081 /*allow_padding =*/ 1,
1082 /*require_ack_eliciting =*/ 0,
1085 /* EL 2(0RTT) - Archetype 1(PROBE) */
1089 /*allow_crypto =*/ 0,
1090 /*allow_handshake_done =*/ 0,
1091 /*allow_path_challenge =*/ 0,
1092 /*allow_path_response =*/ 0,
1093 /*allow_new_conn_id =*/ 1,
1094 /*allow_retire_conn_id =*/ 1,
1095 /*allow_stream_rel =*/ 1,
1096 /*allow_conn_fc =*/ 1,
1097 /*allow_conn_close =*/ 1,
1098 /*allow_cfq_other =*/ 0,
1099 /*allow_new_token =*/ 0,
1100 /*allow_force_ack_eliciting =*/ 0,
1101 /*allow_padding =*/ 1,
1102 /*require_ack_eliciting =*/ 1,
1105 /* EL 2(0RTT) - Archetype 2(ACK_ONLY) */
1109 /*allow_crypto =*/ 0,
1110 /*allow_handshake_done =*/ 0,
1111 /*allow_path_challenge =*/ 0,
1112 /*allow_path_response =*/ 0,
1113 /*allow_new_conn_id =*/ 0,
1114 /*allow_retire_conn_id =*/ 0,
1115 /*allow_stream_rel =*/ 0,
1116 /*allow_conn_fc =*/ 0,
1117 /*allow_conn_close =*/ 0,
1118 /*allow_cfq_other =*/ 0,
1119 /*allow_new_token =*/ 0,
1120 /*allow_force_ack_eliciting =*/ 0,
1121 /*allow_padding =*/ 0,
1122 /*require_ack_eliciting =*/ 0,
1128 /* EL 3(1RTT) - Archetype 0(NORMAL) */
1132 /*allow_crypto =*/ 1,
1133 /*allow_handshake_done =*/ 1,
1134 /*allow_path_challenge =*/ 0,
1135 /*allow_path_response =*/ 1,
1136 /*allow_new_conn_id =*/ 1,
1137 /*allow_retire_conn_id =*/ 1,
1138 /*allow_stream_rel =*/ 1,
1139 /*allow_conn_fc =*/ 1,
1140 /*allow_conn_close =*/ 1,
1141 /*allow_cfq_other =*/ 1,
1142 /*allow_new_token =*/ 1,
1143 /*allow_force_ack_eliciting =*/ 1,
1144 /*allow_padding =*/ 1,
1145 /*require_ack_eliciting =*/ 0,
1148 /* EL 3(1RTT) - Archetype 1(PROBE) */
1152 /*allow_crypto =*/ 1,
1153 /*allow_handshake_done =*/ 1,
1154 /*allow_path_challenge =*/ 0,
1155 /*allow_path_response =*/ 1,
1156 /*allow_new_conn_id =*/ 1,
1157 /*allow_retire_conn_id =*/ 1,
1158 /*allow_stream_rel =*/ 1,
1159 /*allow_conn_fc =*/ 1,
1160 /*allow_conn_close =*/ 1,
1161 /*allow_cfq_other =*/ 1,
1162 /*allow_new_token =*/ 1,
1163 /*allow_force_ack_eliciting =*/ 1,
1164 /*allow_padding =*/ 1,
1165 /*require_ack_eliciting =*/ 1,
1168 /* EL 3(1RTT) - Archetype 2(ACK_ONLY) */
1172 /*allow_crypto =*/ 0,
1173 /*allow_handshake_done =*/ 0,
1174 /*allow_path_challenge =*/ 0,
1175 /*allow_path_response =*/ 0,
1176 /*allow_new_conn_id =*/ 0,
1177 /*allow_retire_conn_id =*/ 0,
1178 /*allow_stream_rel =*/ 0,
1179 /*allow_conn_fc =*/ 0,
1180 /*allow_conn_close =*/ 0,
1181 /*allow_cfq_other =*/ 0,
1182 /*allow_new_token =*/ 0,
1183 /*allow_force_ack_eliciting =*/ 1,
1184 /*allow_padding =*/ 0,
1185 /*require_ack_eliciting =*/ 0,
1191 static int txp_get_archetype_data(uint32_t enc_level
,
1193 struct archetype_data
*a
)
1195 if (enc_level
>= QUIC_ENC_LEVEL_NUM
1196 || archetype
>= TX_PACKETISER_ARCHETYPE_NUM
)
1199 /* No need to avoid copying this as it should not exceed one int in size. */
1200 *a
= archetypes
[enc_level
][archetype
];
1204 static int txp_determine_geometry(OSSL_QUIC_TX_PACKETISER
*txp
,
1207 size_t running_total
,
1209 struct txp_pkt_geom
*geom
)
1211 size_t mdpl
, cmpl
, hdr_len
;
1213 /* Get information about packet archetype. */
1214 if (!txp_get_archetype_data(enc_level
, archetype
, &geom
->adata
))
1217 /* Assemble packet header. */
1218 phdr
->type
= ossl_quic_enc_level_to_pkt_type(enc_level
);
1220 phdr
->pn_len
= txp_determine_pn_len(txp
);
1224 phdr
->version
= QUIC_VERSION_1
;
1225 phdr
->dst_conn_id
= txp
->args
.cur_dcid
;
1226 phdr
->src_conn_id
= txp
->args
.cur_scid
;
1229 * We need to know the length of the payload to get an accurate header
1230 * length for non-1RTT packets, because the Length field found in
1231 * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
1232 * we don't have a good idea of the length of our payload, because the
1233 * length of the payload depends on the room in the datagram after fitting
1234 * the header, which depends on the size of the header.
1236 * In general, it does not matter if a packet is slightly shorter (because
1237 * e.g. we predicted use of a 2-byte length field, but ended up only needing
1238 * a 1-byte length field). However this does matter for Initial packets
1239 * which must be at least 1200 bytes, which is also the assumed default MTU;
1240 * therefore in many cases Initial packets will be padded to 1200 bytes,
1241 * which means if we overestimated the header size, we will be short by a
1242 * few bytes and the server will ignore the packet for being too short. In
1243 * this case, however, such packets always *will* be padded to meet 1200
1244 * bytes, which requires a 2-byte length field, so we don't actually need to
1245 * worry about this. Thus we estimate the header length assuming a 2-byte
1246 * length field here, which should in practice work well in all cases.
1248 phdr
->len
= OSSL_QUIC_VLINT_2B_MAX
- phdr
->pn_len
;
1250 if (enc_level
== QUIC_ENC_LEVEL_INITIAL
) {
1251 phdr
->token
= txp
->initial_token
;
1252 phdr
->token_len
= txp
->initial_token_len
;
1255 phdr
->token_len
= 0;
1258 hdr_len
= ossl_quic_wire_get_encoded_pkt_hdr_len(phdr
->dst_conn_id
.id_len
,
1263 /* MDPL: Maximum datagram payload length. */
1264 mdpl
= txp_get_mdpl(txp
);
1267 * CMPL: Maximum encoded packet size we can put into this datagram given any
1268 * previous packets coalesced into it.
1270 if (running_total
> mdpl
)
1271 /* Should not be possible, but if it happens: */
1274 cmpl
= mdpl
- running_total
;
1276 /* CMPPL: Maximum amount we can put into the current packet payload */
1277 if (!txp_determine_ppl_from_pl(txp
, cmpl
, enc_level
, hdr_len
, &geom
->cmppl
))
1281 geom
->pkt_overhead
= cmpl
- geom
->cmppl
;
1282 geom
->archetype
= archetype
;
1286 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER
*txp
,
1289 OSSL_ACKM_PROBE_INFO
*probe_info
1290 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
1294 * If ACKM has requested probe generation (e.g. due to PTO), we generate a
1295 * Probe-archetype packet. Actually, we determine archetype on a
1296 * per-datagram basis, so if any EL wants a probe, do a pass in which
1297 * we try and generate a probe (if needed) for all ELs.
1299 if (probe_info
->anti_deadlock_initial
> 0
1300 || probe_info
->anti_deadlock_handshake
> 0)
1301 return TX_PACKETISER_ARCHETYPE_PROBE
;
1303 for (pn_space
= QUIC_PN_SPACE_INITIAL
;
1304 pn_space
< QUIC_PN_SPACE_NUM
;
1306 if (probe_info
->pto
[pn_space
] > 0)
1307 return TX_PACKETISER_ARCHETYPE_PROBE
;
1310 * If we are out of CC budget, we cannot send a normal packet,
1311 * but we can do an ACK-only packet (potentially, if we
1312 * want to send an ACK).
1315 return TX_PACKETISER_ARCHETYPE_ACK_ONLY
;
1317 /* All other packets. */
1318 return TX_PACKETISER_ARCHETYPE_NORMAL
;
1321 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER
*txp
,
1325 uint32_t *conn_close_enc_level
)
1327 struct archetype_data a
;
1328 uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
1329 QUIC_CFQ_ITEM
*cfq_item
;
1331 if (!ossl_qtx_is_enc_level_provisioned(txp
->args
.qtx
, enc_level
))
1334 if (!txp_get_archetype_data(enc_level
, archetype
, &a
))
1337 if (!a
.bypass_cc
&& cc_limit
== 0)
1338 /* CC not allowing us to send. */
1342 * We can produce CONNECTION_CLOSE frames on any EL in principle, which
1343 * means we need to choose which EL we would prefer to use. After a
1344 * connection is fully established we have only one provisioned EL and this
1345 * is a non-issue. Where multiple ELs are provisioned, it is possible the
1346 * peer does not have the keys for the EL yet, which suggests in general it
1347 * is preferable to use the lowest EL which is still provisioned.
1349 * However (RFC 9000 s. 10.2.3 & 12.5) we are also required to not send
1350 * application CONNECTION_CLOSE frames in non-1-RTT ELs, so as to not
1351 * potentially leak application data on a connection which has yet to be
1352 * authenticated. Thus when we have an application CONNECTION_CLOSE frame
1353 * queued and need to send it on a non-1-RTT EL, we have to convert it
1354 * into a transport CONNECTION_CLOSE frame which contains no application
1355 * data. Since this loses information, it suggests we should use the 1-RTT
1356 * EL to avoid this if possible, even if a lower EL is also available.
1358 * At the same time, just because we have the 1-RTT EL provisioned locally
1359 * does not necessarily mean the peer does, for example if a handshake
1360 * CRYPTO frame has been lost. It is fairly important that CONNECTION_CLOSE
1361 * is signalled in a way we know our peer can decrypt, as we stop processing
1362 * connection retransmission logic for real after connection close and
1363 * simply 'blindly' retransmit the same CONNECTION_CLOSE frame.
1365 * This is not a major concern for clients, since if a client has a 1-RTT EL
1366 * provisioned the server is guaranteed to also have a 1-RTT EL provisioned.
1368 * TODO(QUIC SERVER): Revisit this when server support is added.
1370 if (*conn_close_enc_level
> enc_level
1371 && *conn_close_enc_level
!= QUIC_ENC_LEVEL_1RTT
)
1372 *conn_close_enc_level
= enc_level
;
1374 /* Do we need to send a PTO probe? */
1375 if (a
.allow_force_ack_eliciting
) {
1376 OSSL_ACKM_PROBE_INFO
*probe_info
1377 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
1379 if ((enc_level
== QUIC_ENC_LEVEL_INITIAL
1380 && probe_info
->anti_deadlock_initial
> 0)
1381 || (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
1382 && probe_info
->anti_deadlock_handshake
> 0)
1383 || probe_info
->pto
[pn_space
] > 0)
1387 /* Does the crypto stream for this EL want to produce anything? */
1388 if (a
.allow_crypto
&& sstream_is_pending(txp
->args
.crypto
[pn_space
]))
1391 /* Does the ACKM for this PN space want to produce anything? */
1392 if (a
.allow_ack
&& (ossl_ackm_is_ack_desired(txp
->args
.ackm
, pn_space
)
1393 || (txp
->want_ack
& (1UL << pn_space
)) != 0))
1396 /* Do we need to force emission of an ACK-eliciting packet? */
1397 if (a
.allow_force_ack_eliciting
1398 && (txp
->force_ack_eliciting
& (1UL << pn_space
)) != 0)
1401 /* Does the connection-level RXFC want to produce a frame? */
1402 if (a
.allow_conn_fc
&& (txp
->want_max_data
1403 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 0)))
1406 /* Do we want to produce a MAX_STREAMS frame? */
1408 && (txp
->want_max_streams_bidi
1409 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
,
1411 || txp
->want_max_streams_uni
1412 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
,
1416 /* Do we want to produce a HANDSHAKE_DONE frame? */
1417 if (a
.allow_handshake_done
&& txp
->want_handshake_done
)
1420 /* Do we want to produce a CONNECTION_CLOSE frame? */
1421 if (a
.allow_conn_close
&& txp
->want_conn_close
&&
1422 *conn_close_enc_level
== enc_level
)
1424 * This is a bit of a special case since CONNECTION_CLOSE can appear in
1425 * most packet types, and when we decide we want to send it this status
1426 * isn't tied to a specific EL. So if we want to send it, we send it
1427 * only on the lowest non-dropped EL.
1431 /* Does the CFQ have any frames queued for this PN space? */
1432 if (enc_level
!= QUIC_ENC_LEVEL_0RTT
)
1433 for (cfq_item
= ossl_quic_cfq_get_priority_head(txp
->args
.cfq
, pn_space
);
1435 cfq_item
= ossl_quic_cfq_item_get_priority_next(cfq_item
, pn_space
)) {
1436 uint64_t frame_type
= ossl_quic_cfq_item_get_frame_type(cfq_item
);
1438 switch (frame_type
) {
1439 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID
:
1440 if (a
.allow_new_conn_id
)
1443 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID
:
1444 if (a
.allow_retire_conn_id
)
1447 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN
:
1448 if (a
.allow_new_token
)
1451 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE
:
1452 if (a
.allow_path_response
)
1456 if (a
.allow_cfq_other
)
1462 if (a
.allow_stream_rel
&& txp
->handshake_complete
) {
1463 QUIC_STREAM_ITER it
;
1465 /* If there are any active streams, 0/1-RTT wants to produce a packet.
1466 * Whether a stream is on the active list is required to be precise
1467 * (i.e., a stream is never on the active list if we cannot produce a
1468 * frame for it), and all stream-related frames are governed by
1469 * a.allow_stream_rel (i.e., if we can send one type of stream-related
1470 * frame, we can send any of them), so we don't need to inspect
1471 * individual streams on the active list, just confirm that the active
1472 * list is non-empty.
1474 ossl_quic_stream_iter_init(&it
, txp
->args
.qsm
, 0);
1475 if (it
.stream
!= NULL
)
1482 static int sstream_is_pending(QUIC_SSTREAM
*sstream
)
1484 OSSL_QUIC_FRAME_STREAM hdr
;
1485 OSSL_QTX_IOVEC iov
[2];
1486 size_t num_iov
= OSSL_NELEM(iov
);
1488 return ossl_quic_sstream_get_stream_frame(sstream
, 0, &hdr
, iov
, &num_iov
);
1491 /* Determine how many bytes we should use for the encoded PN. */
1492 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER
*txp
)
1494 return 4; /* TODO(QUIC FUTURE) */
1497 /* Determine plaintext packet payload length from payload length. */
1498 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER
*txp
,
1509 if (!ossl_qtx_calculate_plaintext_payload_len(txp
->args
.qtx
, enc_level
,
1517 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER
*txp
)
1519 return ossl_qtx_get_mdpl(txp
->args
.qtx
);
1522 static QUIC_SSTREAM
*get_sstream_by_id(uint64_t stream_id
, uint32_t pn_space
,
1525 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1528 if (stream_id
== UINT64_MAX
)
1529 return txp
->args
.crypto
[pn_space
];
1531 s
= ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1538 static void on_regen_notify(uint64_t frame_type
, uint64_t stream_id
,
1539 QUIC_TXPIM_PKT
*pkt
, void *arg
)
1541 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1543 switch (frame_type
) {
1544 case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE
:
1545 txp
->want_handshake_done
= 1;
1547 case OSSL_QUIC_FRAME_TYPE_MAX_DATA
:
1548 txp
->want_max_data
= 1;
1550 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI
:
1551 txp
->want_max_streams_bidi
= 1;
1553 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI
:
1554 txp
->want_max_streams_uni
= 1;
1556 case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN
:
1557 txp
->want_ack
|= (1UL << pkt
->ackm_pkt
.pkt_space
);
1559 case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA
:
1562 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1567 s
->want_max_stream_data
= 1;
1568 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1571 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING
:
1574 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1579 ossl_quic_stream_map_schedule_stop_sending(txp
->args
.qsm
, s
);
1582 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM
:
1585 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1590 s
->want_reset_stream
= 1;
1591 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1600 static int txp_pkt_init(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
,
1601 uint32_t enc_level
, uint32_t archetype
,
1602 size_t running_total
)
1604 if (!txp_determine_geometry(txp
, archetype
, enc_level
,
1605 running_total
, &pkt
->phdr
, &pkt
->geom
))
1609 * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
1612 if (!tx_helper_init(&pkt
->h
, txp
, enc_level
,
1614 pkt
->geom
.adata
.require_ack_eliciting
? 1 : 0))
1619 pkt
->stream_head
= NULL
;
1624 static void txp_pkt_cleanup(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
)
1629 tx_helper_cleanup(&pkt
->h
);
1632 if (pkt
->tpkt
!= NULL
) {
1633 ossl_quic_txpim_pkt_release(txp
->args
.txpim
, pkt
->tpkt
);
1638 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt
*pkt
,
1639 OSSL_QUIC_TX_PACKETISER
*txp
)
1642 * After we have staged and generated our packets, but before we commit
1643 * them, it is possible for the estimated packet overhead (packet header +
1644 * AEAD tag size) to shrink slightly because we generated a short packet
1645 * whose which can be represented in fewer bytes as a variable-length
1646 * integer than we were (pessimistically) budgeting for. We need to account
1647 * for this to ensure that we get our padding calculation exactly right.
1649 * Update pkt_overhead to be accurate now that we know how much data is
1650 * going in a packet.
1652 size_t hdr_len
, ciphertext_len
;
1654 if (pkt
->h
.enc_level
== QUIC_ENC_LEVEL_INITIAL
)
1656 * Don't update overheads for the INITIAL EL - we have not finished
1657 * appending padding to it and would potentially miscalculate the
1658 * correct padding if we now update the pkt_overhead field to switch to
1659 * e.g. a 1-byte length field in the packet header. Since we are padding
1660 * to QUIC_MIN_INITIAL_DGRAM_LEN which requires a 2-byte length field,
1661 * this is guaranteed to be moot anyway. See comment in
1662 * txp_determine_geometry for more information.
1666 if (!ossl_qtx_calculate_ciphertext_payload_len(txp
->args
.qtx
, pkt
->h
.enc_level
,
1667 pkt
->h
.bytes_appended
,
1671 pkt
->phdr
.len
= ciphertext_len
;
1673 hdr_len
= ossl_quic_wire_get_encoded_pkt_hdr_len(pkt
->phdr
.dst_conn_id
.id_len
,
1676 pkt
->geom
.pkt_overhead
= hdr_len
+ ciphertext_len
- pkt
->h
.bytes_appended
;
1680 static void on_confirm_notify(uint64_t frame_type
, uint64_t stream_id
,
1681 QUIC_TXPIM_PKT
*pkt
, void *arg
)
1683 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1685 switch (frame_type
) {
1686 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING
:
1689 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1694 s
->acked_stop_sending
= 1;
1695 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1698 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM
:
1701 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1707 * We must already be in RESET_SENT or RESET_RECVD if we are
1708 * here, so we don't need to check state here.
1710 ossl_quic_stream_map_notify_reset_stream_acked(txp
->args
.qsm
, s
);
1711 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1720 static int txp_pkt_append_padding(struct txp_pkt
*pkt
,
1721 OSSL_QUIC_TX_PACKETISER
*txp
, size_t num_bytes
)
1728 if (!ossl_assert(pkt
->h_valid
))
1731 if (!ossl_assert(pkt
->tpkt
!= NULL
))
1734 wpkt
= tx_helper_begin(&pkt
->h
);
1738 if (!ossl_quic_wire_encode_padding(wpkt
, num_bytes
)) {
1739 tx_helper_rollback(&pkt
->h
);
1743 if (!tx_helper_commit(&pkt
->h
))
1746 pkt
->tpkt
->ackm_pkt
.num_bytes
+= num_bytes
;
1747 /* Cannot be non-inflight if we have a PADDING frame */
1748 pkt
->tpkt
->ackm_pkt
.is_inflight
= 1;
1752 static void on_sstream_updated(uint64_t stream_id
, void *arg
)
1754 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1757 s
= ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1761 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1765 * Returns 1 if we can send that many bytes in closing state, 0 otherwise.
1766 * Also maintains the bytes sent state if it returns a success.
1768 static int try_commit_conn_close(OSSL_QUIC_TX_PACKETISER
*txp
, size_t n
)
1772 /* We can always send the first connection close frame */
1773 if (txp
->closing_bytes_recv
== 0)
1777 * RFC 9000 s. 10.2.1 Closing Connection State:
1778 * To avoid being used for an amplification attack, such
1779 * endpoints MUST limit the cumulative size of packets it sends
1780 * to three times the cumulative size of the packets that are
1781 * received and attributed to the connection.
1783 * An endpoint in the closing state MUST either discard packets
1784 * received from an unvalidated address or limit the cumulative
1785 * size of packets it sends to an unvalidated address to three
1786 * times the size of packets it receives from that address.
1788 res
= txp
->closing_bytes_xmit
+ n
<= txp
->closing_bytes_recv
* 3;
1791 * Attribute the bytes to the connection, if we are allowed to send them
1792 * and this isn't the first closing frame.
1794 if (res
&& txp
->closing_bytes_recv
!= 0)
1795 txp
->closing_bytes_xmit
+= n
;
1799 void ossl_quic_tx_packetiser_record_received_closing_bytes(
1800 OSSL_QUIC_TX_PACKETISER
*txp
, size_t n
)
1802 txp
->closing_bytes_recv
+= n
;
1805 static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER
*txp
,
1806 struct txp_pkt
*pkt
,
1807 int chosen_for_conn_close
,
1808 int *can_be_non_inflight
)
1810 const uint32_t enc_level
= pkt
->h
.enc_level
;
1811 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
1812 const struct archetype_data
*a
= &pkt
->geom
.adata
;
1813 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
1814 struct tx_helper
*h
= &pkt
->h
;
1815 const OSSL_QUIC_FRAME_ACK
*ack
;
1816 OSSL_QUIC_FRAME_ACK ack2
;
1818 tpkt
->ackm_pkt
.largest_acked
= QUIC_PN_INVALID
;
1820 /* ACK Frames (Regenerate) */
1822 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_ACK
1823 && (((txp
->want_ack
& (1UL << pn_space
)) != 0)
1824 || ossl_ackm_is_ack_desired(txp
->args
.ackm
, pn_space
))
1825 && (ack
= ossl_ackm_get_ack_frame(txp
->args
.ackm
, pn_space
)) != NULL
) {
1826 WPACKET
*wpkt
= tx_helper_begin(h
);
1831 /* We do not currently support ECN */
1833 ack2
.ecn_present
= 0;
1835 if (ossl_quic_wire_encode_frame_ack(wpkt
,
1836 txp
->args
.ack_delay_exponent
,
1838 if (!tx_helper_commit(h
))
1841 tpkt
->had_ack_frame
= 1;
1843 if (ack
->num_ack_ranges
> 0)
1844 tpkt
->ackm_pkt
.largest_acked
= ack
->ack_ranges
[0].end
;
1846 if (txp
->ack_tx_cb
!= NULL
)
1847 txp
->ack_tx_cb(&ack2
, pn_space
, txp
->ack_tx_cb_arg
);
1849 tx_helper_rollback(h
);
1853 /* CONNECTION_CLOSE Frames (Regenerate) */
1854 if (a
->allow_conn_close
&& txp
->want_conn_close
&& chosen_for_conn_close
) {
1855 WPACKET
*wpkt
= tx_helper_begin(h
);
1856 OSSL_QUIC_FRAME_CONN_CLOSE f
, *pf
= &txp
->conn_close_frame
;
1863 * Application CONNECTION_CLOSE frames may only be sent in the
1864 * Application PN space, as otherwise they may be sent before a
1865 * connection is authenticated and leak application data. Therefore, if
1866 * we need to send a CONNECTION_CLOSE frame in another PN space and were
1867 * given an application CONNECTION_CLOSE frame, convert it into a
1868 * transport CONNECTION_CLOSE frame, removing any sensitive application
1871 * RFC 9000 s. 10.2.3: "A CONNECTION_CLOSE of type 0x1d MUST be replaced
1872 * by a CONNECTION_CLOSE of type 0x1c when sending the frame in Initial
1873 * or Handshake packets. Otherwise, information about the application
1874 * state might be revealed. Endpoints MUST clear the value of the Reason
1875 * Phrase field and SHOULD use the APPLICATION_ERROR code when
1876 * converting to a CONNECTION_CLOSE of type 0x1c."
1878 if (pn_space
!= QUIC_PN_SPACE_APP
&& pf
->is_app
) {
1882 pf
->error_code
= QUIC_ERR_APPLICATION_ERROR
;
1887 if (ossl_quic_wire_encode_frame_conn_close(wpkt
, pf
)
1888 && WPACKET_get_total_written(wpkt
, &l
)
1889 && try_commit_conn_close(txp
, l
)) {
1890 if (!tx_helper_commit(h
))
1893 tpkt
->had_conn_close
= 1;
1894 *can_be_non_inflight
= 0;
1896 tx_helper_rollback(h
);
1903 static int try_len(size_t space_left
, size_t orig_len
,
1904 size_t base_hdr_len
, size_t lenbytes
,
1905 uint64_t maxn
, size_t *hdr_len
, size_t *payload_len
)
1908 size_t maxn_
= maxn
> SIZE_MAX
? SIZE_MAX
: (size_t)maxn
;
1910 *hdr_len
= base_hdr_len
+ lenbytes
;
1912 if (orig_len
== 0 && space_left
>= *hdr_len
) {
1920 if (n
+ *hdr_len
> space_left
)
1921 n
= (space_left
>= *hdr_len
) ? space_left
- *hdr_len
: 0;
1927 static int determine_len(size_t space_left
, size_t orig_len
,
1928 size_t base_hdr_len
,
1929 uint64_t *hlen
, uint64_t *len
)
1932 size_t chosen_payload_len
= 0;
1933 size_t chosen_hdr_len
= 0;
1934 size_t payload_len
[4], hdr_len
[4];
1935 int i
, valid
[4] = {0};
1937 valid
[0] = try_len(space_left
, orig_len
, base_hdr_len
,
1938 1, OSSL_QUIC_VLINT_1B_MAX
,
1939 &hdr_len
[0], &payload_len
[0]);
1940 valid
[1] = try_len(space_left
, orig_len
, base_hdr_len
,
1941 2, OSSL_QUIC_VLINT_2B_MAX
,
1942 &hdr_len
[1], &payload_len
[1]);
1943 valid
[2] = try_len(space_left
, orig_len
, base_hdr_len
,
1944 4, OSSL_QUIC_VLINT_4B_MAX
,
1945 &hdr_len
[2], &payload_len
[2]);
1946 valid
[3] = try_len(space_left
, orig_len
, base_hdr_len
,
1947 8, OSSL_QUIC_VLINT_8B_MAX
,
1948 &hdr_len
[3], &payload_len
[3]);
1950 for (i
= OSSL_NELEM(valid
) - 1; i
>= 0; --i
)
1951 if (valid
[i
] && payload_len
[i
] >= chosen_payload_len
) {
1952 chosen_payload_len
= payload_len
[i
];
1953 chosen_hdr_len
= hdr_len
[i
];
1957 *hlen
= chosen_hdr_len
;
1958 *len
= chosen_payload_len
;
1963 * Given a CRYPTO frame header with accurate chdr->len and a budget
1964 * (space_left), try to find the optimal value of chdr->len to fill as much of
1965 * the budget as possible. This is slightly hairy because larger values of
1966 * chdr->len cause larger encoded sizes of the length field of the frame, which
1967 * in turn mean less space available for payload data. We check all possible
1968 * encodings and choose the optimal encoding.
1970 static int determine_crypto_len(struct tx_helper
*h
,
1971 OSSL_QUIC_FRAME_CRYPTO
*chdr
,
1977 size_t base_hdr_len
; /* CRYPTO header length without length field */
1979 if (chdr
->len
> SIZE_MAX
)
1982 orig_len
= (size_t)chdr
->len
;
1985 base_hdr_len
= ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr
);
1986 chdr
->len
= orig_len
;
1987 if (base_hdr_len
== 0)
1992 return determine_len(space_left
, orig_len
, base_hdr_len
, hlen
, len
);
1995 static int determine_stream_len(struct tx_helper
*h
,
1996 OSSL_QUIC_FRAME_STREAM
*shdr
,
2002 size_t base_hdr_len
; /* STREAM header length without length field */
2004 if (shdr
->len
> SIZE_MAX
)
2007 orig_len
= (size_t)shdr
->len
;
2010 base_hdr_len
= ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr
);
2011 shdr
->len
= orig_len
;
2012 if (base_hdr_len
== 0)
2015 if (shdr
->has_explicit_len
)
2018 return determine_len(space_left
, orig_len
, base_hdr_len
, hlen
, len
);
2021 static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER
*txp
,
2022 struct txp_pkt
*pkt
,
2023 int *have_ack_eliciting
)
2025 const uint32_t enc_level
= pkt
->h
.enc_level
;
2026 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2027 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2028 struct tx_helper
*h
= &pkt
->h
;
2029 size_t num_stream_iovec
;
2030 OSSL_QUIC_FRAME_STREAM shdr
= {0};
2031 OSSL_QUIC_FRAME_CRYPTO chdr
= {0};
2032 OSSL_QTX_IOVEC iov
[2];
2035 QUIC_TXPIM_CHUNK chunk
= {0};
2036 size_t i
, space_left
;
2039 space_left
= tx_helper_get_space_left(h
);
2041 if (space_left
< MIN_FRAME_SIZE_CRYPTO
)
2042 return 1; /* no point trying */
2044 /* Do we have any CRYPTO data waiting? */
2045 num_stream_iovec
= OSSL_NELEM(iov
);
2046 if (!ossl_quic_sstream_get_stream_frame(txp
->args
.crypto
[pn_space
],
2049 return 1; /* nothing to do */
2051 /* Convert STREAM frame header to CRYPTO frame header */
2052 chdr
.offset
= shdr
.offset
;
2053 chdr
.len
= shdr
.len
;
2056 return 1; /* nothing to do */
2058 /* Find best fit (header length, payload length) combination. */
2059 if (!determine_crypto_len(h
, &chdr
, space_left
, &hdr_bytes
,
2061 return 1; /* can't fit anything */
2064 * Truncate IOVs to match our chosen length.
2066 * The length cannot be more than SIZE_MAX because this length comes
2067 * from our send stream buffer.
2069 ossl_quic_sstream_adjust_iov((size_t)chdr
.len
, iov
, num_stream_iovec
);
2072 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2075 if (!txp_el_ensure_iovec(&txp
->el
[enc_level
], h
->num_iovec
+ 3))
2076 return 0; /* alloc error */
2078 /* Encode the header. */
2079 wpkt
= tx_helper_begin(h
);
2081 return 0; /* alloc error */
2083 if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt
, &chdr
)) {
2084 tx_helper_rollback(h
);
2085 return 1; /* can't fit */
2088 if (!tx_helper_commit(h
))
2089 return 0; /* alloc error */
2091 /* Add payload iovecs to the helper (infallible). */
2092 for (i
= 0; i
< num_stream_iovec
; ++i
)
2093 tx_helper_append_iovec(h
, iov
[i
].buf
, iov
[i
].buf_len
);
2095 *have_ack_eliciting
= 1;
2096 tx_helper_unrestrict(h
); /* no longer need PING */
2098 /* Log chunk to TXPIM. */
2099 chunk
.stream_id
= UINT64_MAX
; /* crypto stream */
2100 chunk
.start
= chdr
.offset
;
2101 chunk
.end
= chdr
.offset
+ chdr
.len
- 1;
2102 chunk
.has_fin
= 0; /* Crypto stream never ends */
2103 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2104 return 0; /* alloc error */
2109 OSSL_QUIC_FRAME_STREAM shdr
;
2111 OSSL_QTX_IOVEC iov
[2];
2112 size_t num_stream_iovec
;
2116 static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER
*txp
,
2117 struct tx_helper
*h
,
2118 QUIC_SSTREAM
*sstream
,
2119 QUIC_TXFC
*stream_txfc
,
2121 struct chunk_info
*chunk
,
2124 uint64_t fc_credit
, fc_swm
, fc_limit
;
2126 chunk
->num_stream_iovec
= OSSL_NELEM(chunk
->iov
);
2127 chunk
->valid
= ossl_quic_sstream_get_stream_frame(sstream
, skip
,
2130 &chunk
->num_stream_iovec
);
2134 if (!ossl_assert(chunk
->shdr
.len
> 0 || chunk
->shdr
.is_fin
))
2135 /* Should only have 0-length chunk if FIN */
2138 chunk
->orig_len
= chunk
->shdr
.len
;
2140 /* Clamp according to connection and stream-level TXFC. */
2141 fc_credit
= ossl_quic_txfc_get_credit(stream_txfc
, consumed
);
2142 fc_swm
= ossl_quic_txfc_get_swm(stream_txfc
);
2143 fc_limit
= fc_swm
+ fc_credit
;
2145 if (chunk
->shdr
.len
> 0 && chunk
->shdr
.offset
+ chunk
->shdr
.len
> fc_limit
) {
2146 chunk
->shdr
.len
= (fc_limit
<= chunk
->shdr
.offset
)
2147 ? 0 : fc_limit
- chunk
->shdr
.offset
;
2148 chunk
->shdr
.is_fin
= 0;
2151 if (chunk
->shdr
.len
== 0 && !chunk
->shdr
.is_fin
) {
2153 * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
2154 * order of offset we don't need to check any later chunks, so stop
2165 * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
2166 * *packet_full is set to 1 if there is no longer enough room for another STREAM
2169 static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER
*txp
,
2170 struct txp_pkt
*pkt
,
2172 QUIC_SSTREAM
*sstream
,
2173 QUIC_TXFC
*stream_txfc
,
2174 QUIC_STREAM
*next_stream
,
2175 int *have_ack_eliciting
,
2177 uint64_t *new_credit_consumed
,
2178 uint64_t conn_consumed
)
2181 struct chunk_info chunks
[2] = {0};
2182 const uint32_t enc_level
= pkt
->h
.enc_level
;
2183 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2184 struct tx_helper
*h
= &pkt
->h
;
2185 OSSL_QUIC_FRAME_STREAM
*shdr
;
2187 QUIC_TXPIM_CHUNK chunk
;
2188 size_t i
, j
, space_left
;
2189 int can_fill_payload
, use_explicit_len
;
2190 int could_have_following_chunk
;
2192 uint64_t hdr_len_implicit
, payload_len_implicit
;
2193 uint64_t hdr_len_explicit
, payload_len_explicit
;
2194 uint64_t fc_swm
, fc_new_hwm
;
2196 fc_swm
= ossl_quic_txfc_get_swm(stream_txfc
);
2197 fc_new_hwm
= fc_swm
;
2200 * Load the first two chunks if any offered by the send stream. We retrieve
2201 * the next chunk in advance so we can determine if we need to send any more
2202 * chunks from the same stream after this one, which is needed when
2203 * determining when we can use an implicit length in a STREAM frame.
2205 for (i
= 0; i
< 2; ++i
) {
2206 if (!txp_plan_stream_chunk(txp
, h
, sstream
, stream_txfc
, i
, &chunks
[i
],
2210 if (i
== 0 && !chunks
[i
].valid
) {
2211 /* No chunks, nothing to do. */
2218 space_left
= tx_helper_get_space_left(h
);
2220 if (!chunks
[i
% 2].valid
) {
2221 /* Out of chunks; we're done. */
2226 if (space_left
< MIN_FRAME_SIZE_STREAM
) {
2232 if (!ossl_assert(!h
->done_implicit
))
2234 * Logic below should have ensured we didn't append an
2235 * implicit-length unless we filled the packet or didn't have
2236 * another stream to handle, so this should not be possible.
2240 shdr
= &chunks
[i
% 2].shdr
;
2241 orig_len
= chunks
[i
% 2].orig_len
;
2243 /* Load next chunk for lookahead. */
2244 if (!txp_plan_stream_chunk(txp
, h
, sstream
, stream_txfc
, i
+ 1,
2245 &chunks
[(i
+ 1) % 2], conn_consumed
))
2249 * Find best fit (header length, payload length) combination for if we
2250 * use an implicit length.
2252 shdr
->has_explicit_len
= 0;
2253 hdr_len_implicit
= payload_len_implicit
= 0;
2254 if (!determine_stream_len(h
, shdr
, space_left
,
2255 &hdr_len_implicit
, &payload_len_implicit
)) {
2258 goto err
; /* can't fit anything */
2262 * If there is a next stream, we don't use the implicit length so we can
2263 * add more STREAM frames after this one, unless there is enough data
2264 * for this STREAM frame to fill the packet.
2266 can_fill_payload
= (hdr_len_implicit
+ payload_len_implicit
2270 * Is there is a stream after this one, or another chunk pending
2271 * transmission in this stream?
2273 could_have_following_chunk
2274 = (next_stream
!= NULL
|| chunks
[(i
+ 1) % 2].valid
);
2276 /* Choose between explicit or implicit length representations. */
2277 use_explicit_len
= !((can_fill_payload
|| !could_have_following_chunk
)
2278 && !pkt
->force_pad
);
2280 if (use_explicit_len
) {
2282 * Find best fit (header length, payload length) combination for if
2283 * we use an explicit length.
2285 shdr
->has_explicit_len
= 1;
2286 hdr_len_explicit
= payload_len_explicit
= 0;
2287 if (!determine_stream_len(h
, shdr
, space_left
,
2288 &hdr_len_explicit
, &payload_len_explicit
)) {
2291 goto err
; /* can't fit anything */
2294 shdr
->len
= payload_len_explicit
;
2297 shdr
->has_explicit_len
= 0;
2298 shdr
->len
= payload_len_implicit
;
2301 /* If this is a FIN, don't keep filling the packet with more FINs. */
2303 chunks
[(i
+ 1) % 2].valid
= 0;
2306 * We are now committed to our length (shdr->len can't change).
2307 * If we truncated the chunk, clear the FIN bit.
2309 if (shdr
->len
< orig_len
)
2312 /* Truncate IOVs to match our chosen length. */
2313 ossl_quic_sstream_adjust_iov((size_t)shdr
->len
, chunks
[i
% 2].iov
,
2314 chunks
[i
% 2].num_stream_iovec
);
2317 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2320 if (!txp_el_ensure_iovec(&txp
->el
[enc_level
], h
->num_iovec
+ 3))
2321 goto err
; /* alloc error */
2323 /* Encode the header. */
2324 wpkt
= tx_helper_begin(h
);
2326 goto err
; /* alloc error */
2328 shdr
->stream_id
= id
;
2329 if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt
, shdr
))) {
2330 /* (Should not be possible.) */
2331 tx_helper_rollback(h
);
2334 goto err
; /* can't fit */
2337 if (!tx_helper_commit(h
))
2338 goto err
; /* alloc error */
2340 /* Add payload iovecs to the helper (infallible). */
2341 for (j
= 0; j
< chunks
[i
% 2].num_stream_iovec
; ++j
)
2342 tx_helper_append_iovec(h
, chunks
[i
% 2].iov
[j
].buf
,
2343 chunks
[i
% 2].iov
[j
].buf_len
);
2345 *have_ack_eliciting
= 1;
2346 tx_helper_unrestrict(h
); /* no longer need PING */
2347 if (!shdr
->has_explicit_len
)
2348 h
->done_implicit
= 1;
2350 /* Log new TXFC credit which was consumed. */
2351 if (shdr
->len
> 0 && shdr
->offset
+ shdr
->len
> fc_new_hwm
)
2352 fc_new_hwm
= shdr
->offset
+ shdr
->len
;
2354 /* Log chunk to TXPIM. */
2355 chunk
.stream_id
= shdr
->stream_id
;
2356 chunk
.start
= shdr
->offset
;
2357 chunk
.end
= shdr
->offset
+ shdr
->len
- 1;
2358 chunk
.has_fin
= shdr
->is_fin
;
2359 chunk
.has_stop_sending
= 0;
2360 chunk
.has_reset_stream
= 0;
2361 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2362 goto err
; /* alloc error */
2364 if (shdr
->len
< orig_len
) {
2366 * If we did not serialize all of this chunk we definitely do not
2367 * want to try the next chunk
2375 *new_credit_consumed
= fc_new_hwm
- fc_swm
;
2379 static void txp_enlink_tmp(QUIC_STREAM
**tmp_head
, QUIC_STREAM
*stream
)
2381 stream
->txp_next
= *tmp_head
;
2385 static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER
*txp
,
2386 struct txp_pkt
*pkt
,
2387 int *have_ack_eliciting
,
2388 QUIC_STREAM
**tmp_head
)
2390 QUIC_STREAM_ITER it
;
2393 QUIC_STREAM
*stream
, *snext
;
2394 struct tx_helper
*h
= &pkt
->h
;
2395 uint64_t conn_consumed
= 0;
2397 for (ossl_quic_stream_iter_init(&it
, txp
->args
.qsm
, 1);
2398 it
.stream
!= NULL
;) {
2401 ossl_quic_stream_iter_next(&it
);
2404 stream
->txp_sent_fc
= 0;
2405 stream
->txp_sent_stop_sending
= 0;
2406 stream
->txp_sent_reset_stream
= 0;
2407 stream
->txp_blocked
= 0;
2408 stream
->txp_txfc_new_credit_consumed
= 0;
2410 /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
2411 if (stream
->want_stop_sending
) {
2412 OSSL_QUIC_FRAME_STOP_SENDING f
;
2414 wpkt
= tx_helper_begin(h
);
2416 return 0; /* alloc error */
2418 f
.stream_id
= stream
->id
;
2419 f
.app_error_code
= stream
->stop_sending_aec
;
2420 if (!ossl_quic_wire_encode_frame_stop_sending(wpkt
, &f
)) {
2421 tx_helper_rollback(h
); /* can't fit */
2422 txp_enlink_tmp(tmp_head
, stream
);
2426 if (!tx_helper_commit(h
))
2427 return 0; /* alloc error */
2429 *have_ack_eliciting
= 1;
2430 tx_helper_unrestrict(h
); /* no longer need PING */
2431 stream
->txp_sent_stop_sending
= 1;
2434 if (stream
->want_reset_stream
) {
2435 OSSL_QUIC_FRAME_RESET_STREAM f
;
2437 if (!ossl_assert(stream
->send_state
== QUIC_SSTREAM_STATE_RESET_SENT
))
2440 wpkt
= tx_helper_begin(h
);
2442 return 0; /* alloc error */
2444 f
.stream_id
= stream
->id
;
2445 f
.app_error_code
= stream
->reset_stream_aec
;
2446 if (!ossl_quic_stream_send_get_final_size(stream
, &f
.final_size
))
2447 return 0; /* should not be possible */
2449 if (!ossl_quic_wire_encode_frame_reset_stream(wpkt
, &f
)) {
2450 tx_helper_rollback(h
); /* can't fit */
2451 txp_enlink_tmp(tmp_head
, stream
);
2455 if (!tx_helper_commit(h
))
2456 return 0; /* alloc error */
2458 *have_ack_eliciting
= 1;
2459 tx_helper_unrestrict(h
); /* no longer need PING */
2460 stream
->txp_sent_reset_stream
= 1;
2463 * The final size of the stream as indicated by RESET_STREAM is used
2464 * to ensure a consistent view of flow control state by both
2465 * parties; if we happen to send a RESET_STREAM that consumes more
2466 * flow control credit, make sure we account for that.
2468 if (!ossl_assert(f
.final_size
<= ossl_quic_txfc_get_swm(&stream
->txfc
)))
2471 stream
->txp_txfc_new_credit_consumed
2472 = f
.final_size
- ossl_quic_txfc_get_swm(&stream
->txfc
);
2476 * Stream Flow Control Frames (MAX_STREAM_DATA)
2478 * RFC 9000 s. 13.3: "An endpoint SHOULD stop sending MAX_STREAM_DATA
2479 * frames when the receiving part of the stream enters a "Size Known" or
2480 * "Reset Recvd" state." -- In practice, RECV is the only state
2481 * in which it makes sense to generate more MAX_STREAM_DATA frames.
2483 if (stream
->recv_state
== QUIC_RSTREAM_STATE_RECV
2484 && (stream
->want_max_stream_data
2485 || ossl_quic_rxfc_has_cwm_changed(&stream
->rxfc
, 0))) {
2487 wpkt
= tx_helper_begin(h
);
2489 return 0; /* alloc error */
2491 cwm
= ossl_quic_rxfc_get_cwm(&stream
->rxfc
);
2493 if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt
, stream
->id
,
2495 tx_helper_rollback(h
); /* can't fit */
2496 txp_enlink_tmp(tmp_head
, stream
);
2500 if (!tx_helper_commit(h
))
2501 return 0; /* alloc error */
2503 *have_ack_eliciting
= 1;
2504 tx_helper_unrestrict(h
); /* no longer need PING */
2505 stream
->txp_sent_fc
= 1;
2509 * Stream Data Frames (STREAM)
2511 * RFC 9000 s. 3.3: A sender MUST NOT send a STREAM [...] frame for a
2512 * stream in the "Reset Sent" state [or any terminal state]. We don't
2513 * send any more STREAM frames if we are sending, have sent, or are
2514 * planning to send, RESET_STREAM. The other terminal state is Data
2515 * Recvd, but txp_generate_stream_frames() is guaranteed to generate
2516 * nothing in this case.
2518 if (ossl_quic_stream_has_send_buffer(stream
)
2519 && !ossl_quic_stream_send_is_reset(stream
)) {
2520 int packet_full
= 0;
2522 if (!ossl_assert(!stream
->want_reset_stream
))
2525 if (!txp_generate_stream_frames(txp
, pkt
,
2526 stream
->id
, stream
->sstream
,
2531 &stream
->txp_txfc_new_credit_consumed
,
2533 /* Fatal error (allocation, etc.) */
2534 txp_enlink_tmp(tmp_head
, stream
);
2537 conn_consumed
+= stream
->txp_txfc_new_credit_consumed
;
2540 txp_enlink_tmp(tmp_head
, stream
);
2545 txp_enlink_tmp(tmp_head
, stream
);
2551 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER
*txp
,
2552 struct txp_pkt
*pkt
,
2553 int chosen_for_conn_close
)
2555 int rc
= TXP_ERR_SUCCESS
;
2556 const uint32_t enc_level
= pkt
->h
.enc_level
;
2557 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2558 int have_ack_eliciting
= 0, done_pre_token
= 0;
2559 const struct archetype_data a
= pkt
->geom
.adata
;
2561 * Cleared if we encode any non-ACK-eliciting frame type which rules out the
2562 * packet being a non-inflight frame. This means any non-ACK ACK-eliciting
2563 * frame, even PADDING frames. ACK eliciting frames always cause a packet to
2564 * become ineligible for non-inflight treatment so it is not necessary to
2565 * clear this in cases where have_ack_eliciting is set, as it is ignored in
2568 int can_be_non_inflight
= 1;
2569 QUIC_CFQ_ITEM
*cfq_item
;
2570 QUIC_TXPIM_PKT
*tpkt
= NULL
;
2571 struct tx_helper
*h
= &pkt
->h
;
2573 /* Maximum PN reached? */
2574 if (!ossl_quic_pn_valid(txp
->next_pn
[pn_space
]))
2577 if (!ossl_assert(pkt
->tpkt
== NULL
))
2580 if ((pkt
->tpkt
= tpkt
= ossl_quic_txpim_pkt_alloc(txp
->args
.txpim
)) == NULL
)
2584 * Frame Serialization
2585 * ===================
2587 * We now serialize frames into the packet in descending order of priority.
2590 /* HANDSHAKE_DONE (Regenerate) */
2591 if (a
.allow_handshake_done
&& txp
->want_handshake_done
2592 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_HANDSHAKE_DONE
) {
2593 WPACKET
*wpkt
= tx_helper_begin(h
);
2598 if (ossl_quic_wire_encode_frame_handshake_done(wpkt
)) {
2599 tpkt
->had_handshake_done_frame
= 1;
2600 have_ack_eliciting
= 1;
2602 if (!tx_helper_commit(h
))
2605 tx_helper_unrestrict(h
); /* no longer need PING */
2607 tx_helper_rollback(h
);
2611 /* MAX_DATA (Regenerate) */
2613 && (txp
->want_max_data
2614 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 0))
2615 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_DATA
) {
2616 WPACKET
*wpkt
= tx_helper_begin(h
);
2617 uint64_t cwm
= ossl_quic_rxfc_get_cwm(txp
->args
.conn_rxfc
);
2622 if (ossl_quic_wire_encode_frame_max_data(wpkt
, cwm
)) {
2623 tpkt
->had_max_data_frame
= 1;
2624 have_ack_eliciting
= 1;
2626 if (!tx_helper_commit(h
))
2629 tx_helper_unrestrict(h
); /* no longer need PING */
2631 tx_helper_rollback(h
);
2635 /* MAX_STREAMS_BIDI (Regenerate) */
2637 && (txp
->want_max_streams_bidi
2638 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
, 0))
2639 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI
) {
2640 WPACKET
*wpkt
= tx_helper_begin(h
);
2641 uint64_t max_streams
2642 = ossl_quic_rxfc_get_cwm(txp
->args
.max_streams_bidi_rxfc
);
2647 if (ossl_quic_wire_encode_frame_max_streams(wpkt
, /*is_uni=*/0,
2649 tpkt
->had_max_streams_bidi_frame
= 1;
2650 have_ack_eliciting
= 1;
2652 if (!tx_helper_commit(h
))
2655 tx_helper_unrestrict(h
); /* no longer need PING */
2657 tx_helper_rollback(h
);
2661 /* MAX_STREAMS_UNI (Regenerate) */
2663 && (txp
->want_max_streams_uni
2664 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
, 0))
2665 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI
) {
2666 WPACKET
*wpkt
= tx_helper_begin(h
);
2667 uint64_t max_streams
2668 = ossl_quic_rxfc_get_cwm(txp
->args
.max_streams_uni_rxfc
);
2673 if (ossl_quic_wire_encode_frame_max_streams(wpkt
, /*is_uni=*/1,
2675 tpkt
->had_max_streams_uni_frame
= 1;
2676 have_ack_eliciting
= 1;
2678 if (!tx_helper_commit(h
))
2681 tx_helper_unrestrict(h
); /* no longer need PING */
2683 tx_helper_rollback(h
);
2688 for (cfq_item
= ossl_quic_cfq_get_priority_head(txp
->args
.cfq
, pn_space
);
2690 cfq_item
= ossl_quic_cfq_item_get_priority_next(cfq_item
, pn_space
)) {
2691 uint64_t frame_type
= ossl_quic_cfq_item_get_frame_type(cfq_item
);
2692 const unsigned char *encoded
= ossl_quic_cfq_item_get_encoded(cfq_item
);
2693 size_t encoded_len
= ossl_quic_cfq_item_get_encoded_len(cfq_item
);
2695 switch (frame_type
) {
2696 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID
:
2697 if (!a
.allow_new_conn_id
)
2700 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID
:
2701 if (!a
.allow_retire_conn_id
)
2704 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN
:
2705 if (!a
.allow_new_token
)
2709 * NEW_TOKEN frames are handled via GCR, but some
2710 * Regenerate-strategy frames should come before them (namely
2711 * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
2712 * we find a NEW_TOKEN frame, do these now. If there are no
2713 * NEW_TOKEN frames in the GCR queue we will handle these below.
2715 if (!done_pre_token
)
2716 if (txp_generate_pre_token(txp
, pkt
,
2717 chosen_for_conn_close
,
2718 &can_be_non_inflight
))
2722 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE
:
2723 if (!a
.allow_path_response
)
2727 * RFC 9000 s. 8.2.2: An endpoint MUST expand datagrams that
2728 * contain a PATH_RESPONSE frame to at least the smallest
2729 * allowed maximum datagram size of 1200 bytes.
2734 if (!a
.allow_cfq_other
)
2740 * If the frame is too big, don't try to schedule any more GCR frames in
2741 * this packet rather than sending subsequent ones out of order.
2743 if (encoded_len
> tx_helper_get_space_left(h
))
2746 if (!tx_helper_append_iovec(h
, encoded
, encoded_len
))
2749 ossl_quic_txpim_pkt_add_cfq_item(tpkt
, cfq_item
);
2751 if (ossl_quic_frame_type_is_ack_eliciting(frame_type
)) {
2752 have_ack_eliciting
= 1;
2753 tx_helper_unrestrict(h
); /* no longer need PING */
2758 * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
2759 * PATH_RESPONSE (as desired) before, do so now.
2761 if (!done_pre_token
)
2762 if (txp_generate_pre_token(txp
, pkt
,
2763 chosen_for_conn_close
,
2764 &can_be_non_inflight
))
2769 if (!txp_generate_crypto_frames(txp
, pkt
, &have_ack_eliciting
))
2772 /* Stream-specific frames */
2773 if (a
.allow_stream_rel
&& txp
->handshake_complete
)
2774 if (!txp_generate_stream_related(txp
, pkt
,
2775 &have_ack_eliciting
,
2780 tx_helper_unrestrict(h
);
2782 if ((a
.require_ack_eliciting
2783 || (txp
->force_ack_eliciting
& (1UL << pn_space
)) != 0)
2784 && !have_ack_eliciting
&& a
.allow_ping
) {
2787 wpkt
= tx_helper_begin(h
);
2791 if (!ossl_quic_wire_encode_frame_ping(wpkt
)
2792 || !tx_helper_commit(h
))
2794 * We treat a request to be ACK-eliciting as a requirement, so this
2799 have_ack_eliciting
= 1;
2802 /* PADDING is added by ossl_quic_tx_packetiser_generate(). */
2808 if (have_ack_eliciting
)
2809 can_be_non_inflight
= 0;
2812 tpkt
->ackm_pkt
.num_bytes
= h
->bytes_appended
+ pkt
->geom
.pkt_overhead
;
2813 tpkt
->ackm_pkt
.pkt_num
= txp
->next_pn
[pn_space
];
2814 /* largest_acked is set in txp_generate_pre_token */
2815 tpkt
->ackm_pkt
.pkt_space
= pn_space
;
2816 tpkt
->ackm_pkt
.is_inflight
= !can_be_non_inflight
;
2817 tpkt
->ackm_pkt
.is_ack_eliciting
= have_ack_eliciting
;
2818 tpkt
->ackm_pkt
.is_pto_probe
= 0;
2819 tpkt
->ackm_pkt
.is_mtu_probe
= 0;
2820 tpkt
->ackm_pkt
.time
= txp
->args
.now(txp
->args
.now_arg
);
2821 tpkt
->pkt_type
= pkt
->phdr
.type
;
2828 * Handler for fatal errors, i.e. errors causing us to abort the entire
2829 * packet rather than just one frame. Examples of such errors include
2830 * allocation errors.
2833 ossl_quic_txpim_pkt_release(txp
->args
.txpim
, tpkt
);
2836 return TXP_ERR_INTERNAL
;
2840 * Commits and queues a packet for transmission. There is no backing out after
2845 * - Sends the packet to the QTX for encryption and transmission;
2847 * - Records the packet as having been transmitted in FIFM. ACKM is informed,
2848 * etc. and the TXPIM record is filed.
2850 * - Informs various subsystems of frames that were sent and clears frame
2851 * wanted flags so that we do not generate the same frames again.
2855 * - pkt is a txp_pkt for the correct EL;
2857 * - pkt->tpkt is valid;
2859 * - pkt->tpkt->ackm_pkt has been fully filled in;
2861 * - Stream chunk records have been appended to pkt->tpkt for STREAM and
2862 * CRYPTO frames, but not for RESET_STREAM or STOP_SENDING frames;
2864 * - The chosen stream list for the packet can be fully walked from
2865 * pkt->stream_head using stream->txp_next;
2867 * - pkt->has_ack_eliciting is set correctly.
2870 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER
*txp
,
2871 struct txp_pkt
*pkt
,
2873 int *txpim_pkt_reffed
)
2876 uint32_t enc_level
= pkt
->h
.enc_level
;
2877 uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2878 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2879 QUIC_STREAM
*stream
;
2881 struct archetype_data a
;
2883 *txpim_pkt_reffed
= 0;
2885 /* Cannot send a packet with an empty payload. */
2886 if (pkt
->h
.bytes_appended
== 0)
2889 if (!txp_get_archetype_data(enc_level
, archetype
, &a
))
2892 /* Packet Information for QTX */
2893 txpkt
.hdr
= &pkt
->phdr
;
2894 txpkt
.iovec
= txp
->el
[enc_level
].iovec
;
2895 txpkt
.num_iovec
= pkt
->h
.num_iovec
;
2897 txpkt
.peer
= BIO_ADDR_family(&txp
->args
.peer
) == AF_UNSPEC
2898 ? NULL
: &txp
->args
.peer
;
2899 txpkt
.pn
= txp
->next_pn
[pn_space
];
2900 txpkt
.flags
= OSSL_QTX_PKT_FLAG_COALESCE
; /* always try to coalesce */
2902 /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
2903 for (stream
= pkt
->stream_head
; stream
!= NULL
; stream
= stream
->txp_next
)
2904 if (stream
->txp_sent_stop_sending
|| stream
->txp_sent_reset_stream
) {
2905 /* Log STOP_SENDING/RESET_STREAM chunk to TXPIM. */
2906 QUIC_TXPIM_CHUNK chunk
;
2908 chunk
.stream_id
= stream
->id
;
2909 chunk
.start
= UINT64_MAX
;
2912 chunk
.has_stop_sending
= stream
->txp_sent_stop_sending
;
2913 chunk
.has_reset_stream
= stream
->txp_sent_reset_stream
;
2914 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2915 return 0; /* alloc error */
2918 /* Dispatch to FIFD. */
2919 if (!ossl_quic_fifd_pkt_commit(&txp
->fifd
, tpkt
))
2923 * Transmission and Post-Packet Generation Bookkeeping
2924 * ===================================================
2926 * No backing out anymore - at this point the ACKM has recorded the packet
2927 * as having been sent, so we need to increment our next PN counter, or
2928 * the ACKM will complain when we try to record a duplicate packet with
2929 * the same PN later. At this point actually sending the packet may still
2930 * fail. In this unlikely event it will simply be handled as though it
2931 * were a lost packet.
2933 ++txp
->next_pn
[pn_space
];
2934 *txpim_pkt_reffed
= 1;
2936 /* Send the packet. */
2937 if (!ossl_qtx_write_pkt(txp
->args
.qtx
, &txpkt
))
2941 * Record FC and stream abort frames as sent; deactivate streams which no
2942 * longer have anything to do.
2944 for (stream
= pkt
->stream_head
; stream
!= NULL
; stream
= stream
->txp_next
) {
2945 if (stream
->txp_sent_fc
) {
2946 stream
->want_max_stream_data
= 0;
2947 ossl_quic_rxfc_has_cwm_changed(&stream
->rxfc
, 1);
2950 if (stream
->txp_sent_stop_sending
)
2951 stream
->want_stop_sending
= 0;
2953 if (stream
->txp_sent_reset_stream
)
2954 stream
->want_reset_stream
= 0;
2956 if (stream
->txp_txfc_new_credit_consumed
> 0) {
2957 if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream
->txfc
,
2958 stream
->txp_txfc_new_credit_consumed
)))
2960 * Should not be possible, but we should continue with our
2961 * bookkeeping as we have already committed the packet to the
2962 * FIFD. Just change the value we return.
2966 stream
->txp_txfc_new_credit_consumed
= 0;
2970 * If we no longer need to generate any flow control (MAX_STREAM_DATA),
2971 * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
2972 * the stream is drained of data or TXFC-blocked), we can mark the
2973 * stream as inactive.
2975 ossl_quic_stream_map_update_state(txp
->args
.qsm
, stream
);
2977 if (ossl_quic_stream_has_send_buffer(stream
)
2978 && !ossl_quic_sstream_has_pending(stream
->sstream
)
2979 && ossl_quic_sstream_get_final_size(stream
->sstream
, NULL
))
2981 * Transition to DATA_SENT if stream has a final size and we have
2984 ossl_quic_stream_map_notify_all_data_sent(txp
->args
.qsm
, stream
);
2987 /* We have now sent the packet, so update state accordingly. */
2988 if (tpkt
->ackm_pkt
.is_ack_eliciting
)
2989 txp
->force_ack_eliciting
&= ~(1UL << pn_space
);
2991 if (tpkt
->had_handshake_done_frame
)
2992 txp
->want_handshake_done
= 0;
2994 if (tpkt
->had_max_data_frame
) {
2995 txp
->want_max_data
= 0;
2996 ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 1);
2999 if (tpkt
->had_max_streams_bidi_frame
) {
3000 txp
->want_max_streams_bidi
= 0;
3001 ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
, 1);
3004 if (tpkt
->had_max_streams_uni_frame
) {
3005 txp
->want_max_streams_uni
= 0;
3006 ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
, 1);
3009 if (tpkt
->had_ack_frame
)
3010 txp
->want_ack
&= ~(1UL << pn_space
);
3012 if (tpkt
->had_conn_close
)
3013 txp
->want_conn_close
= 0;
3016 * Decrement probe request counts if we have sent a packet that meets
3017 * the requirement of a probe, namely being ACK-eliciting.
3019 if (tpkt
->ackm_pkt
.is_ack_eliciting
) {
3020 OSSL_ACKM_PROBE_INFO
*probe_info
3021 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
3023 if (enc_level
== QUIC_ENC_LEVEL_INITIAL
3024 && probe_info
->anti_deadlock_initial
> 0)
3025 --probe_info
->anti_deadlock_initial
;
3027 if (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
3028 && probe_info
->anti_deadlock_handshake
> 0)
3029 --probe_info
->anti_deadlock_handshake
;
3031 if (a
.allow_force_ack_eliciting
/* (i.e., not for 0-RTT) */
3032 && probe_info
->pto
[pn_space
] > 0)
3033 --probe_info
->pto
[pn_space
];
3039 /* Ensure the iovec array is at least num elements long. */
3040 static int txp_el_ensure_iovec(struct txp_el
*el
, size_t num
)
3042 OSSL_QTX_IOVEC
*iovec
;
3044 if (el
->alloc_iovec
>= num
)
3047 num
= el
->alloc_iovec
!= 0 ? el
->alloc_iovec
* 2 : 8;
3049 iovec
= OPENSSL_realloc(el
->iovec
, sizeof(OSSL_QTX_IOVEC
) * num
);
3054 el
->alloc_iovec
= num
;
3058 int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER
*txp
,
3059 const OSSL_QUIC_FRAME_CONN_CLOSE
*f
)
3061 char *reason
= NULL
;
3062 size_t reason_len
= f
->reason_len
;
3063 size_t max_reason_len
= txp_get_mdpl(txp
) / 2;
3065 if (txp
->want_conn_close
)
3069 * Arbitrarily limit the length of the reason length string to half of the
3072 if (reason_len
> max_reason_len
)
3073 reason_len
= max_reason_len
;
3075 if (reason_len
> 0) {
3076 reason
= OPENSSL_memdup(f
->reason
, reason_len
);
3081 txp
->conn_close_frame
= *f
;
3082 txp
->conn_close_frame
.reason
= reason
;
3083 txp
->conn_close_frame
.reason_len
= reason_len
;
3084 txp
->want_conn_close
= 1;
3088 void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER
*txp
,
3089 ossl_msg_cb msg_callback
,
3090 SSL
*msg_callback_ssl
)
3092 txp
->msg_callback
= msg_callback
;
3093 txp
->msg_callback_ssl
= msg_callback_ssl
;
3096 void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER
*txp
,
3097 void *msg_callback_arg
)
3099 txp
->msg_callback_arg
= msg_callback_arg
;
3102 QUIC_PN
ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER
*txp
,
3105 if (pn_space
>= QUIC_PN_SPACE_NUM
)
3108 return txp
->next_pn
[pn_space
];
3111 OSSL_TIME
ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER
*txp
)
3114 * TXP-specific deadline computations which rely on TXP innards. This is in
3115 * turn relied on by the QUIC_CHANNEL code to determine the channel event
3116 * handling deadline.
3118 OSSL_TIME deadline
= ossl_time_infinite();
3119 uint32_t enc_level
, pn_space
;
3122 * ACK generation is not CC-gated - packets containing only ACKs are allowed
3123 * to bypass CC. We want to generate ACK frames even if we are currently
3124 * restricted by CC so the peer knows we have received data. The generate
3125 * call will take care of selecting the correct packet archetype.
3127 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
3128 enc_level
< QUIC_ENC_LEVEL_NUM
;
3130 if (ossl_qtx_is_enc_level_provisioned(txp
->args
.qtx
, enc_level
)) {
3131 pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
3132 deadline
= ossl_time_min(deadline
,
3133 ossl_ackm_get_ack_deadline(txp
->args
.ackm
, pn_space
));
3136 /* When will CC let us send more? */
3137 if (txp
->args
.cc_method
->get_tx_allowance(txp
->args
.cc_data
) == 0)
3138 deadline
= ossl_time_min(deadline
,
3139 txp
->args
.cc_method
->get_wakeup_deadline(txp
->args
.cc_data
));