2 * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #include "internal/quic_txp.h"
11 #include "internal/quic_fifd.h"
12 #include "internal/quic_stream_map.h"
13 #include "internal/quic_error.h"
14 #include "internal/common.h"
15 #include <openssl/err.h>
17 #define MIN_CRYPTO_HDR_SIZE 3
19 #define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
20 #define MIN_FRAME_SIZE_MAX_DATA 2
21 #define MIN_FRAME_SIZE_ACK 5
22 #define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
23 #define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
24 #define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
25 #define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
32 /* Generate normal packets containing most frame types, subject to EL. */
33 #define TX_PACKETISER_ARCHETYPE_NORMAL 0
36 * A probe packet is different in that:
37 * - It bypasses CC, but *is* counted as in flight for purposes of CC;
38 * - It must be ACK-eliciting.
40 #define TX_PACKETISER_ARCHETYPE_PROBE 1
43 * An ACK-only packet is different in that:
44 * - It bypasses CC, and is considered a 'non-inflight' packet;
45 * - It may not contain anything other than an ACK frame, not even padding.
47 #define TX_PACKETISER_ARCHETYPE_ACK_ONLY 2
49 #define TX_PACKETISER_ARCHETYPE_NUM 3
51 struct ossl_quic_tx_packetiser_st
{
52 OSSL_QUIC_TX_PACKETISER_ARGS args
;
55 * Opaque initial token blob provided by caller. TXP frees using the
56 * callback when it is no longer needed.
58 const unsigned char *initial_token
;
59 size_t initial_token_len
;
60 ossl_quic_initial_token_free_fn
*initial_token_free_cb
;
61 void *initial_token_free_cb_arg
;
63 /* Subcomponents of the TXP that we own. */
64 QUIC_FIFD fifd
; /* QUIC Frame-in-Flight Dispatcher */
67 uint64_t next_pn
[QUIC_PN_SPACE_NUM
]; /* Next PN to use in given PN space. */
68 OSSL_TIME last_tx_time
; /* Last time a packet was generated, or 0. */
70 /* Internal state - frame (re)generation flags. */
71 unsigned int want_handshake_done
: 1;
72 unsigned int want_max_data
: 1;
73 unsigned int want_max_streams_bidi
: 1;
74 unsigned int want_max_streams_uni
: 1;
76 /* Internal state - frame (re)generation flags - per PN space. */
77 unsigned int want_ack
: QUIC_PN_SPACE_NUM
;
78 unsigned int force_ack_eliciting
: QUIC_PN_SPACE_NUM
;
81 * Internal state - connection close terminal state.
82 * Once this is set, it is not unset unlike other want_ flags - we keep
83 * sending it in every packet.
85 unsigned int want_conn_close
: 1;
87 /* Has the handshake been completed? */
88 unsigned int handshake_complete
: 1;
90 OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame
;
93 * Counts of the number of bytes received and sent while in the closing
96 uint64_t closing_bytes_recv
;
97 uint64_t closing_bytes_xmit
;
99 /* Internal state - packet assembly. */
101 unsigned char *scratch
; /* scratch buffer for packet assembly */
102 size_t scratch_len
; /* number of bytes allocated for scratch */
103 OSSL_QTX_IOVEC
*iovec
; /* scratch iovec array for use with QTX */
104 size_t alloc_iovec
; /* size of iovec array */
105 } el
[QUIC_ENC_LEVEL_NUM
];
107 /* Message callback related arguments */
108 ossl_msg_cb msg_callback
;
109 void *msg_callback_arg
;
110 SSL
*msg_callback_ssl
;
113 void (*ack_tx_cb
)(const OSSL_QUIC_FRAME_ACK
*ack
,
120 * The TX helper records state used while generating frames into packets. It
121 * enables serialization into the packet to be done "transactionally" where
122 * serialization of a frame can be rolled back if it fails midway (e.g. if it
126 OSSL_QUIC_TX_PACKETISER
*txp
;
128 * The Maximum Packet Payload Length in bytes. This is the amount of
129 * space we have to generate frames into.
133 * Number of bytes we have generated so far.
135 size_t bytes_appended
;
137 * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
138 * will reference this scratch buffer. When we need to use more of it (e.g.
139 * when we need to put frame headers somewhere), we append to the scratch
140 * buffer, resizing if necessary, and increase this accordingly.
142 size_t scratch_bytes
;
144 * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
145 * until reserve_allowed is set to 1. Currently this is always at most 1, as
146 * a PING frame takes up one byte and this mechanism is only used to ensure
147 * we can encode a PING frame if we have been asked to ensure a packet is
148 * ACK-eliciting and we are unusure if we are going to add any other
149 * ACK-eliciting frames before we reach our MaxPPL budget.
153 * Number of iovecs we have currently appended. This is the number of
154 * entries valid in txp->iovec.
157 /* The EL this TX helper is being used for. */
160 * Whether we are allowed to make use of the reserve bytes in our MaxPPL
161 * budget. This is used to ensure we have room to append a PING frame later
162 * if we need to. Once we know we will not need to append a PING frame, this
165 unsigned int reserve_allowed
: 1;
167 * Set to 1 if we have appended a STREAM frame with an implicit length. If
168 * this happens we should never append another frame after that frame as it
169 * cannot be validly encoded. This is just a safety check.
171 unsigned int done_implicit
: 1;
174 * The fields in this structure are valid if active is set, which means
175 * that a serialization transaction is currently in progress.
179 unsigned int active
: 1;
183 static void tx_helper_rollback(struct tx_helper
*h
);
184 static int txp_el_ensure_iovec(struct txp_el
*el
, size_t num
);
186 /* Initialises the TX helper. */
187 static int tx_helper_init(struct tx_helper
*h
, OSSL_QUIC_TX_PACKETISER
*txp
,
188 uint32_t enc_level
, size_t max_ppl
, size_t reserve
)
190 if (reserve
> max_ppl
)
194 h
->enc_level
= enc_level
;
195 h
->max_ppl
= max_ppl
;
196 h
->reserve
= reserve
;
198 h
->bytes_appended
= 0;
199 h
->scratch_bytes
= 0;
200 h
->reserve_allowed
= 0;
201 h
->done_implicit
= 0;
205 if (max_ppl
> h
->txp
->el
[enc_level
].scratch_len
) {
206 unsigned char *scratch
;
208 scratch
= OPENSSL_realloc(h
->txp
->el
[enc_level
].scratch
, max_ppl
);
212 h
->txp
->el
[enc_level
].scratch
= scratch
;
213 h
->txp
->el
[enc_level
].scratch_len
= max_ppl
;
219 static void tx_helper_cleanup(struct tx_helper
*h
)
222 tx_helper_rollback(h
);
227 static void tx_helper_unrestrict(struct tx_helper
*h
)
229 h
->reserve_allowed
= 1;
233 * Append an extent of memory to the iovec list. The memory must remain
234 * allocated until we finish generating the packet and call the QTX.
236 * In general, the buffers passed to this function will be from one of two
239 * - Application data contained in stream buffers managed elsewhere
240 * in the QUIC stack; or
242 * - Control frame data appended into txp->scratch using tx_helper_begin and
246 static int tx_helper_append_iovec(struct tx_helper
*h
,
247 const unsigned char *buf
,
250 struct txp_el
*el
= &h
->txp
->el
[h
->enc_level
];
255 if (!ossl_assert(!h
->done_implicit
))
258 if (!txp_el_ensure_iovec(el
, h
->num_iovec
+ 1))
261 el
->iovec
[h
->num_iovec
].buf
= buf
;
262 el
->iovec
[h
->num_iovec
].buf_len
= buf_len
;
265 h
->bytes_appended
+= buf_len
;
270 * How many more bytes of space do we have left in our plaintext packet payload?
272 static size_t tx_helper_get_space_left(struct tx_helper
*h
)
275 - (h
->reserve_allowed
? 0 : h
->reserve
) - h
->bytes_appended
;
279 * Begin a control frame serialization transaction. This allows the
280 * serialization of the control frame to be backed out if it turns out it won't
281 * fit. Write the control frame to the returned WPACKET. Ensure you always
282 * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
285 static WPACKET
*tx_helper_begin(struct tx_helper
*h
)
287 size_t space_left
, len
;
289 struct txp_el
*el
= &h
->txp
->el
[h
->enc_level
];
291 if (!ossl_assert(!h
->txn
.active
))
294 if (!ossl_assert(!h
->done_implicit
))
297 data
= (unsigned char *)el
->scratch
+ h
->scratch_bytes
;
298 len
= el
->scratch_len
- h
->scratch_bytes
;
300 space_left
= tx_helper_get_space_left(h
);
301 if (!ossl_assert(space_left
<= len
))
304 if (!WPACKET_init_static_len(&h
->txn
.wpkt
, data
, len
, 0))
307 if (!WPACKET_set_max_size(&h
->txn
.wpkt
, space_left
)) {
308 WPACKET_cleanup(&h
->txn
.wpkt
);
317 static void tx_helper_end(struct tx_helper
*h
, int success
)
320 WPACKET_finish(&h
->txn
.wpkt
);
322 WPACKET_cleanup(&h
->txn
.wpkt
);
328 /* Abort a control frame serialization transaction. */
329 static void tx_helper_rollback(struct tx_helper
*h
)
337 /* Commit a control frame. */
338 static int tx_helper_commit(struct tx_helper
*h
)
345 if (!WPACKET_get_total_written(&h
->txn
.wpkt
, &l
)) {
350 if (!tx_helper_append_iovec(h
, h
->txn
.data
, l
)) {
355 if (h
->txp
->msg_callback
!= NULL
&& l
> 0) {
357 int ctype
= SSL3_RT_QUIC_FRAME_FULL
;
360 if (!PACKET_buf_init(&pkt
, h
->txn
.data
, l
)
361 || !ossl_quic_wire_peek_frame_header(&pkt
, &ftype
, NULL
)) {
366 if (ftype
== OSSL_QUIC_FRAME_TYPE_PADDING
)
367 ctype
= SSL3_RT_QUIC_FRAME_PADDING
;
368 else if (OSSL_QUIC_FRAME_TYPE_IS_STREAM(ftype
)
369 || ftype
== OSSL_QUIC_FRAME_TYPE_CRYPTO
)
370 ctype
= SSL3_RT_QUIC_FRAME_HEADER
;
372 h
->txp
->msg_callback(1, OSSL_QUIC1_VERSION
, ctype
, h
->txn
.data
, l
,
373 h
->txp
->msg_callback_ssl
,
374 h
->txp
->msg_callback_arg
);
377 h
->scratch_bytes
+= l
;
382 struct archetype_data
{
383 unsigned int allow_ack
: 1;
384 unsigned int allow_ping
: 1;
385 unsigned int allow_crypto
: 1;
386 unsigned int allow_handshake_done
: 1;
387 unsigned int allow_path_challenge
: 1;
388 unsigned int allow_path_response
: 1;
389 unsigned int allow_new_conn_id
: 1;
390 unsigned int allow_retire_conn_id
: 1;
391 unsigned int allow_stream_rel
: 1;
392 unsigned int allow_conn_fc
: 1;
393 unsigned int allow_conn_close
: 1;
394 unsigned int allow_cfq_other
: 1;
395 unsigned int allow_new_token
: 1;
396 unsigned int allow_force_ack_eliciting
: 1;
397 unsigned int allow_padding
: 1;
398 unsigned int require_ack_eliciting
: 1;
399 unsigned int bypass_cc
: 1;
402 struct txp_pkt_geom
{
403 size_t cmpl
, cmppl
, hwm
, pkt_overhead
;
405 struct archetype_data adata
;
411 QUIC_TXPIM_PKT
*tpkt
;
412 QUIC_STREAM
*stream_head
;
414 struct txp_pkt_geom geom
;
418 static QUIC_SSTREAM
*get_sstream_by_id(uint64_t stream_id
, uint32_t pn_space
,
420 static void on_regen_notify(uint64_t frame_type
, uint64_t stream_id
,
421 QUIC_TXPIM_PKT
*pkt
, void *arg
);
422 static void on_confirm_notify(uint64_t frame_type
, uint64_t stream_id
,
423 QUIC_TXPIM_PKT
*pkt
, void *arg
);
424 static void on_sstream_updated(uint64_t stream_id
, void *arg
);
425 static int sstream_is_pending(QUIC_SSTREAM
*sstream
);
426 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER
*txp
,
430 uint32_t *conn_close_enc_level
);
431 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER
*txp
);
432 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER
*txp
,
437 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER
*txp
);
438 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER
*txp
,
440 int chosen_for_conn_close
);
441 static int txp_pkt_init(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
,
442 uint32_t enc_level
, uint32_t archetype
,
443 size_t running_total
);
444 static void txp_pkt_cleanup(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
);
445 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt
*pkt
,
446 OSSL_QUIC_TX_PACKETISER
*txp
);
447 static int txp_pkt_append_padding(struct txp_pkt
*pkt
,
448 OSSL_QUIC_TX_PACKETISER
*txp
, size_t num_bytes
);
449 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER
*txp
, struct txp_pkt
*pkt
,
450 uint32_t archetype
, int *txpim_pkt_reffed
);
451 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER
*txp
,
454 OSSL_QUIC_TX_PACKETISER
*ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS
*args
)
456 OSSL_QUIC_TX_PACKETISER
*txp
;
460 || args
->txpim
== NULL
462 || args
->ackm
== NULL
464 || args
->conn_txfc
== NULL
465 || args
->conn_rxfc
== NULL
466 || args
->max_streams_bidi_rxfc
== NULL
467 || args
->max_streams_uni_rxfc
== NULL
) {
468 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
472 txp
= OPENSSL_zalloc(sizeof(*txp
));
477 txp
->last_tx_time
= ossl_time_zero();
479 if (!ossl_quic_fifd_init(&txp
->fifd
,
480 txp
->args
.cfq
, txp
->args
.ackm
, txp
->args
.txpim
,
481 get_sstream_by_id
, txp
,
482 on_regen_notify
, txp
,
483 on_confirm_notify
, txp
,
484 on_sstream_updated
, txp
,
493 void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER
*txp
)
500 ossl_quic_tx_packetiser_set_initial_token(txp
, NULL
, 0, NULL
, NULL
);
501 ossl_quic_fifd_cleanup(&txp
->fifd
);
502 OPENSSL_free(txp
->conn_close_frame
.reason
);
504 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
505 enc_level
< QUIC_ENC_LEVEL_NUM
;
507 OPENSSL_free(txp
->el
[enc_level
].iovec
);
508 OPENSSL_free(txp
->el
[enc_level
].scratch
);
515 * Determine if an Initial packet token length is reasonable based on the
516 * current MDPL, returning 1 if it is OK.
518 * The real PMTU to the peer could differ from our (pessimistic) understanding
519 * of the PMTU, therefore it is possible we could receive an Initial token from
520 * a server in a Retry packet which is bigger than the MDPL. In this case it is
521 * impossible for us ever to make forward progress and we need to error out
522 * and fail the connection attempt.
524 * The specific boundary condition is complex: for example, after the size of
525 * the Initial token, there are the Initial packet header overheads and then
526 * encryption/AEAD tag overheads. After that, the minimum room for frame data in
527 * order to guarantee forward progress must be guaranteed. For example, a crypto
528 * stream needs to always be able to serialize at least one byte in a CRYPTO
529 * frame in order to make forward progress. Because the offset field of a CRYPTO
530 * frame uses a variable-length integer, the number of bytes needed to ensure
533 * Rather than trying to get this boundary condition check actually right,
534 * require a reasonable amount of slack to avoid pathological behaviours. (After
535 * all, transmitting a CRYPTO stream one byte at a time is probably not
538 * We choose 160 bytes as the required margin, which is double the rough
539 * estimation of the minimum we would require to guarantee forward progress
540 * under worst case packet overheads.
542 #define TXP_REQUIRED_TOKEN_MARGIN 160
544 static int txp_check_token_len(size_t token_len
, size_t mdpl
)
549 if (token_len
>= mdpl
)
552 if (TXP_REQUIRED_TOKEN_MARGIN
>= mdpl
)
553 /* (should not be possible because MDPL must be at least 1200) */
556 if (token_len
> mdpl
- TXP_REQUIRED_TOKEN_MARGIN
)
562 int ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER
*txp
,
563 const unsigned char *token
,
565 ossl_quic_initial_token_free_fn
*free_cb
,
568 if (!txp_check_token_len(token_len
, txp_get_mdpl(txp
)))
571 if (txp
->initial_token
!= NULL
&& txp
->initial_token_free_cb
!= NULL
)
572 txp
->initial_token_free_cb(txp
->initial_token
, txp
->initial_token_len
,
573 txp
->initial_token_free_cb_arg
);
575 txp
->initial_token
= token
;
576 txp
->initial_token_len
= token_len
;
577 txp
->initial_token_free_cb
= free_cb
;
578 txp
->initial_token_free_cb_arg
= free_cb_arg
;
582 int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER
*txp
,
583 const QUIC_CONN_ID
*dcid
)
586 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
590 txp
->args
.cur_dcid
= *dcid
;
594 int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER
*txp
,
595 const QUIC_CONN_ID
*scid
)
598 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_NULL_PARAMETER
);
602 txp
->args
.cur_scid
= *scid
;
606 /* Change the destination L4 address the TXP uses to send datagrams. */
607 int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER
*txp
,
608 const BIO_ADDR
*peer
)
611 BIO_ADDR_clear(&txp
->args
.peer
);
615 txp
->args
.peer
= *peer
;
619 void ossl_quic_tx_packetiser_set_ack_tx_cb(OSSL_QUIC_TX_PACKETISER
*txp
,
620 void (*cb
)(const OSSL_QUIC_FRAME_ACK
*ack
,
626 txp
->ack_tx_cb_arg
= cb_arg
;
629 int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER
*txp
,
632 if (enc_level
>= QUIC_ENC_LEVEL_NUM
) {
633 ERR_raise(ERR_LIB_SSL
, ERR_R_PASSED_INVALID_ARGUMENT
);
637 if (enc_level
!= QUIC_ENC_LEVEL_0RTT
)
638 txp
->args
.crypto
[ossl_quic_enc_level_to_pn_space(enc_level
)] = NULL
;
643 void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER
*txp
)
645 txp
->handshake_complete
= 1;
648 void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER
*txp
)
650 txp
->want_handshake_done
= 1;
653 void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER
*txp
,
656 txp
->force_ack_eliciting
|= (1UL << pn_space
);
659 void ossl_quic_tx_packetiser_schedule_ack(OSSL_QUIC_TX_PACKETISER
*txp
,
662 txp
->want_ack
|= (1UL << pn_space
);
665 #define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
666 #define TXP_ERR_SUCCESS 1 /* Success */
667 #define TXP_ERR_SPACE 2 /* Not enough room for another packet */
668 #define TXP_ERR_INPUT 3 /* Invalid/malformed input */
671 * Generates a datagram by polling the various ELs to determine if they want to
672 * generate any frames, and generating a datagram which coalesces packets for
675 int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER
*txp
,
676 QUIC_TXP_STATUS
*status
)
679 * Called to generate one or more datagrams, each containing one or more
682 * There are some tricky things to note here:
684 * - The TXP is only concerned with generating encrypted packets;
685 * other packets use a different path.
687 * - Any datagram containing an Initial packet must have a payload length
688 * (DPL) of at least 1200 bytes. This padding need not necessarily be
689 * found in the Initial packet.
691 * - It is desirable to be able to coalesce an Initial packet
692 * with a Handshake packet. Since, before generating the Handshake
693 * packet, we do not know how long it will be, we cannot know the
694 * correct amount of padding to ensure a DPL of at least 1200 bytes.
695 * Thus this padding must added to the Handshake packet (or whatever
696 * packet is the last in the datagram).
698 * - However, at the time that we generate the Initial packet,
699 * we do not actually know for sure that we will be followed
700 * in the datagram by another packet. For example, suppose we have
701 * some queued data (e.g. crypto stream data for the HANDSHAKE EL)
702 * it looks like we will want to send on the HANDSHAKE EL.
703 * We could assume padding will be placed in the Handshake packet
704 * subsequently and avoid adding any padding to the Initial packet
705 * (which would leave no room for the Handshake packet in the
708 * However, this is not actually a safe assumption. Suppose that we
709 * are using a link with a MDPL of 1200 bytes, the minimum allowed by
710 * QUIC. Suppose that the Initial packet consumes 1195 bytes in total.
711 * Since it is not possible to fit a Handshake packet in just 5 bytes,
712 * upon trying to add a Handshake packet after generating the Initial
713 * packet, we will discover we have no room to fit it! This is not a
714 * problem in itself as another datagram can be sent subsequently, but
715 * it is a problem because we were counting to use that packet to hold
716 * the essential padding. But if we have already finished encrypting
717 * the Initial packet, we cannot go and add padding to it anymore.
718 * This leaves us stuck.
720 * Because of this, we have to plan multiple packets simultaneously, such
721 * that we can start generating a Handshake (or 0-RTT or 1-RTT, or so on)
722 * packet while still having the option to go back and add padding to the
723 * Initial packet if it turns out to be needed.
725 * Trying to predict ahead of time (e.g. during Initial packet generation)
726 * whether we will successfully generate a subsequent packet is fraught with
727 * error as it relies on a large number of variables:
729 * - Do we have room to fit a packet header? (Consider that due to
730 * variable-length integer encoding this is highly variable and can even
731 * depend on payload length due to a variable-length Length field.)
733 * - Can we fit even a single one of the frames we want to put in this
734 * packet in the packet? (Each frame type has a bespoke encoding. While
735 * our encodings of some frame types are adaptive based on the available
736 * room - e.g. STREAM frames - ultimately all frame types have some
737 * absolute minimum number of bytes to be successfully encoded. For
738 * example, if after an Initial packet there is enough room to encode
739 * only one byte of frame data, it is quite likely we can't send any of
740 * the frames we wanted to send.) While this is not strictly a problem
741 * because we could just fill the packet with padding frames, this is a
742 * pointless packet and is wasteful.
744 * Thus we adopt a multi-phase architecture:
746 * 1. Archetype Selection: Determine desired packet archetype.
748 * 2. Packet Staging: Generation of packet information and packet payload
749 * data (frame data) into staging areas.
751 * 3. Packet Adjustment: Adjustment of staged packets, adding padding to
752 * the staged packets if needed.
754 * 4. Commit: The packets are sent to the QTX and recorded as having been
759 uint32_t archetype
, enc_level
;
760 uint32_t conn_close_enc_level
= QUIC_ENC_LEVEL_NUM
;
761 struct txp_pkt pkt
[QUIC_ENC_LEVEL_NUM
];
762 size_t pkts_done
= 0;
763 uint64_t cc_limit
= txp
->args
.cc_method
->get_tx_allowance(txp
->args
.cc_data
);
764 int need_padding
= 0, txpim_pkt_reffed
;
766 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
767 enc_level
< QUIC_ENC_LEVEL_NUM
;
769 pkt
[enc_level
].h_valid
= 0;
771 memset(status
, 0, sizeof(*status
));
774 * Should not be needed, but a sanity check in case anyone else has been
777 ossl_qtx_finish_dgram(txp
->args
.qtx
);
779 /* 1. Archetype Selection */
780 archetype
= txp_determine_archetype(txp
, cc_limit
);
782 /* 2. Packet Staging */
783 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
784 enc_level
< QUIC_ENC_LEVEL_NUM
;
786 size_t running_total
= (enc_level
> QUIC_ENC_LEVEL_INITIAL
)
787 ? pkt
[enc_level
- 1].geom
.hwm
: 0;
789 pkt
[enc_level
].geom
.hwm
= running_total
;
791 if (!txp_should_try_staging(txp
, enc_level
, archetype
, cc_limit
,
792 &conn_close_enc_level
))
795 if (!txp_pkt_init(&pkt
[enc_level
], txp
, enc_level
, archetype
,
798 * If this fails this is not a fatal error - it means the geometry
799 * planning determined there was not enough space for another
800 * packet. So just proceed with what we've already planned for.
804 rc
= txp_generate_for_el(txp
, &pkt
[enc_level
],
805 conn_close_enc_level
== enc_level
);
806 if (rc
!= TXP_ERR_SUCCESS
)
809 if (pkt
[enc_level
].force_pad
)
811 * txp_generate_for_el emitted a frame which forces packet padding.
815 pkt
[enc_level
].geom
.hwm
= running_total
816 + pkt
[enc_level
].h
.bytes_appended
817 + pkt
[enc_level
].geom
.pkt_overhead
;
820 /* 3. Packet Adjustment */
821 if (pkt
[QUIC_ENC_LEVEL_INITIAL
].h_valid
822 && pkt
[QUIC_ENC_LEVEL_INITIAL
].h
.bytes_appended
> 0)
824 * We have an Initial packet in this datagram, so we need to make sure
825 * the total size of the datagram is adequate.
830 size_t total_dgram_size
= 0;
831 const size_t min_dpl
= QUIC_MIN_INITIAL_DGRAM_LEN
;
832 uint32_t pad_el
= QUIC_ENC_LEVEL_NUM
;
834 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
835 enc_level
< QUIC_ENC_LEVEL_NUM
;
837 if (pkt
[enc_level
].h_valid
&& pkt
[enc_level
].h
.bytes_appended
> 0) {
838 if (pad_el
== QUIC_ENC_LEVEL_NUM
840 * We might not be able to add padding, for example if we
841 * are using the ACK_ONLY archetype.
843 && pkt
[enc_level
].geom
.adata
.allow_padding
844 && !pkt
[enc_level
].h
.done_implicit
)
847 txp_pkt_postgen_update_pkt_overhead(&pkt
[enc_level
], txp
);
848 total_dgram_size
+= pkt
[enc_level
].geom
.pkt_overhead
849 + pkt
[enc_level
].h
.bytes_appended
;
852 if (pad_el
!= QUIC_ENC_LEVEL_NUM
&& total_dgram_size
< min_dpl
) {
853 size_t deficit
= min_dpl
- total_dgram_size
;
855 if (!txp_pkt_append_padding(&pkt
[pad_el
], txp
, deficit
))
858 total_dgram_size
+= deficit
;
861 * Padding frames make a packet ineligible for being a non-inflight
864 pkt
[pad_el
].tpkt
->ackm_pkt
.is_inflight
= 1;
868 * If we have failed to make a datagram of adequate size, for example
869 * because we have a padding requirement but are using the ACK_ONLY
870 * archetype (because we are CC limited), which precludes us from
871 * sending padding, give up on generating the datagram - there is
874 if (total_dgram_size
< min_dpl
) {
881 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
882 enc_level
< QUIC_ENC_LEVEL_NUM
;
885 if (!pkt
[enc_level
].h_valid
)
886 /* Did not attempt to generate a packet for this EL. */
889 if (pkt
[enc_level
].h
.bytes_appended
== 0)
890 /* Nothing was generated for this EL, so skip. */
893 rc
= txp_pkt_commit(txp
, &pkt
[enc_level
], archetype
,
896 status
->sent_ack_eliciting
897 = status
->sent_ack_eliciting
898 || pkt
[enc_level
].tpkt
->ackm_pkt
.is_ack_eliciting
;
900 if (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
)
901 status
->sent_handshake
902 = (pkt
[enc_level
].h_valid
903 && pkt
[enc_level
].h
.bytes_appended
> 0);
906 if (txpim_pkt_reffed
)
907 pkt
[enc_level
].tpkt
= NULL
; /* don't free */
915 /* Flush & Cleanup */
918 ossl_qtx_finish_dgram(txp
->args
.qtx
);
920 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
921 enc_level
< QUIC_ENC_LEVEL_NUM
;
923 txp_pkt_cleanup(&pkt
[enc_level
], txp
);
925 status
->sent_pkt
= pkts_done
;
930 static const struct archetype_data archetypes
[QUIC_ENC_LEVEL_NUM
][TX_PACKETISER_ARCHETYPE_NUM
] = {
933 /* EL 0(INITIAL) - Archetype 0(NORMAL) */
937 /*allow_crypto =*/ 1,
938 /*allow_handshake_done =*/ 0,
939 /*allow_path_challenge =*/ 0,
940 /*allow_path_response =*/ 0,
941 /*allow_new_conn_id =*/ 0,
942 /*allow_retire_conn_id =*/ 0,
943 /*allow_stream_rel =*/ 0,
944 /*allow_conn_fc =*/ 0,
945 /*allow_conn_close =*/ 1,
946 /*allow_cfq_other =*/ 0,
947 /*allow_new_token =*/ 0,
948 /*allow_force_ack_eliciting =*/ 1,
949 /*allow_padding =*/ 1,
950 /*require_ack_eliciting =*/ 0,
953 /* EL 0(INITIAL) - Archetype 1(PROBE) */
957 /*allow_crypto =*/ 1,
958 /*allow_handshake_done =*/ 0,
959 /*allow_path_challenge =*/ 0,
960 /*allow_path_response =*/ 0,
961 /*allow_new_conn_id =*/ 0,
962 /*allow_retire_conn_id =*/ 0,
963 /*allow_stream_rel =*/ 0,
964 /*allow_conn_fc =*/ 0,
965 /*allow_conn_close =*/ 1,
966 /*allow_cfq_other =*/ 0,
967 /*allow_new_token =*/ 0,
968 /*allow_force_ack_eliciting =*/ 1,
969 /*allow_padding =*/ 1,
970 /*require_ack_eliciting =*/ 1,
973 /* EL 0(INITIAL) - Archetype 2(ACK_ONLY) */
977 /*allow_crypto =*/ 0,
978 /*allow_handshake_done =*/ 0,
979 /*allow_path_challenge =*/ 0,
980 /*allow_path_response =*/ 0,
981 /*allow_new_conn_id =*/ 0,
982 /*allow_retire_conn_id =*/ 0,
983 /*allow_stream_rel =*/ 0,
984 /*allow_conn_fc =*/ 0,
985 /*allow_conn_close =*/ 0,
986 /*allow_cfq_other =*/ 0,
987 /*allow_new_token =*/ 0,
988 /*allow_force_ack_eliciting =*/ 1,
989 /*allow_padding =*/ 0,
990 /*require_ack_eliciting =*/ 0,
994 /* EL 1(HANDSHAKE) */
996 /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
1000 /*allow_crypto =*/ 1,
1001 /*allow_handshake_done =*/ 0,
1002 /*allow_path_challenge =*/ 0,
1003 /*allow_path_response =*/ 0,
1004 /*allow_new_conn_id =*/ 0,
1005 /*allow_retire_conn_id =*/ 0,
1006 /*allow_stream_rel =*/ 0,
1007 /*allow_conn_fc =*/ 0,
1008 /*allow_conn_close =*/ 1,
1009 /*allow_cfq_other =*/ 0,
1010 /*allow_new_token =*/ 0,
1011 /*allow_force_ack_eliciting =*/ 1,
1012 /*allow_padding =*/ 1,
1013 /*require_ack_eliciting =*/ 0,
1016 /* EL 1(HANDSHAKE) - Archetype 1(PROBE) */
1020 /*allow_crypto =*/ 1,
1021 /*allow_handshake_done =*/ 0,
1022 /*allow_path_challenge =*/ 0,
1023 /*allow_path_response =*/ 0,
1024 /*allow_new_conn_id =*/ 0,
1025 /*allow_retire_conn_id =*/ 0,
1026 /*allow_stream_rel =*/ 0,
1027 /*allow_conn_fc =*/ 0,
1028 /*allow_conn_close =*/ 1,
1029 /*allow_cfq_other =*/ 0,
1030 /*allow_new_token =*/ 0,
1031 /*allow_force_ack_eliciting =*/ 1,
1032 /*allow_padding =*/ 1,
1033 /*require_ack_eliciting =*/ 1,
1036 /* EL 1(HANDSHAKE) - Archetype 2(ACK_ONLY) */
1040 /*allow_crypto =*/ 0,
1041 /*allow_handshake_done =*/ 0,
1042 /*allow_path_challenge =*/ 0,
1043 /*allow_path_response =*/ 0,
1044 /*allow_new_conn_id =*/ 0,
1045 /*allow_retire_conn_id =*/ 0,
1046 /*allow_stream_rel =*/ 0,
1047 /*allow_conn_fc =*/ 0,
1048 /*allow_conn_close =*/ 0,
1049 /*allow_cfq_other =*/ 0,
1050 /*allow_new_token =*/ 0,
1051 /*allow_force_ack_eliciting =*/ 1,
1052 /*allow_padding =*/ 0,
1053 /*require_ack_eliciting =*/ 0,
1059 /* EL 2(0RTT) - Archetype 0(NORMAL) */
1063 /*allow_crypto =*/ 0,
1064 /*allow_handshake_done =*/ 0,
1065 /*allow_path_challenge =*/ 0,
1066 /*allow_path_response =*/ 0,
1067 /*allow_new_conn_id =*/ 1,
1068 /*allow_retire_conn_id =*/ 1,
1069 /*allow_stream_rel =*/ 1,
1070 /*allow_conn_fc =*/ 1,
1071 /*allow_conn_close =*/ 1,
1072 /*allow_cfq_other =*/ 0,
1073 /*allow_new_token =*/ 0,
1074 /*allow_force_ack_eliciting =*/ 0,
1075 /*allow_padding =*/ 1,
1076 /*require_ack_eliciting =*/ 0,
1079 /* EL 2(0RTT) - Archetype 1(PROBE) */
1083 /*allow_crypto =*/ 0,
1084 /*allow_handshake_done =*/ 0,
1085 /*allow_path_challenge =*/ 0,
1086 /*allow_path_response =*/ 0,
1087 /*allow_new_conn_id =*/ 1,
1088 /*allow_retire_conn_id =*/ 1,
1089 /*allow_stream_rel =*/ 1,
1090 /*allow_conn_fc =*/ 1,
1091 /*allow_conn_close =*/ 1,
1092 /*allow_cfq_other =*/ 0,
1093 /*allow_new_token =*/ 0,
1094 /*allow_force_ack_eliciting =*/ 0,
1095 /*allow_padding =*/ 1,
1096 /*require_ack_eliciting =*/ 1,
1099 /* EL 2(0RTT) - Archetype 2(ACK_ONLY) */
1103 /*allow_crypto =*/ 0,
1104 /*allow_handshake_done =*/ 0,
1105 /*allow_path_challenge =*/ 0,
1106 /*allow_path_response =*/ 0,
1107 /*allow_new_conn_id =*/ 0,
1108 /*allow_retire_conn_id =*/ 0,
1109 /*allow_stream_rel =*/ 0,
1110 /*allow_conn_fc =*/ 0,
1111 /*allow_conn_close =*/ 0,
1112 /*allow_cfq_other =*/ 0,
1113 /*allow_new_token =*/ 0,
1114 /*allow_force_ack_eliciting =*/ 0,
1115 /*allow_padding =*/ 0,
1116 /*require_ack_eliciting =*/ 0,
1122 /* EL 3(1RTT) - Archetype 0(NORMAL) */
1126 /*allow_crypto =*/ 1,
1127 /*allow_handshake_done =*/ 1,
1128 /*allow_path_challenge =*/ 0,
1129 /*allow_path_response =*/ 1,
1130 /*allow_new_conn_id =*/ 1,
1131 /*allow_retire_conn_id =*/ 1,
1132 /*allow_stream_rel =*/ 1,
1133 /*allow_conn_fc =*/ 1,
1134 /*allow_conn_close =*/ 1,
1135 /*allow_cfq_other =*/ 1,
1136 /*allow_new_token =*/ 1,
1137 /*allow_force_ack_eliciting =*/ 1,
1138 /*allow_padding =*/ 1,
1139 /*require_ack_eliciting =*/ 0,
1142 /* EL 3(1RTT) - Archetype 1(PROBE) */
1146 /*allow_crypto =*/ 1,
1147 /*allow_handshake_done =*/ 1,
1148 /*allow_path_challenge =*/ 0,
1149 /*allow_path_response =*/ 1,
1150 /*allow_new_conn_id =*/ 1,
1151 /*allow_retire_conn_id =*/ 1,
1152 /*allow_stream_rel =*/ 1,
1153 /*allow_conn_fc =*/ 1,
1154 /*allow_conn_close =*/ 1,
1155 /*allow_cfq_other =*/ 1,
1156 /*allow_new_token =*/ 1,
1157 /*allow_force_ack_eliciting =*/ 1,
1158 /*allow_padding =*/ 1,
1159 /*require_ack_eliciting =*/ 1,
1162 /* EL 3(1RTT) - Archetype 2(ACK_ONLY) */
1166 /*allow_crypto =*/ 0,
1167 /*allow_handshake_done =*/ 0,
1168 /*allow_path_challenge =*/ 0,
1169 /*allow_path_response =*/ 0,
1170 /*allow_new_conn_id =*/ 0,
1171 /*allow_retire_conn_id =*/ 0,
1172 /*allow_stream_rel =*/ 0,
1173 /*allow_conn_fc =*/ 0,
1174 /*allow_conn_close =*/ 0,
1175 /*allow_cfq_other =*/ 0,
1176 /*allow_new_token =*/ 0,
1177 /*allow_force_ack_eliciting =*/ 1,
1178 /*allow_padding =*/ 0,
1179 /*require_ack_eliciting =*/ 0,
1185 static int txp_get_archetype_data(uint32_t enc_level
,
1187 struct archetype_data
*a
)
1189 if (enc_level
>= QUIC_ENC_LEVEL_NUM
1190 || archetype
>= TX_PACKETISER_ARCHETYPE_NUM
)
1193 /* No need to avoid copying this as it should not exceed one int in size. */
1194 *a
= archetypes
[enc_level
][archetype
];
1198 static int txp_determine_geometry(OSSL_QUIC_TX_PACKETISER
*txp
,
1201 size_t running_total
,
1203 struct txp_pkt_geom
*geom
)
1205 size_t mdpl
, cmpl
, hdr_len
;
1207 /* Get information about packet archetype. */
1208 if (!txp_get_archetype_data(enc_level
, archetype
, &geom
->adata
))
1211 /* Assemble packet header. */
1212 phdr
->type
= ossl_quic_enc_level_to_pkt_type(enc_level
);
1214 phdr
->pn_len
= txp_determine_pn_len(txp
);
1218 phdr
->version
= QUIC_VERSION_1
;
1219 phdr
->dst_conn_id
= txp
->args
.cur_dcid
;
1220 phdr
->src_conn_id
= txp
->args
.cur_scid
;
1223 * We need to know the length of the payload to get an accurate header
1224 * length for non-1RTT packets, because the Length field found in
1225 * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
1226 * we don't have a good idea of the length of our payload, because the
1227 * length of the payload depends on the room in the datagram after fitting
1228 * the header, which depends on the size of the header.
1230 * In general, it does not matter if a packet is slightly shorter (because
1231 * e.g. we predicted use of a 2-byte length field, but ended up only needing
1232 * a 1-byte length field). However this does matter for Initial packets
1233 * which must be at least 1200 bytes, which is also the assumed default MTU;
1234 * therefore in many cases Initial packets will be padded to 1200 bytes,
1235 * which means if we overestimated the header size, we will be short by a
1236 * few bytes and the server will ignore the packet for being too short. In
1237 * this case, however, such packets always *will* be padded to meet 1200
1238 * bytes, which requires a 2-byte length field, so we don't actually need to
1239 * worry about this. Thus we estimate the header length assuming a 2-byte
1240 * length field here, which should in practice work well in all cases.
1242 phdr
->len
= OSSL_QUIC_VLINT_2B_MAX
- phdr
->pn_len
;
1244 if (enc_level
== QUIC_ENC_LEVEL_INITIAL
) {
1245 phdr
->token
= txp
->initial_token
;
1246 phdr
->token_len
= txp
->initial_token_len
;
1249 phdr
->token_len
= 0;
1252 hdr_len
= ossl_quic_wire_get_encoded_pkt_hdr_len(phdr
->dst_conn_id
.id_len
,
1257 /* MDPL: Maximum datagram payload length. */
1258 mdpl
= txp_get_mdpl(txp
);
1261 * CMPL: Maximum encoded packet size we can put into this datagram given any
1262 * previous packets coalesced into it.
1264 if (running_total
> mdpl
)
1265 /* Should not be possible, but if it happens: */
1268 cmpl
= mdpl
- running_total
;
1270 /* CMPPL: Maximum amount we can put into the current packet payload */
1271 if (!txp_determine_ppl_from_pl(txp
, cmpl
, enc_level
, hdr_len
, &geom
->cmppl
))
1275 geom
->pkt_overhead
= cmpl
- geom
->cmppl
;
1276 geom
->archetype
= archetype
;
1280 static uint32_t txp_determine_archetype(OSSL_QUIC_TX_PACKETISER
*txp
,
1283 OSSL_ACKM_PROBE_INFO
*probe_info
1284 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
1288 * If ACKM has requested probe generation (e.g. due to PTO), we generate a
1289 * Probe-archetype packet. Actually, we determine archetype on a
1290 * per-datagram basis, so if any EL wants a probe, do a pass in which
1291 * we try and generate a probe (if needed) for all ELs.
1293 if (probe_info
->anti_deadlock_initial
> 0
1294 || probe_info
->anti_deadlock_handshake
> 0)
1295 return TX_PACKETISER_ARCHETYPE_PROBE
;
1297 for (pn_space
= QUIC_PN_SPACE_INITIAL
;
1298 pn_space
< QUIC_PN_SPACE_NUM
;
1300 if (probe_info
->pto
[pn_space
] > 0)
1301 return TX_PACKETISER_ARCHETYPE_PROBE
;
1304 * If we are out of CC budget, we cannot send a normal packet,
1305 * but we can do an ACK-only packet (potentially, if we
1306 * want to send an ACK).
1309 return TX_PACKETISER_ARCHETYPE_ACK_ONLY
;
1311 /* All other packets. */
1312 return TX_PACKETISER_ARCHETYPE_NORMAL
;
1315 static int txp_should_try_staging(OSSL_QUIC_TX_PACKETISER
*txp
,
1319 uint32_t *conn_close_enc_level
)
1321 struct archetype_data a
;
1322 uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
1323 QUIC_CFQ_ITEM
*cfq_item
;
1325 if (!ossl_qtx_is_enc_level_provisioned(txp
->args
.qtx
, enc_level
))
1328 if (!txp_get_archetype_data(enc_level
, archetype
, &a
))
1331 if (!a
.bypass_cc
&& cc_limit
== 0)
1332 /* CC not allowing us to send. */
1336 * We can produce CONNECTION_CLOSE frames on any EL in principle, which
1337 * means we need to choose which EL we would prefer to use. After a
1338 * connection is fully established we have only one provisioned EL and this
1339 * is a non-issue. Where multiple ELs are provisioned, it is possible the
1340 * peer does not have the keys for the EL yet, which suggests in general it
1341 * is preferable to use the lowest EL which is still provisioned.
1343 * However (RFC 9000 s. 10.2.3 & 12.5) we are also required to not send
1344 * application CONNECTION_CLOSE frames in non-1-RTT ELs, so as to not
1345 * potentially leak application data on a connection which has yet to be
1346 * authenticated. Thus when we have an application CONNECTION_CLOSE frame
1347 * queued and need to send it on a non-1-RTT EL, we have to convert it
1348 * into a transport CONNECTION_CLOSE frame which contains no application
1349 * data. Since this loses information, it suggests we should use the 1-RTT
1350 * EL to avoid this if possible, even if a lower EL is also available.
1352 * At the same time, just because we have the 1-RTT EL provisioned locally
1353 * does not necessarily mean the peer does, for example if a handshake
1354 * CRYPTO frame has been lost. It is fairly important that CONNECTION_CLOSE
1355 * is signalled in a way we know our peer can decrypt, as we stop processing
1356 * connection retransmission logic for real after connection close and
1357 * simply 'blindly' retransmit the same CONNECTION_CLOSE frame.
1359 * This is not a major concern for clients, since if a client has a 1-RTT EL
1360 * provisioned the server is guaranteed to also have a 1-RTT EL provisioned.
1362 * TODO(QUIC SERVER): Revisit this when server support is added.
1364 if (*conn_close_enc_level
> enc_level
1365 && *conn_close_enc_level
!= QUIC_ENC_LEVEL_1RTT
)
1366 *conn_close_enc_level
= enc_level
;
1368 /* Do we need to send a PTO probe? */
1369 if (a
.allow_force_ack_eliciting
) {
1370 OSSL_ACKM_PROBE_INFO
*probe_info
1371 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
1373 if ((enc_level
== QUIC_ENC_LEVEL_INITIAL
1374 && probe_info
->anti_deadlock_initial
> 0)
1375 || (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
1376 && probe_info
->anti_deadlock_handshake
> 0)
1377 || probe_info
->pto
[pn_space
] > 0)
1381 /* Does the crypto stream for this EL want to produce anything? */
1382 if (a
.allow_crypto
&& sstream_is_pending(txp
->args
.crypto
[pn_space
]))
1385 /* Does the ACKM for this PN space want to produce anything? */
1386 if (a
.allow_ack
&& (ossl_ackm_is_ack_desired(txp
->args
.ackm
, pn_space
)
1387 || (txp
->want_ack
& (1UL << pn_space
)) != 0))
1390 /* Do we need to force emission of an ACK-eliciting packet? */
1391 if (a
.allow_force_ack_eliciting
1392 && (txp
->force_ack_eliciting
& (1UL << pn_space
)) != 0)
1395 /* Does the connection-level RXFC want to produce a frame? */
1396 if (a
.allow_conn_fc
&& (txp
->want_max_data
1397 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 0)))
1400 /* Do we want to produce a MAX_STREAMS frame? */
1402 && (txp
->want_max_streams_bidi
1403 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
,
1405 || txp
->want_max_streams_uni
1406 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
,
1410 /* Do we want to produce a HANDSHAKE_DONE frame? */
1411 if (a
.allow_handshake_done
&& txp
->want_handshake_done
)
1414 /* Do we want to produce a CONNECTION_CLOSE frame? */
1415 if (a
.allow_conn_close
&& txp
->want_conn_close
&&
1416 *conn_close_enc_level
== enc_level
)
1418 * This is a bit of a special case since CONNECTION_CLOSE can appear in
1419 * most packet types, and when we decide we want to send it this status
1420 * isn't tied to a specific EL. So if we want to send it, we send it
1421 * only on the lowest non-dropped EL.
1425 /* Does the CFQ have any frames queued for this PN space? */
1426 if (enc_level
!= QUIC_ENC_LEVEL_0RTT
)
1427 for (cfq_item
= ossl_quic_cfq_get_priority_head(txp
->args
.cfq
, pn_space
);
1429 cfq_item
= ossl_quic_cfq_item_get_priority_next(cfq_item
, pn_space
)) {
1430 uint64_t frame_type
= ossl_quic_cfq_item_get_frame_type(cfq_item
);
1432 switch (frame_type
) {
1433 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID
:
1434 if (a
.allow_new_conn_id
)
1437 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID
:
1438 if (a
.allow_retire_conn_id
)
1441 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN
:
1442 if (a
.allow_new_token
)
1445 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE
:
1446 if (a
.allow_path_response
)
1450 if (a
.allow_cfq_other
)
1456 if (a
.allow_stream_rel
&& txp
->handshake_complete
) {
1457 QUIC_STREAM_ITER it
;
1459 /* If there are any active streams, 0/1-RTT wants to produce a packet.
1460 * Whether a stream is on the active list is required to be precise
1461 * (i.e., a stream is never on the active list if we cannot produce a
1462 * frame for it), and all stream-related frames are governed by
1463 * a.allow_stream_rel (i.e., if we can send one type of stream-related
1464 * frame, we can send any of them), so we don't need to inspect
1465 * individual streams on the active list, just confirm that the active
1466 * list is non-empty.
1468 ossl_quic_stream_iter_init(&it
, txp
->args
.qsm
, 0);
1469 if (it
.stream
!= NULL
)
1476 static int sstream_is_pending(QUIC_SSTREAM
*sstream
)
1478 OSSL_QUIC_FRAME_STREAM hdr
;
1479 OSSL_QTX_IOVEC iov
[2];
1480 size_t num_iov
= OSSL_NELEM(iov
);
1482 return ossl_quic_sstream_get_stream_frame(sstream
, 0, &hdr
, iov
, &num_iov
);
1485 /* Determine how many bytes we should use for the encoded PN. */
1486 static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER
*txp
)
1488 return 4; /* TODO(QUIC FUTURE) */
1491 /* Determine plaintext packet payload length from payload length. */
1492 static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER
*txp
,
1503 if (!ossl_qtx_calculate_plaintext_payload_len(txp
->args
.qtx
, enc_level
,
1511 static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER
*txp
)
1513 return ossl_qtx_get_mdpl(txp
->args
.qtx
);
1516 static QUIC_SSTREAM
*get_sstream_by_id(uint64_t stream_id
, uint32_t pn_space
,
1519 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1522 if (stream_id
== UINT64_MAX
)
1523 return txp
->args
.crypto
[pn_space
];
1525 s
= ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1532 static void on_regen_notify(uint64_t frame_type
, uint64_t stream_id
,
1533 QUIC_TXPIM_PKT
*pkt
, void *arg
)
1535 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1537 switch (frame_type
) {
1538 case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE
:
1539 txp
->want_handshake_done
= 1;
1541 case OSSL_QUIC_FRAME_TYPE_MAX_DATA
:
1542 txp
->want_max_data
= 1;
1544 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI
:
1545 txp
->want_max_streams_bidi
= 1;
1547 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI
:
1548 txp
->want_max_streams_uni
= 1;
1550 case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN
:
1551 txp
->want_ack
|= (1UL << pkt
->ackm_pkt
.pkt_space
);
1553 case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA
:
1556 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1561 s
->want_max_stream_data
= 1;
1562 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1565 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING
:
1568 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1573 ossl_quic_stream_map_schedule_stop_sending(txp
->args
.qsm
, s
);
1576 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM
:
1579 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1584 s
->want_reset_stream
= 1;
1585 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1594 static int txp_pkt_init(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
,
1595 uint32_t enc_level
, uint32_t archetype
,
1596 size_t running_total
)
1598 if (!txp_determine_geometry(txp
, archetype
, enc_level
,
1599 running_total
, &pkt
->phdr
, &pkt
->geom
))
1603 * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
1606 if (!tx_helper_init(&pkt
->h
, txp
, enc_level
,
1608 pkt
->geom
.adata
.require_ack_eliciting
? 1 : 0))
1613 pkt
->stream_head
= NULL
;
1618 static void txp_pkt_cleanup(struct txp_pkt
*pkt
, OSSL_QUIC_TX_PACKETISER
*txp
)
1623 tx_helper_cleanup(&pkt
->h
);
1626 if (pkt
->tpkt
!= NULL
) {
1627 ossl_quic_txpim_pkt_release(txp
->args
.txpim
, pkt
->tpkt
);
1632 static int txp_pkt_postgen_update_pkt_overhead(struct txp_pkt
*pkt
,
1633 OSSL_QUIC_TX_PACKETISER
*txp
)
1636 * After we have staged and generated our packets, but before we commit
1637 * them, it is possible for the estimated packet overhead (packet header +
1638 * AEAD tag size) to shrink slightly because we generated a short packet
1639 * whose which can be represented in fewer bytes as a variable-length
1640 * integer than we were (pessimistically) budgeting for. We need to account
1641 * for this to ensure that we get our padding calculation exactly right.
1643 * Update pkt_overhead to be accurate now that we know how much data is
1644 * going in a packet.
1646 size_t hdr_len
, ciphertext_len
;
1648 if (pkt
->h
.enc_level
== QUIC_ENC_LEVEL_INITIAL
)
1650 * Don't update overheads for the INITIAL EL - we have not finished
1651 * appending padding to it and would potentially miscalculate the
1652 * correct padding if we now update the pkt_overhead field to switch to
1653 * e.g. a 1-byte length field in the packet header. Since we are padding
1654 * to QUIC_MIN_INITIAL_DGRAM_LEN which requires a 2-byte length field,
1655 * this is guaranteed to be moot anyway. See comment in
1656 * txp_determine_geometry for more information.
1660 if (!ossl_qtx_calculate_ciphertext_payload_len(txp
->args
.qtx
, pkt
->h
.enc_level
,
1661 pkt
->h
.bytes_appended
,
1665 pkt
->phdr
.len
= ciphertext_len
;
1667 hdr_len
= ossl_quic_wire_get_encoded_pkt_hdr_len(pkt
->phdr
.dst_conn_id
.id_len
,
1670 pkt
->geom
.pkt_overhead
= hdr_len
+ ciphertext_len
- pkt
->h
.bytes_appended
;
1674 static void on_confirm_notify(uint64_t frame_type
, uint64_t stream_id
,
1675 QUIC_TXPIM_PKT
*pkt
, void *arg
)
1677 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1679 switch (frame_type
) {
1680 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING
:
1683 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1688 s
->acked_stop_sending
= 1;
1689 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1692 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM
:
1695 = ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1701 * We must already be in RESET_SENT or RESET_RECVD if we are
1702 * here, so we don't need to check state here.
1704 ossl_quic_stream_map_notify_reset_stream_acked(txp
->args
.qsm
, s
);
1705 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1714 static int txp_pkt_append_padding(struct txp_pkt
*pkt
,
1715 OSSL_QUIC_TX_PACKETISER
*txp
, size_t num_bytes
)
1722 if (!ossl_assert(pkt
->h_valid
))
1725 if (!ossl_assert(pkt
->tpkt
!= NULL
))
1728 wpkt
= tx_helper_begin(&pkt
->h
);
1732 if (!ossl_quic_wire_encode_padding(wpkt
, num_bytes
)) {
1733 tx_helper_rollback(&pkt
->h
);
1737 if (!tx_helper_commit(&pkt
->h
))
1740 pkt
->tpkt
->ackm_pkt
.num_bytes
+= num_bytes
;
1741 /* Cannot be non-inflight if we have a PADDING frame */
1742 pkt
->tpkt
->ackm_pkt
.is_inflight
= 1;
1746 static void on_sstream_updated(uint64_t stream_id
, void *arg
)
1748 OSSL_QUIC_TX_PACKETISER
*txp
= arg
;
1751 s
= ossl_quic_stream_map_get_by_id(txp
->args
.qsm
, stream_id
);
1755 ossl_quic_stream_map_update_state(txp
->args
.qsm
, s
);
1759 * Returns 1 if we can send that many bytes in closing state, 0 otherwise.
1760 * Also maintains the bytes sent state if it returns a success.
1762 static int try_commit_conn_close(OSSL_QUIC_TX_PACKETISER
*txp
, size_t n
)
1766 /* We can always send the first connection close frame */
1767 if (txp
->closing_bytes_recv
== 0)
1771 * RFC 9000 s. 10.2.1 Closing Connection State:
1772 * To avoid being used for an amplification attack, such
1773 * endpoints MUST limit the cumulative size of packets it sends
1774 * to three times the cumulative size of the packets that are
1775 * received and attributed to the connection.
1777 * An endpoint in the closing state MUST either discard packets
1778 * received from an unvalidated address or limit the cumulative
1779 * size of packets it sends to an unvalidated address to three
1780 * times the size of packets it receives from that address.
1782 res
= txp
->closing_bytes_xmit
+ n
<= txp
->closing_bytes_recv
* 3;
1785 * Attribute the bytes to the connection, if we are allowed to send them
1786 * and this isn't the first closing frame.
1788 if (res
&& txp
->closing_bytes_recv
!= 0)
1789 txp
->closing_bytes_xmit
+= n
;
1793 void ossl_quic_tx_packetiser_record_received_closing_bytes(
1794 OSSL_QUIC_TX_PACKETISER
*txp
, size_t n
)
1796 txp
->closing_bytes_recv
+= n
;
1799 static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER
*txp
,
1800 struct txp_pkt
*pkt
,
1801 int chosen_for_conn_close
,
1802 int *can_be_non_inflight
)
1804 const uint32_t enc_level
= pkt
->h
.enc_level
;
1805 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
1806 const struct archetype_data
*a
= &pkt
->geom
.adata
;
1807 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
1808 struct tx_helper
*h
= &pkt
->h
;
1809 const OSSL_QUIC_FRAME_ACK
*ack
;
1810 OSSL_QUIC_FRAME_ACK ack2
;
1812 tpkt
->ackm_pkt
.largest_acked
= QUIC_PN_INVALID
;
1814 /* ACK Frames (Regenerate) */
1816 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_ACK
1817 && (((txp
->want_ack
& (1UL << pn_space
)) != 0)
1818 || ossl_ackm_is_ack_desired(txp
->args
.ackm
, pn_space
))
1819 && (ack
= ossl_ackm_get_ack_frame(txp
->args
.ackm
, pn_space
)) != NULL
) {
1820 WPACKET
*wpkt
= tx_helper_begin(h
);
1825 /* We do not currently support ECN */
1827 ack2
.ecn_present
= 0;
1829 if (ossl_quic_wire_encode_frame_ack(wpkt
,
1830 txp
->args
.ack_delay_exponent
,
1832 if (!tx_helper_commit(h
))
1835 tpkt
->had_ack_frame
= 1;
1837 if (ack
->num_ack_ranges
> 0)
1838 tpkt
->ackm_pkt
.largest_acked
= ack
->ack_ranges
[0].end
;
1840 if (txp
->ack_tx_cb
!= NULL
)
1841 txp
->ack_tx_cb(&ack2
, pn_space
, txp
->ack_tx_cb_arg
);
1843 tx_helper_rollback(h
);
1847 /* CONNECTION_CLOSE Frames (Regenerate) */
1848 if (a
->allow_conn_close
&& txp
->want_conn_close
&& chosen_for_conn_close
) {
1849 WPACKET
*wpkt
= tx_helper_begin(h
);
1850 OSSL_QUIC_FRAME_CONN_CLOSE f
, *pf
= &txp
->conn_close_frame
;
1857 * Application CONNECTION_CLOSE frames may only be sent in the
1858 * Application PN space, as otherwise they may be sent before a
1859 * connection is authenticated and leak application data. Therefore, if
1860 * we need to send a CONNECTION_CLOSE frame in another PN space and were
1861 * given an application CONNECTION_CLOSE frame, convert it into a
1862 * transport CONNECTION_CLOSE frame, removing any sensitive application
1865 * RFC 9000 s. 10.2.3: "A CONNECTION_CLOSE of type 0x1d MUST be replaced
1866 * by a CONNECTION_CLOSE of type 0x1c when sending the frame in Initial
1867 * or Handshake packets. Otherwise, information about the application
1868 * state might be revealed. Endpoints MUST clear the value of the Reason
1869 * Phrase field and SHOULD use the APPLICATION_ERROR code when
1870 * converting to a CONNECTION_CLOSE of type 0x1c."
1872 if (pn_space
!= QUIC_PN_SPACE_APP
&& pf
->is_app
) {
1876 pf
->error_code
= QUIC_ERR_APPLICATION_ERROR
;
1881 if (ossl_quic_wire_encode_frame_conn_close(wpkt
, pf
)
1882 && WPACKET_get_total_written(wpkt
, &l
)
1883 && try_commit_conn_close(txp
, l
)) {
1884 if (!tx_helper_commit(h
))
1887 tpkt
->had_conn_close
= 1;
1888 *can_be_non_inflight
= 0;
1890 tx_helper_rollback(h
);
1897 static int try_len(size_t space_left
, size_t orig_len
,
1898 size_t base_hdr_len
, size_t lenbytes
,
1899 uint64_t maxn
, size_t *hdr_len
, size_t *payload_len
)
1902 size_t maxn_
= maxn
> SIZE_MAX
? SIZE_MAX
: (size_t)maxn
;
1904 *hdr_len
= base_hdr_len
+ lenbytes
;
1906 if (orig_len
== 0 && space_left
>= *hdr_len
) {
1914 if (n
+ *hdr_len
> space_left
)
1915 n
= (space_left
>= *hdr_len
) ? space_left
- *hdr_len
: 0;
1921 static int determine_len(size_t space_left
, size_t orig_len
,
1922 size_t base_hdr_len
,
1923 uint64_t *hlen
, uint64_t *len
)
1926 size_t chosen_payload_len
= 0;
1927 size_t chosen_hdr_len
= 0;
1928 size_t payload_len
[4], hdr_len
[4];
1929 int i
, valid
[4] = {0};
1931 valid
[0] = try_len(space_left
, orig_len
, base_hdr_len
,
1932 1, OSSL_QUIC_VLINT_1B_MAX
,
1933 &hdr_len
[0], &payload_len
[0]);
1934 valid
[1] = try_len(space_left
, orig_len
, base_hdr_len
,
1935 2, OSSL_QUIC_VLINT_2B_MAX
,
1936 &hdr_len
[1], &payload_len
[1]);
1937 valid
[2] = try_len(space_left
, orig_len
, base_hdr_len
,
1938 4, OSSL_QUIC_VLINT_4B_MAX
,
1939 &hdr_len
[2], &payload_len
[2]);
1940 valid
[3] = try_len(space_left
, orig_len
, base_hdr_len
,
1941 8, OSSL_QUIC_VLINT_8B_MAX
,
1942 &hdr_len
[3], &payload_len
[3]);
1944 for (i
= OSSL_NELEM(valid
) - 1; i
>= 0; --i
)
1945 if (valid
[i
] && payload_len
[i
] >= chosen_payload_len
) {
1946 chosen_payload_len
= payload_len
[i
];
1947 chosen_hdr_len
= hdr_len
[i
];
1951 *hlen
= chosen_hdr_len
;
1952 *len
= chosen_payload_len
;
1957 * Given a CRYPTO frame header with accurate chdr->len and a budget
1958 * (space_left), try to find the optimal value of chdr->len to fill as much of
1959 * the budget as possible. This is slightly hairy because larger values of
1960 * chdr->len cause larger encoded sizes of the length field of the frame, which
1961 * in turn mean less space available for payload data. We check all possible
1962 * encodings and choose the optimal encoding.
1964 static int determine_crypto_len(struct tx_helper
*h
,
1965 OSSL_QUIC_FRAME_CRYPTO
*chdr
,
1971 size_t base_hdr_len
; /* CRYPTO header length without length field */
1973 if (chdr
->len
> SIZE_MAX
)
1976 orig_len
= (size_t)chdr
->len
;
1979 base_hdr_len
= ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr
);
1980 chdr
->len
= orig_len
;
1981 if (base_hdr_len
== 0)
1986 return determine_len(space_left
, orig_len
, base_hdr_len
, hlen
, len
);
1989 static int determine_stream_len(struct tx_helper
*h
,
1990 OSSL_QUIC_FRAME_STREAM
*shdr
,
1996 size_t base_hdr_len
; /* STREAM header length without length field */
1998 if (shdr
->len
> SIZE_MAX
)
2001 orig_len
= (size_t)shdr
->len
;
2004 base_hdr_len
= ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr
);
2005 shdr
->len
= orig_len
;
2006 if (base_hdr_len
== 0)
2009 if (shdr
->has_explicit_len
)
2012 return determine_len(space_left
, orig_len
, base_hdr_len
, hlen
, len
);
2015 static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER
*txp
,
2016 struct txp_pkt
*pkt
,
2017 int *have_ack_eliciting
)
2019 const uint32_t enc_level
= pkt
->h
.enc_level
;
2020 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2021 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2022 struct tx_helper
*h
= &pkt
->h
;
2023 size_t num_stream_iovec
;
2024 OSSL_QUIC_FRAME_STREAM shdr
= {0};
2025 OSSL_QUIC_FRAME_CRYPTO chdr
= {0};
2026 OSSL_QTX_IOVEC iov
[2];
2029 QUIC_TXPIM_CHUNK chunk
= {0};
2030 size_t i
, space_left
;
2033 space_left
= tx_helper_get_space_left(h
);
2035 if (space_left
< MIN_FRAME_SIZE_CRYPTO
)
2036 return 1; /* no point trying */
2038 /* Do we have any CRYPTO data waiting? */
2039 num_stream_iovec
= OSSL_NELEM(iov
);
2040 if (!ossl_quic_sstream_get_stream_frame(txp
->args
.crypto
[pn_space
],
2043 return 1; /* nothing to do */
2045 /* Convert STREAM frame header to CRYPTO frame header */
2046 chdr
.offset
= shdr
.offset
;
2047 chdr
.len
= shdr
.len
;
2050 return 1; /* nothing to do */
2052 /* Find best fit (header length, payload length) combination. */
2053 if (!determine_crypto_len(h
, &chdr
, space_left
, &hdr_bytes
,
2055 return 1; /* can't fit anything */
2058 * Truncate IOVs to match our chosen length.
2060 * The length cannot be more than SIZE_MAX because this length comes
2061 * from our send stream buffer.
2063 ossl_quic_sstream_adjust_iov((size_t)chdr
.len
, iov
, num_stream_iovec
);
2066 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2069 if (!txp_el_ensure_iovec(&txp
->el
[enc_level
], h
->num_iovec
+ 3))
2070 return 0; /* alloc error */
2072 /* Encode the header. */
2073 wpkt
= tx_helper_begin(h
);
2075 return 0; /* alloc error */
2077 if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt
, &chdr
)) {
2078 tx_helper_rollback(h
);
2079 return 1; /* can't fit */
2082 if (!tx_helper_commit(h
))
2083 return 0; /* alloc error */
2085 /* Add payload iovecs to the helper (infallible). */
2086 for (i
= 0; i
< num_stream_iovec
; ++i
)
2087 tx_helper_append_iovec(h
, iov
[i
].buf
, iov
[i
].buf_len
);
2089 *have_ack_eliciting
= 1;
2090 tx_helper_unrestrict(h
); /* no longer need PING */
2092 /* Log chunk to TXPIM. */
2093 chunk
.stream_id
= UINT64_MAX
; /* crypto stream */
2094 chunk
.start
= chdr
.offset
;
2095 chunk
.end
= chdr
.offset
+ chdr
.len
- 1;
2096 chunk
.has_fin
= 0; /* Crypto stream never ends */
2097 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2098 return 0; /* alloc error */
2103 OSSL_QUIC_FRAME_STREAM shdr
;
2105 OSSL_QTX_IOVEC iov
[2];
2106 size_t num_stream_iovec
;
2110 static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER
*txp
,
2111 struct tx_helper
*h
,
2112 QUIC_SSTREAM
*sstream
,
2113 QUIC_TXFC
*stream_txfc
,
2115 struct chunk_info
*chunk
,
2118 uint64_t fc_credit
, fc_swm
, fc_limit
;
2120 chunk
->num_stream_iovec
= OSSL_NELEM(chunk
->iov
);
2121 chunk
->valid
= ossl_quic_sstream_get_stream_frame(sstream
, skip
,
2124 &chunk
->num_stream_iovec
);
2128 if (!ossl_assert(chunk
->shdr
.len
> 0 || chunk
->shdr
.is_fin
))
2129 /* Should only have 0-length chunk if FIN */
2132 chunk
->orig_len
= chunk
->shdr
.len
;
2134 /* Clamp according to connection and stream-level TXFC. */
2135 fc_credit
= ossl_quic_txfc_get_credit(stream_txfc
, consumed
);
2136 fc_swm
= ossl_quic_txfc_get_swm(stream_txfc
);
2137 fc_limit
= fc_swm
+ fc_credit
;
2139 if (chunk
->shdr
.len
> 0 && chunk
->shdr
.offset
+ chunk
->shdr
.len
> fc_limit
) {
2140 chunk
->shdr
.len
= (fc_limit
<= chunk
->shdr
.offset
)
2141 ? 0 : fc_limit
- chunk
->shdr
.offset
;
2142 chunk
->shdr
.is_fin
= 0;
2145 if (chunk
->shdr
.len
== 0 && !chunk
->shdr
.is_fin
) {
2147 * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
2148 * order of offset we don't need to check any later chunks, so stop
2159 * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
2160 * *packet_full is set to 1 if there is no longer enough room for another STREAM
2163 static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER
*txp
,
2164 struct txp_pkt
*pkt
,
2166 QUIC_SSTREAM
*sstream
,
2167 QUIC_TXFC
*stream_txfc
,
2168 QUIC_STREAM
*next_stream
,
2169 int *have_ack_eliciting
,
2171 uint64_t *new_credit_consumed
,
2172 uint64_t conn_consumed
)
2175 struct chunk_info chunks
[2] = {0};
2176 const uint32_t enc_level
= pkt
->h
.enc_level
;
2177 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2178 struct tx_helper
*h
= &pkt
->h
;
2179 OSSL_QUIC_FRAME_STREAM
*shdr
;
2181 QUIC_TXPIM_CHUNK chunk
;
2182 size_t i
, j
, space_left
;
2183 int can_fill_payload
, use_explicit_len
;
2184 int could_have_following_chunk
;
2186 uint64_t hdr_len_implicit
, payload_len_implicit
;
2187 uint64_t hdr_len_explicit
, payload_len_explicit
;
2188 uint64_t fc_swm
, fc_new_hwm
;
2190 fc_swm
= ossl_quic_txfc_get_swm(stream_txfc
);
2191 fc_new_hwm
= fc_swm
;
2194 * Load the first two chunks if any offered by the send stream. We retrieve
2195 * the next chunk in advance so we can determine if we need to send any more
2196 * chunks from the same stream after this one, which is needed when
2197 * determining when we can use an implicit length in a STREAM frame.
2199 for (i
= 0; i
< 2; ++i
) {
2200 if (!txp_plan_stream_chunk(txp
, h
, sstream
, stream_txfc
, i
, &chunks
[i
],
2204 if (i
== 0 && !chunks
[i
].valid
) {
2205 /* No chunks, nothing to do. */
2212 space_left
= tx_helper_get_space_left(h
);
2214 if (!chunks
[i
% 2].valid
) {
2215 /* Out of chunks; we're done. */
2220 if (space_left
< MIN_FRAME_SIZE_STREAM
) {
2226 if (!ossl_assert(!h
->done_implicit
))
2228 * Logic below should have ensured we didn't append an
2229 * implicit-length unless we filled the packet or didn't have
2230 * another stream to handle, so this should not be possible.
2234 shdr
= &chunks
[i
% 2].shdr
;
2235 orig_len
= chunks
[i
% 2].orig_len
;
2237 /* Load next chunk for lookahead. */
2238 if (!txp_plan_stream_chunk(txp
, h
, sstream
, stream_txfc
, i
+ 1,
2239 &chunks
[(i
+ 1) % 2], conn_consumed
))
2243 * Find best fit (header length, payload length) combination for if we
2244 * use an implicit length.
2246 shdr
->has_explicit_len
= 0;
2247 hdr_len_implicit
= payload_len_implicit
= 0;
2248 if (!determine_stream_len(h
, shdr
, space_left
,
2249 &hdr_len_implicit
, &payload_len_implicit
)) {
2252 goto err
; /* can't fit anything */
2256 * If there is a next stream, we don't use the implicit length so we can
2257 * add more STREAM frames after this one, unless there is enough data
2258 * for this STREAM frame to fill the packet.
2260 can_fill_payload
= (hdr_len_implicit
+ payload_len_implicit
2264 * Is there is a stream after this one, or another chunk pending
2265 * transmission in this stream?
2267 could_have_following_chunk
2268 = (next_stream
!= NULL
|| chunks
[(i
+ 1) % 2].valid
);
2270 /* Choose between explicit or implicit length representations. */
2271 use_explicit_len
= !((can_fill_payload
|| !could_have_following_chunk
)
2272 && !pkt
->force_pad
);
2274 if (use_explicit_len
) {
2276 * Find best fit (header length, payload length) combination for if
2277 * we use an explicit length.
2279 shdr
->has_explicit_len
= 1;
2280 hdr_len_explicit
= payload_len_explicit
= 0;
2281 if (!determine_stream_len(h
, shdr
, space_left
,
2282 &hdr_len_explicit
, &payload_len_explicit
)) {
2285 goto err
; /* can't fit anything */
2288 shdr
->len
= payload_len_explicit
;
2291 shdr
->has_explicit_len
= 0;
2292 shdr
->len
= payload_len_implicit
;
2295 /* If this is a FIN, don't keep filling the packet with more FINs. */
2297 chunks
[(i
+ 1) % 2].valid
= 0;
2300 * We are now committed to our length (shdr->len can't change).
2301 * If we truncated the chunk, clear the FIN bit.
2303 if (shdr
->len
< orig_len
)
2306 /* Truncate IOVs to match our chosen length. */
2307 ossl_quic_sstream_adjust_iov((size_t)shdr
->len
, chunks
[i
% 2].iov
,
2308 chunks
[i
% 2].num_stream_iovec
);
2311 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
2314 if (!txp_el_ensure_iovec(&txp
->el
[enc_level
], h
->num_iovec
+ 3))
2315 goto err
; /* alloc error */
2317 /* Encode the header. */
2318 wpkt
= tx_helper_begin(h
);
2320 goto err
; /* alloc error */
2322 shdr
->stream_id
= id
;
2323 if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt
, shdr
))) {
2324 /* (Should not be possible.) */
2325 tx_helper_rollback(h
);
2328 goto err
; /* can't fit */
2331 if (!tx_helper_commit(h
))
2332 goto err
; /* alloc error */
2334 /* Add payload iovecs to the helper (infallible). */
2335 for (j
= 0; j
< chunks
[i
% 2].num_stream_iovec
; ++j
)
2336 tx_helper_append_iovec(h
, chunks
[i
% 2].iov
[j
].buf
,
2337 chunks
[i
% 2].iov
[j
].buf_len
);
2339 *have_ack_eliciting
= 1;
2340 tx_helper_unrestrict(h
); /* no longer need PING */
2341 if (!shdr
->has_explicit_len
)
2342 h
->done_implicit
= 1;
2344 /* Log new TXFC credit which was consumed. */
2345 if (shdr
->len
> 0 && shdr
->offset
+ shdr
->len
> fc_new_hwm
)
2346 fc_new_hwm
= shdr
->offset
+ shdr
->len
;
2348 /* Log chunk to TXPIM. */
2349 chunk
.stream_id
= shdr
->stream_id
;
2350 chunk
.start
= shdr
->offset
;
2351 chunk
.end
= shdr
->offset
+ shdr
->len
- 1;
2352 chunk
.has_fin
= shdr
->is_fin
;
2353 chunk
.has_stop_sending
= 0;
2354 chunk
.has_reset_stream
= 0;
2355 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2356 goto err
; /* alloc error */
2358 if (shdr
->len
< orig_len
) {
2360 * If we did not serialize all of this chunk we definitely do not
2361 * want to try the next chunk
2369 *new_credit_consumed
= fc_new_hwm
- fc_swm
;
2373 static void txp_enlink_tmp(QUIC_STREAM
**tmp_head
, QUIC_STREAM
*stream
)
2375 stream
->txp_next
= *tmp_head
;
2379 static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER
*txp
,
2380 struct txp_pkt
*pkt
,
2381 int *have_ack_eliciting
,
2382 QUIC_STREAM
**tmp_head
)
2384 QUIC_STREAM_ITER it
;
2387 QUIC_STREAM
*stream
, *snext
;
2388 struct tx_helper
*h
= &pkt
->h
;
2389 uint64_t conn_consumed
= 0;
2391 for (ossl_quic_stream_iter_init(&it
, txp
->args
.qsm
, 1);
2392 it
.stream
!= NULL
;) {
2395 ossl_quic_stream_iter_next(&it
);
2398 stream
->txp_sent_fc
= 0;
2399 stream
->txp_sent_stop_sending
= 0;
2400 stream
->txp_sent_reset_stream
= 0;
2401 stream
->txp_blocked
= 0;
2402 stream
->txp_txfc_new_credit_consumed
= 0;
2404 /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
2405 if (stream
->want_stop_sending
) {
2406 OSSL_QUIC_FRAME_STOP_SENDING f
;
2408 wpkt
= tx_helper_begin(h
);
2410 return 0; /* alloc error */
2412 f
.stream_id
= stream
->id
;
2413 f
.app_error_code
= stream
->stop_sending_aec
;
2414 if (!ossl_quic_wire_encode_frame_stop_sending(wpkt
, &f
)) {
2415 tx_helper_rollback(h
); /* can't fit */
2416 txp_enlink_tmp(tmp_head
, stream
);
2420 if (!tx_helper_commit(h
))
2421 return 0; /* alloc error */
2423 *have_ack_eliciting
= 1;
2424 tx_helper_unrestrict(h
); /* no longer need PING */
2425 stream
->txp_sent_stop_sending
= 1;
2428 if (stream
->want_reset_stream
) {
2429 OSSL_QUIC_FRAME_RESET_STREAM f
;
2431 if (!ossl_assert(stream
->send_state
== QUIC_SSTREAM_STATE_RESET_SENT
))
2434 wpkt
= tx_helper_begin(h
);
2436 return 0; /* alloc error */
2438 f
.stream_id
= stream
->id
;
2439 f
.app_error_code
= stream
->reset_stream_aec
;
2440 if (!ossl_quic_stream_send_get_final_size(stream
, &f
.final_size
))
2441 return 0; /* should not be possible */
2443 if (!ossl_quic_wire_encode_frame_reset_stream(wpkt
, &f
)) {
2444 tx_helper_rollback(h
); /* can't fit */
2445 txp_enlink_tmp(tmp_head
, stream
);
2449 if (!tx_helper_commit(h
))
2450 return 0; /* alloc error */
2452 *have_ack_eliciting
= 1;
2453 tx_helper_unrestrict(h
); /* no longer need PING */
2454 stream
->txp_sent_reset_stream
= 1;
2457 * The final size of the stream as indicated by RESET_STREAM is used
2458 * to ensure a consistent view of flow control state by both
2459 * parties; if we happen to send a RESET_STREAM that consumes more
2460 * flow control credit, make sure we account for that.
2462 if (!ossl_assert(f
.final_size
<= ossl_quic_txfc_get_swm(&stream
->txfc
)))
2465 stream
->txp_txfc_new_credit_consumed
2466 = f
.final_size
- ossl_quic_txfc_get_swm(&stream
->txfc
);
2470 * Stream Flow Control Frames (MAX_STREAM_DATA)
2472 * RFC 9000 s. 13.3: "An endpoint SHOULD stop sending MAX_STREAM_DATA
2473 * frames when the receiving part of the stream enters a "Size Known" or
2474 * "Reset Recvd" state." -- In practice, RECV is the only state
2475 * in which it makes sense to generate more MAX_STREAM_DATA frames.
2477 if (stream
->recv_state
== QUIC_RSTREAM_STATE_RECV
2478 && (stream
->want_max_stream_data
2479 || ossl_quic_rxfc_has_cwm_changed(&stream
->rxfc
, 0))) {
2481 wpkt
= tx_helper_begin(h
);
2483 return 0; /* alloc error */
2485 cwm
= ossl_quic_rxfc_get_cwm(&stream
->rxfc
);
2487 if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt
, stream
->id
,
2489 tx_helper_rollback(h
); /* can't fit */
2490 txp_enlink_tmp(tmp_head
, stream
);
2494 if (!tx_helper_commit(h
))
2495 return 0; /* alloc error */
2497 *have_ack_eliciting
= 1;
2498 tx_helper_unrestrict(h
); /* no longer need PING */
2499 stream
->txp_sent_fc
= 1;
2503 * Stream Data Frames (STREAM)
2505 * RFC 9000 s. 3.3: A sender MUST NOT send a STREAM [...] frame for a
2506 * stream in the "Reset Sent" state [or any terminal state]. We don't
2507 * send any more STREAM frames if we are sending, have sent, or are
2508 * planning to send, RESET_STREAM. The other terminal state is Data
2509 * Recvd, but txp_generate_stream_frames() is guaranteed to generate
2510 * nothing in this case.
2512 if (ossl_quic_stream_has_send_buffer(stream
)
2513 && !ossl_quic_stream_send_is_reset(stream
)) {
2514 int packet_full
= 0;
2516 if (!ossl_assert(!stream
->want_reset_stream
))
2519 if (!txp_generate_stream_frames(txp
, pkt
,
2520 stream
->id
, stream
->sstream
,
2525 &stream
->txp_txfc_new_credit_consumed
,
2527 /* Fatal error (allocation, etc.) */
2528 txp_enlink_tmp(tmp_head
, stream
);
2531 conn_consumed
+= stream
->txp_txfc_new_credit_consumed
;
2534 txp_enlink_tmp(tmp_head
, stream
);
2539 txp_enlink_tmp(tmp_head
, stream
);
2545 static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER
*txp
,
2546 struct txp_pkt
*pkt
,
2547 int chosen_for_conn_close
)
2549 int rc
= TXP_ERR_SUCCESS
;
2550 const uint32_t enc_level
= pkt
->h
.enc_level
;
2551 const uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2552 int have_ack_eliciting
= 0, done_pre_token
= 0;
2553 const struct archetype_data a
= pkt
->geom
.adata
;
2555 * Cleared if we encode any non-ACK-eliciting frame type which rules out the
2556 * packet being a non-inflight frame. This means any non-ACK ACK-eliciting
2557 * frame, even PADDING frames. ACK eliciting frames always cause a packet to
2558 * become ineligible for non-inflight treatment so it is not necessary to
2559 * clear this in cases where have_ack_eliciting is set, as it is ignored in
2562 int can_be_non_inflight
= 1;
2563 QUIC_CFQ_ITEM
*cfq_item
;
2564 QUIC_TXPIM_PKT
*tpkt
= NULL
;
2565 struct tx_helper
*h
= &pkt
->h
;
2567 /* Maximum PN reached? */
2568 if (!ossl_quic_pn_valid(txp
->next_pn
[pn_space
]))
2571 if (!ossl_assert(pkt
->tpkt
== NULL
))
2574 if ((pkt
->tpkt
= tpkt
= ossl_quic_txpim_pkt_alloc(txp
->args
.txpim
)) == NULL
)
2578 * Frame Serialization
2579 * ===================
2581 * We now serialize frames into the packet in descending order of priority.
2584 /* HANDSHAKE_DONE (Regenerate) */
2585 if (a
.allow_handshake_done
&& txp
->want_handshake_done
2586 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_HANDSHAKE_DONE
) {
2587 WPACKET
*wpkt
= tx_helper_begin(h
);
2592 if (ossl_quic_wire_encode_frame_handshake_done(wpkt
)) {
2593 tpkt
->had_handshake_done_frame
= 1;
2594 have_ack_eliciting
= 1;
2596 if (!tx_helper_commit(h
))
2599 tx_helper_unrestrict(h
); /* no longer need PING */
2601 tx_helper_rollback(h
);
2605 /* MAX_DATA (Regenerate) */
2607 && (txp
->want_max_data
2608 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 0))
2609 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_DATA
) {
2610 WPACKET
*wpkt
= tx_helper_begin(h
);
2611 uint64_t cwm
= ossl_quic_rxfc_get_cwm(txp
->args
.conn_rxfc
);
2616 if (ossl_quic_wire_encode_frame_max_data(wpkt
, cwm
)) {
2617 tpkt
->had_max_data_frame
= 1;
2618 have_ack_eliciting
= 1;
2620 if (!tx_helper_commit(h
))
2623 tx_helper_unrestrict(h
); /* no longer need PING */
2625 tx_helper_rollback(h
);
2629 /* MAX_STREAMS_BIDI (Regenerate) */
2631 && (txp
->want_max_streams_bidi
2632 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
, 0))
2633 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI
) {
2634 WPACKET
*wpkt
= tx_helper_begin(h
);
2635 uint64_t max_streams
2636 = ossl_quic_rxfc_get_cwm(txp
->args
.max_streams_bidi_rxfc
);
2641 if (ossl_quic_wire_encode_frame_max_streams(wpkt
, /*is_uni=*/0,
2643 tpkt
->had_max_streams_bidi_frame
= 1;
2644 have_ack_eliciting
= 1;
2646 if (!tx_helper_commit(h
))
2649 tx_helper_unrestrict(h
); /* no longer need PING */
2651 tx_helper_rollback(h
);
2655 /* MAX_STREAMS_UNI (Regenerate) */
2657 && (txp
->want_max_streams_uni
2658 || ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
, 0))
2659 && tx_helper_get_space_left(h
) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI
) {
2660 WPACKET
*wpkt
= tx_helper_begin(h
);
2661 uint64_t max_streams
2662 = ossl_quic_rxfc_get_cwm(txp
->args
.max_streams_uni_rxfc
);
2667 if (ossl_quic_wire_encode_frame_max_streams(wpkt
, /*is_uni=*/1,
2669 tpkt
->had_max_streams_uni_frame
= 1;
2670 have_ack_eliciting
= 1;
2672 if (!tx_helper_commit(h
))
2675 tx_helper_unrestrict(h
); /* no longer need PING */
2677 tx_helper_rollback(h
);
2682 for (cfq_item
= ossl_quic_cfq_get_priority_head(txp
->args
.cfq
, pn_space
);
2684 cfq_item
= ossl_quic_cfq_item_get_priority_next(cfq_item
, pn_space
)) {
2685 uint64_t frame_type
= ossl_quic_cfq_item_get_frame_type(cfq_item
);
2686 const unsigned char *encoded
= ossl_quic_cfq_item_get_encoded(cfq_item
);
2687 size_t encoded_len
= ossl_quic_cfq_item_get_encoded_len(cfq_item
);
2689 switch (frame_type
) {
2690 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID
:
2691 if (!a
.allow_new_conn_id
)
2694 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID
:
2695 if (!a
.allow_retire_conn_id
)
2698 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN
:
2699 if (!a
.allow_new_token
)
2703 * NEW_TOKEN frames are handled via GCR, but some
2704 * Regenerate-strategy frames should come before them (namely
2705 * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
2706 * we find a NEW_TOKEN frame, do these now. If there are no
2707 * NEW_TOKEN frames in the GCR queue we will handle these below.
2709 if (!done_pre_token
)
2710 if (txp_generate_pre_token(txp
, pkt
,
2711 chosen_for_conn_close
,
2712 &can_be_non_inflight
))
2716 case OSSL_QUIC_FRAME_TYPE_PATH_RESPONSE
:
2717 if (!a
.allow_path_response
)
2721 * RFC 9000 s. 8.2.2: An endpoint MUST expand datagrams that
2722 * contain a PATH_RESPONSE frame to at least the smallest
2723 * allowed maximum datagram size of 1200 bytes.
2728 if (!a
.allow_cfq_other
)
2734 * If the frame is too big, don't try to schedule any more GCR frames in
2735 * this packet rather than sending subsequent ones out of order.
2737 if (encoded_len
> tx_helper_get_space_left(h
))
2740 if (!tx_helper_append_iovec(h
, encoded
, encoded_len
))
2743 ossl_quic_txpim_pkt_add_cfq_item(tpkt
, cfq_item
);
2745 if (ossl_quic_frame_type_is_ack_eliciting(frame_type
)) {
2746 have_ack_eliciting
= 1;
2747 tx_helper_unrestrict(h
); /* no longer need PING */
2752 * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
2753 * PATH_RESPONSE (as desired) before, do so now.
2755 if (!done_pre_token
)
2756 if (txp_generate_pre_token(txp
, pkt
,
2757 chosen_for_conn_close
,
2758 &can_be_non_inflight
))
2763 if (!txp_generate_crypto_frames(txp
, pkt
, &have_ack_eliciting
))
2766 /* Stream-specific frames */
2767 if (a
.allow_stream_rel
&& txp
->handshake_complete
)
2768 if (!txp_generate_stream_related(txp
, pkt
,
2769 &have_ack_eliciting
,
2774 tx_helper_unrestrict(h
);
2776 if ((a
.require_ack_eliciting
2777 || (txp
->force_ack_eliciting
& (1UL << pn_space
)) != 0)
2778 && !have_ack_eliciting
&& a
.allow_ping
) {
2781 wpkt
= tx_helper_begin(h
);
2785 if (!ossl_quic_wire_encode_frame_ping(wpkt
)
2786 || !tx_helper_commit(h
))
2788 * We treat a request to be ACK-eliciting as a requirement, so this
2793 have_ack_eliciting
= 1;
2796 /* PADDING is added by ossl_quic_tx_packetiser_generate(). */
2802 if (have_ack_eliciting
)
2803 can_be_non_inflight
= 0;
2806 tpkt
->ackm_pkt
.num_bytes
= h
->bytes_appended
+ pkt
->geom
.pkt_overhead
;
2807 tpkt
->ackm_pkt
.pkt_num
= txp
->next_pn
[pn_space
];
2808 /* largest_acked is set in txp_generate_pre_token */
2809 tpkt
->ackm_pkt
.pkt_space
= pn_space
;
2810 tpkt
->ackm_pkt
.is_inflight
= !can_be_non_inflight
;
2811 tpkt
->ackm_pkt
.is_ack_eliciting
= have_ack_eliciting
;
2812 tpkt
->ackm_pkt
.is_pto_probe
= 0;
2813 tpkt
->ackm_pkt
.is_mtu_probe
= 0;
2814 tpkt
->ackm_pkt
.time
= txp
->args
.now(txp
->args
.now_arg
);
2815 tpkt
->pkt_type
= pkt
->phdr
.type
;
2822 * Handler for fatal errors, i.e. errors causing us to abort the entire
2823 * packet rather than just one frame. Examples of such errors include
2824 * allocation errors.
2827 ossl_quic_txpim_pkt_release(txp
->args
.txpim
, tpkt
);
2830 return TXP_ERR_INTERNAL
;
2834 * Commits and queues a packet for transmission. There is no backing out after
2839 * - Sends the packet to the QTX for encryption and transmission;
2841 * - Records the packet as having been transmitted in FIFM. ACKM is informed,
2842 * etc. and the TXPIM record is filed.
2844 * - Informs various subsystems of frames that were sent and clears frame
2845 * wanted flags so that we do not generate the same frames again.
2849 * - pkt is a txp_pkt for the correct EL;
2851 * - pkt->tpkt is valid;
2853 * - pkt->tpkt->ackm_pkt has been fully filled in;
2855 * - Stream chunk records have been appended to pkt->tpkt for STREAM and
2856 * CRYPTO frames, but not for RESET_STREAM or STOP_SENDING frames;
2858 * - The chosen stream list for the packet can be fully walked from
2859 * pkt->stream_head using stream->txp_next;
2861 * - pkt->has_ack_eliciting is set correctly.
2864 static int txp_pkt_commit(OSSL_QUIC_TX_PACKETISER
*txp
,
2865 struct txp_pkt
*pkt
,
2867 int *txpim_pkt_reffed
)
2870 uint32_t enc_level
= pkt
->h
.enc_level
;
2871 uint32_t pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
2872 QUIC_TXPIM_PKT
*tpkt
= pkt
->tpkt
;
2873 QUIC_STREAM
*stream
;
2875 struct archetype_data a
;
2877 *txpim_pkt_reffed
= 0;
2879 /* Cannot send a packet with an empty payload. */
2880 if (pkt
->h
.bytes_appended
== 0)
2883 if (!txp_get_archetype_data(enc_level
, archetype
, &a
))
2886 /* Packet Information for QTX */
2887 txpkt
.hdr
= &pkt
->phdr
;
2888 txpkt
.iovec
= txp
->el
[enc_level
].iovec
;
2889 txpkt
.num_iovec
= pkt
->h
.num_iovec
;
2891 txpkt
.peer
= BIO_ADDR_family(&txp
->args
.peer
) == AF_UNSPEC
2892 ? NULL
: &txp
->args
.peer
;
2893 txpkt
.pn
= txp
->next_pn
[pn_space
];
2894 txpkt
.flags
= OSSL_QTX_PKT_FLAG_COALESCE
; /* always try to coalesce */
2896 /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
2897 for (stream
= pkt
->stream_head
; stream
!= NULL
; stream
= stream
->txp_next
)
2898 if (stream
->txp_sent_stop_sending
|| stream
->txp_sent_reset_stream
) {
2899 /* Log STOP_SENDING/RESET_STREAM chunk to TXPIM. */
2900 QUIC_TXPIM_CHUNK chunk
;
2902 chunk
.stream_id
= stream
->id
;
2903 chunk
.start
= UINT64_MAX
;
2906 chunk
.has_stop_sending
= stream
->txp_sent_stop_sending
;
2907 chunk
.has_reset_stream
= stream
->txp_sent_reset_stream
;
2908 if (!ossl_quic_txpim_pkt_append_chunk(tpkt
, &chunk
))
2909 return 0; /* alloc error */
2912 /* Dispatch to FIFD. */
2913 if (!ossl_quic_fifd_pkt_commit(&txp
->fifd
, tpkt
))
2917 * Transmission and Post-Packet Generation Bookkeeping
2918 * ===================================================
2920 * No backing out anymore - at this point the ACKM has recorded the packet
2921 * as having been sent, so we need to increment our next PN counter, or
2922 * the ACKM will complain when we try to record a duplicate packet with
2923 * the same PN later. At this point actually sending the packet may still
2924 * fail. In this unlikely event it will simply be handled as though it
2925 * were a lost packet.
2927 ++txp
->next_pn
[pn_space
];
2928 *txpim_pkt_reffed
= 1;
2930 /* Send the packet. */
2931 if (!ossl_qtx_write_pkt(txp
->args
.qtx
, &txpkt
))
2935 * Record FC and stream abort frames as sent; deactivate streams which no
2936 * longer have anything to do.
2938 for (stream
= pkt
->stream_head
; stream
!= NULL
; stream
= stream
->txp_next
) {
2939 if (stream
->txp_sent_fc
) {
2940 stream
->want_max_stream_data
= 0;
2941 ossl_quic_rxfc_has_cwm_changed(&stream
->rxfc
, 1);
2944 if (stream
->txp_sent_stop_sending
)
2945 stream
->want_stop_sending
= 0;
2947 if (stream
->txp_sent_reset_stream
)
2948 stream
->want_reset_stream
= 0;
2950 if (stream
->txp_txfc_new_credit_consumed
> 0) {
2951 if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream
->txfc
,
2952 stream
->txp_txfc_new_credit_consumed
)))
2954 * Should not be possible, but we should continue with our
2955 * bookkeeping as we have already committed the packet to the
2956 * FIFD. Just change the value we return.
2960 stream
->txp_txfc_new_credit_consumed
= 0;
2964 * If we no longer need to generate any flow control (MAX_STREAM_DATA),
2965 * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
2966 * the stream is drained of data or TXFC-blocked), we can mark the
2967 * stream as inactive.
2969 ossl_quic_stream_map_update_state(txp
->args
.qsm
, stream
);
2971 if (ossl_quic_stream_has_send_buffer(stream
)
2972 && !ossl_quic_sstream_has_pending(stream
->sstream
)
2973 && ossl_quic_sstream_get_final_size(stream
->sstream
, NULL
))
2975 * Transition to DATA_SENT if stream has a final size and we have
2978 ossl_quic_stream_map_notify_all_data_sent(txp
->args
.qsm
, stream
);
2981 /* We have now sent the packet, so update state accordingly. */
2982 if (tpkt
->ackm_pkt
.is_ack_eliciting
)
2983 txp
->force_ack_eliciting
&= ~(1UL << pn_space
);
2985 if (tpkt
->had_handshake_done_frame
)
2986 txp
->want_handshake_done
= 0;
2988 if (tpkt
->had_max_data_frame
) {
2989 txp
->want_max_data
= 0;
2990 ossl_quic_rxfc_has_cwm_changed(txp
->args
.conn_rxfc
, 1);
2993 if (tpkt
->had_max_streams_bidi_frame
) {
2994 txp
->want_max_streams_bidi
= 0;
2995 ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_bidi_rxfc
, 1);
2998 if (tpkt
->had_max_streams_uni_frame
) {
2999 txp
->want_max_streams_uni
= 0;
3000 ossl_quic_rxfc_has_cwm_changed(txp
->args
.max_streams_uni_rxfc
, 1);
3003 if (tpkt
->had_ack_frame
)
3004 txp
->want_ack
&= ~(1UL << pn_space
);
3006 if (tpkt
->had_conn_close
)
3007 txp
->want_conn_close
= 0;
3010 * Decrement probe request counts if we have sent a packet that meets
3011 * the requirement of a probe, namely being ACK-eliciting.
3013 if (tpkt
->ackm_pkt
.is_ack_eliciting
) {
3014 OSSL_ACKM_PROBE_INFO
*probe_info
3015 = ossl_ackm_get0_probe_request(txp
->args
.ackm
);
3017 if (enc_level
== QUIC_ENC_LEVEL_INITIAL
3018 && probe_info
->anti_deadlock_initial
> 0)
3019 --probe_info
->anti_deadlock_initial
;
3021 if (enc_level
== QUIC_ENC_LEVEL_HANDSHAKE
3022 && probe_info
->anti_deadlock_handshake
> 0)
3023 --probe_info
->anti_deadlock_handshake
;
3025 if (a
.allow_force_ack_eliciting
/* (i.e., not for 0-RTT) */
3026 && probe_info
->pto
[pn_space
] > 0)
3027 --probe_info
->pto
[pn_space
];
3033 /* Ensure the iovec array is at least num elements long. */
3034 static int txp_el_ensure_iovec(struct txp_el
*el
, size_t num
)
3036 OSSL_QTX_IOVEC
*iovec
;
3038 if (el
->alloc_iovec
>= num
)
3041 num
= el
->alloc_iovec
!= 0 ? el
->alloc_iovec
* 2 : 8;
3043 iovec
= OPENSSL_realloc(el
->iovec
, sizeof(OSSL_QTX_IOVEC
) * num
);
3048 el
->alloc_iovec
= num
;
3052 int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER
*txp
,
3053 const OSSL_QUIC_FRAME_CONN_CLOSE
*f
)
3055 char *reason
= NULL
;
3056 size_t reason_len
= f
->reason_len
;
3057 size_t max_reason_len
= txp_get_mdpl(txp
) / 2;
3059 if (txp
->want_conn_close
)
3063 * Arbitrarily limit the length of the reason length string to half of the
3066 if (reason_len
> max_reason_len
)
3067 reason_len
= max_reason_len
;
3069 if (reason_len
> 0) {
3070 reason
= OPENSSL_memdup(f
->reason
, reason_len
);
3075 txp
->conn_close_frame
= *f
;
3076 txp
->conn_close_frame
.reason
= reason
;
3077 txp
->conn_close_frame
.reason_len
= reason_len
;
3078 txp
->want_conn_close
= 1;
3082 void ossl_quic_tx_packetiser_set_msg_callback(OSSL_QUIC_TX_PACKETISER
*txp
,
3083 ossl_msg_cb msg_callback
,
3084 SSL
*msg_callback_ssl
)
3086 txp
->msg_callback
= msg_callback
;
3087 txp
->msg_callback_ssl
= msg_callback_ssl
;
3090 void ossl_quic_tx_packetiser_set_msg_callback_arg(OSSL_QUIC_TX_PACKETISER
*txp
,
3091 void *msg_callback_arg
)
3093 txp
->msg_callback_arg
= msg_callback_arg
;
3096 QUIC_PN
ossl_quic_tx_packetiser_get_next_pn(OSSL_QUIC_TX_PACKETISER
*txp
,
3099 if (pn_space
>= QUIC_PN_SPACE_NUM
)
3102 return txp
->next_pn
[pn_space
];
3105 OSSL_TIME
ossl_quic_tx_packetiser_get_deadline(OSSL_QUIC_TX_PACKETISER
*txp
)
3108 * TXP-specific deadline computations which rely on TXP innards. This is in
3109 * turn relied on by the QUIC_CHANNEL code to determine the channel event
3110 * handling deadline.
3112 OSSL_TIME deadline
= ossl_time_infinite();
3113 uint32_t enc_level
, pn_space
;
3116 * ACK generation is not CC-gated - packets containing only ACKs are allowed
3117 * to bypass CC. We want to generate ACK frames even if we are currently
3118 * restricted by CC so the peer knows we have received data. The generate
3119 * call will take care of selecting the correct packet archetype.
3121 for (enc_level
= QUIC_ENC_LEVEL_INITIAL
;
3122 enc_level
< QUIC_ENC_LEVEL_NUM
;
3124 if (ossl_qtx_is_enc_level_provisioned(txp
->args
.qtx
, enc_level
)) {
3125 pn_space
= ossl_quic_enc_level_to_pn_space(enc_level
);
3126 deadline
= ossl_time_min(deadline
,
3127 ossl_ackm_get_ack_deadline(txp
->args
.ackm
, pn_space
));
3130 /* When will CC let us send more? */
3131 if (txp
->args
.cc_method
->get_tx_allowance(txp
->args
.cc_data
) == 0)
3132 deadline
= ossl_time_min(deadline
,
3133 txp
->args
.cc_method
->get_wakeup_deadline(txp
->args
.cc_data
));