]> git.ipfire.org Git - thirdparty/openssl.git/blame - ssl/quic/quic_txp.c
projects / thirdparty / openssl.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
QUIC APL: Fix locking in XSO code and fix tests
[thirdparty/openssl.git] / ssl / quic / quic_txp.c
CommitLineData
a73078b7
HL		 1/*
2 * Copyright 2022 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10#include "internal/quic_txp.h"
11#include "internal/quic_fifd.h"
12#include "internal/quic_stream_map.h"
13#include "internal/common.h"
14#include <openssl/err.h>
15
16#define MIN_CRYPTO_HDR_SIZE 3
17
18#define MIN_FRAME_SIZE_HANDSHAKE_DONE 1
19#define MIN_FRAME_SIZE_MAX_DATA 2
20#define MIN_FRAME_SIZE_ACK 5
21#define MIN_FRAME_SIZE_CRYPTO (MIN_CRYPTO_HDR_SIZE + 1)
22#define MIN_FRAME_SIZE_STREAM 3 /* minimum useful size (for non-FIN) */
23#define MIN_FRAME_SIZE_MAX_STREAMS_BIDI 2
24#define MIN_FRAME_SIZE_MAX_STREAMS_UNI 2
25
26struct ossl_quic_tx_packetiser_st {
27 OSSL_QUIC_TX_PACKETISER_ARGS args;
28
29 /*
30 * Opaque initial token blob provided by caller. TXP frees using the
31 * callback when it is no longer needed.
32 */
33 const unsigned char *initial_token;
34 size_t initial_token_len;
35 ossl_quic_initial_token_free_fn *initial_token_free_cb;
36 void *initial_token_free_cb_arg;
37
38 /* Subcomponents of the TXP that we own. */
39 QUIC_FIFD fifd; /* QUIC Frame-in-Flight Dispatcher */
40
41 /* Internal state. */
42 uint64_t next_pn[QUIC_PN_SPACE_NUM]; /* Next PN to use in given PN space. */
43 OSSL_TIME last_tx_time; /* Last time a packet was generated, or 0. */
44
45 /* Internal state - frame (re)generation flags. */
46 unsigned int want_handshake_done : 1;
47 unsigned int want_max_data : 1;
48 unsigned int want_max_streams_bidi : 1;
49 unsigned int want_max_streams_uni : 1;
50
51 /* Internal state - frame (re)generation flags - per PN space. */
52 unsigned int want_ack : QUIC_PN_SPACE_NUM;
53 unsigned int force_ack_eliciting : QUIC_PN_SPACE_NUM;
54
55 /*
56 * Internal state - connection close terminal state.
57 * Once this is set, it is not unset unlike other want_ flags - we keep
58 * sending it in every packet.
59 */
60 unsigned int want_conn_close : 1;
61
cda88baf
HL		 62 /* Has the handshake been completed? */
63 unsigned int handshake_complete : 1;
64
a73078b7
HL		 65 OSSL_QUIC_FRAME_CONN_CLOSE conn_close_frame;
66
67 /* Internal state - packet assembly. */
68 unsigned char *scratch; /* scratch buffer for packet assembly */
69 size_t scratch_len; /* number of bytes allocated for scratch */
70 OSSL_QTX_IOVEC *iovec; /* scratch iovec array for use with QTX */
71 size_t alloc_iovec; /* size of iovec array */
72};
73
74/*
75 * The TX helper records state used while generating frames into packets. It
76 * enables serialization into the packet to be done "transactionally" where
77 * serialization of a frame can be rolled back if it fails midway (e.g. if it
78 * does not fit).
79 */
80struct tx_helper {
81 OSSL_QUIC_TX_PACKETISER *txp;
82 /*
83 * The Maximum Packet Payload Length in bytes. This is the amount of
84 * space we have to generate frames into.
85 */
86 size_t max_ppl;
87 /*
88 * Number of bytes we have generated so far.
89 */
90 size_t bytes_appended;
91 /*
92 * Number of scratch bytes in txp->scratch we have used so far. Some iovecs
93 * will reference this scratch buffer. When we need to use more of it (e.g.
94 * when we need to put frame headers somewhere), we append to the scratch
95 * buffer, resizing if necessary, and increase this accordingly.
96 */
97 size_t scratch_bytes;
98 /*
99 * Bytes reserved in the MaxPPL budget. We keep this number of bytes spare
100 * until reserve_allowed is set to 1. Currently this is always at most 1, as
101 * a PING frame takes up one byte and this mechanism is only used to ensure
102 * we can encode a PING frame if we have been asked to ensure a packet is
103 * ACK-eliciting and we are unusure if we are going to add any other
104 * ACK-eliciting frames before we reach our MaxPPL budget.
105 */
106 size_t reserve;
107 /*
108 * Number of iovecs we have currently appended. This is the number of
109 * entries valid in txp->iovec.
110 */
111 size_t num_iovec;
112 /*
113 * Whether we are allowed to make use of the reserve bytes in our MaxPPL
114 * budget. This is used to ensure we have room to append a PING frame later
115 * if we need to. Once we know we will not need to append a PING frame, this
116 * is set to 1.
117 */
118 unsigned int reserve_allowed : 1;
119 /*
120 * Set to 1 if we have appended a STREAM frame with an implicit length. If
121 * this happens we should never append another frame after that frame as it
122 * cannot be validly encoded. This is just a safety check.
123 */
124 unsigned int done_implicit : 1;
125 struct {
126 /*
127 * The fields in this structure are valid if active is set, which means
128 * that a serialization transaction is currently in progress.
129 */
130 unsigned char *data;
131 WPACKET wpkt;
132 unsigned int active : 1;
133 } txn;
134};
135
136static void tx_helper_rollback(struct tx_helper *h);
137static int txp_ensure_iovec(OSSL_QUIC_TX_PACKETISER *txp, size_t num);
138
139/* Initialises the TX helper. */
140static int tx_helper_init(struct tx_helper *h, OSSL_QUIC_TX_PACKETISER *txp,
141 size_t max_ppl, size_t reserve)
142{
143 if (reserve > max_ppl)
144 return 0;
145
146 h->txp = txp;
147 h->max_ppl = max_ppl;
148 h->reserve = reserve;
149 h->num_iovec = 0;
150 h->bytes_appended = 0;
151 h->scratch_bytes = 0;
152 h->reserve_allowed = 0;
153 h->done_implicit = 0;
154 h->txn.data = NULL;
155 h->txn.active = 0;
156
157 if (max_ppl > h->txp->scratch_len) {
158 unsigned char *scratch;
159
160 scratch = OPENSSL_realloc(h->txp->scratch, max_ppl);
161 if (scratch == NULL)
162 return 0;
163
164 h->txp->scratch = scratch;
165 h->txp->scratch_len = max_ppl;
166 }
167
168 return 1;
169}
170
171static void tx_helper_cleanup(struct tx_helper *h)
172{
173 if (h->txn.active)
174 tx_helper_rollback(h);
175
176 h->txp = NULL;
177}
178
179static void tx_helper_unrestrict(struct tx_helper *h)
180{
181 h->reserve_allowed = 1;
182}
183
184/*
185 * Append an extent of memory to the iovec list. The memory must remain
186 * allocated until we finish generating the packet and call the QTX.
187 *
188 * In general, the buffers passed to this function will be from one of two
189 * ranges:
190 *
191 * - Application data contained in stream buffers managed elsewhere
192 * in the QUIC stack; or
193 *
194 * - Control frame data appended into txp->scratch using tx_helper_begin and
195 * tx_helper_commit.
196 *
197 */
198static int tx_helper_append_iovec(struct tx_helper *h,
199 const unsigned char *buf,
200 size_t buf_len)
201{
202 if (buf_len == 0)
203 return 1;
204
205 if (!ossl_assert(!h->done_implicit))
206 return 0;
207
208 if (!txp_ensure_iovec(h->txp, h->num_iovec + 1))
209 return 0;
210
211 h->txp->iovec[h->num_iovec].buf = buf;
212 h->txp->iovec[h->num_iovec].buf_len = buf_len;
213
214 ++h->num_iovec;
215 h->bytes_appended += buf_len;
216 return 1;
217}
218
219/*
220 * How many more bytes of space do we have left in our plaintext packet payload?
221 */
222static size_t tx_helper_get_space_left(struct tx_helper *h)
223{
224 return h->max_ppl
225 - (h->reserve_allowed ? 0 : h->reserve) - h->bytes_appended;
226}
227
228/*
229 * Begin a control frame serialization transaction. This allows the
230 * serialization of the control frame to be backed out if it turns out it won't
231 * fit. Write the control frame to the returned WPACKET. Ensure you always
232 * call tx_helper_rollback or tx_helper_commit (or tx_helper_cleanup). Returns
233 * NULL on failure.
234 */
235static WPACKET *tx_helper_begin(struct tx_helper *h)
236{
237 size_t space_left, len;
238 unsigned char *data;
239
240 if (!ossl_assert(!h->txn.active))
241 return NULL;
242
243 if (!ossl_assert(!h->done_implicit))
244 return NULL;
245
246 data = (unsigned char *)h->txp->scratch + h->scratch_bytes;
247 len = h->txp->scratch_len - h->scratch_bytes;
248
249 space_left = tx_helper_get_space_left(h);
250 if (!ossl_assert(space_left <= len))
251 return NULL;
252
253 if (!WPACKET_init_static_len(&h->txn.wpkt, data, len, 0))
254 return NULL;
255
256 if (!WPACKET_set_max_size(&h->txn.wpkt, space_left)) {
257 WPACKET_cleanup(&h->txn.wpkt);
258 return NULL;
259 }
260
261 h->txn.data = data;
262 h->txn.active = 1;
263 return &h->txn.wpkt;
264}
265
266static void tx_helper_end(struct tx_helper *h, int success)
267{
268 if (success)
269 WPACKET_finish(&h->txn.wpkt);
270 else
271 WPACKET_cleanup(&h->txn.wpkt);
272
273 h->txn.active = 0;
274 h->txn.data = NULL;
275}
276
277/* Abort a control frame serialization transaction. */
278static void tx_helper_rollback(struct tx_helper *h)
279{
280 if (!h->txn.active)
281 return;
282
283 tx_helper_end(h, 0);
284}
285
286/* Commit a control frame. */
287static int tx_helper_commit(struct tx_helper *h)
288{
289 size_t l = 0;
290
291 if (!h->txn.active)
292 return 0;
293
294 if (!WPACKET_get_total_written(&h->txn.wpkt, &l)) {
295 tx_helper_end(h, 0);
296 return 0;
297 }
298
299 if (!tx_helper_append_iovec(h, h->txn.data, l)) {
300 tx_helper_end(h, 0);
301 return 0;
302 }
303
304 h->scratch_bytes += l;
305 tx_helper_end(h, 1);
306 return 1;
307}
308
309static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
310 void *arg);
311static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
312 QUIC_TXPIM_PKT *pkt, void *arg);
313static int sstream_is_pending(QUIC_SSTREAM *sstream);
314static int txp_el_pending(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
7f9d1249 315 uint32_t archetype,
fee8f48e 316 int cc_can_send,
7f9d1249 317 uint32_t *conn_close_enc_level);
a73078b7
HL		 318static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
319 uint32_t archetype,
fee8f48e 320 int cc_can_send,
091f532e
HL		 321 int is_last_in_dgram,
322 int dgram_contains_initial,
134b79c0
HL		 323 int chosen_for_conn_close,
324 int *sent_ack_eliciting);
a73078b7
HL		 325static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp);
326static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
327 size_t pl,
328 uint32_t enc_level,
329 size_t hdr_len,
330 size_t *r);
331static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp);
332static int txp_generate_for_el_actual(OSSL_QUIC_TX_PACKETISER *txp,
333 uint32_t enc_level,
334 uint32_t archetype,
335 size_t min_ppl,
336 size_t max_ppl,
337 size_t pkt_overhead,
7f9d1249 338 QUIC_PKT_HDR *phdr,
134b79c0
HL		 339 int chosen_for_conn_close,
340 int *sent_ack_eliciting);
a73078b7
HL		 341
342OSSL_QUIC_TX_PACKETISER *ossl_quic_tx_packetiser_new(const OSSL_QUIC_TX_PACKETISER_ARGS *args)
343{
344 OSSL_QUIC_TX_PACKETISER *txp;
345
346 if (args == NULL
347 || args->qtx == NULL
348 || args->txpim == NULL
349 || args->cfq == NULL
350 || args->ackm == NULL
351 || args->qsm == NULL
352 || args->conn_txfc == NULL
353 || args->conn_rxfc == NULL) {
354 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
355 return NULL;
356 }
357
358 txp = OPENSSL_zalloc(sizeof(*txp));
359 if (txp == NULL)
360 return NULL;
361
362 txp->args = *args;
363 txp->last_tx_time = ossl_time_zero();
364
365 if (!ossl_quic_fifd_init(&txp->fifd,
366 txp->args.cfq, txp->args.ackm, txp->args.txpim,
367 get_sstream_by_id, txp,
368 on_regen_notify, txp)) {
369 OPENSSL_free(txp);
370 return NULL;
371 }
372
373 return txp;
374}
375
376void ossl_quic_tx_packetiser_free(OSSL_QUIC_TX_PACKETISER *txp)
377{
378 if (txp == NULL)
379 return;
380
381 ossl_quic_tx_packetiser_set_initial_token(txp, NULL, 0, NULL, NULL);
382 ossl_quic_fifd_cleanup(&txp->fifd);
383 OPENSSL_free(txp->iovec);
384 OPENSSL_free(txp->conn_close_frame.reason);
385 OPENSSL_free(txp->scratch);
386 OPENSSL_free(txp);
387}
388
389void ossl_quic_tx_packetiser_set_initial_token(OSSL_QUIC_TX_PACKETISER *txp,
390 const unsigned char *token,
391 size_t token_len,
392 ossl_quic_initial_token_free_fn *free_cb,
393 void *free_cb_arg)
394{
395 if (txp->initial_token != NULL && txp->initial_token_free_cb != NULL)
396 txp->initial_token_free_cb(txp->initial_token, txp->initial_token_len,
397 txp->initial_token_free_cb_arg);
398
399 txp->initial_token = token;
400 txp->initial_token_len = token_len;
401 txp->initial_token_free_cb = free_cb;
402 txp->initial_token_free_cb_arg = free_cb_arg;
403}
404
405int ossl_quic_tx_packetiser_set_cur_dcid(OSSL_QUIC_TX_PACKETISER *txp,
406 const QUIC_CONN_ID *dcid)
407{
408 if (dcid == NULL) {
409 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
410 return 0;
411 }
412
413 txp->args.cur_dcid = *dcid;
414 return 1;
415}
416
417int ossl_quic_tx_packetiser_set_cur_scid(OSSL_QUIC_TX_PACKETISER *txp,
418 const QUIC_CONN_ID *scid)
419{
420 if (scid == NULL) {
421 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
422 return 0;
423 }
424
425 txp->args.cur_scid = *scid;
426 return 1;
427}
428
429/* Change the destination L4 address the TXP uses to send datagrams. */
430int ossl_quic_tx_packetiser_set_peer(OSSL_QUIC_TX_PACKETISER *txp,
431 const BIO_ADDR *peer)
432{
433 if (peer == NULL) {
434 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_NULL_PARAMETER);
435 return 0;
436 }
437
438 txp->args.peer = *peer;
439 return 1;
440}
441
442int ossl_quic_tx_packetiser_discard_enc_level(OSSL_QUIC_TX_PACKETISER *txp,
443 uint32_t enc_level)
444{
445 if (enc_level >= QUIC_ENC_LEVEL_NUM) {
446 ERR_raise(ERR_LIB_SSL, ERR_R_PASSED_INVALID_ARGUMENT);
447 return 0;
448 }
449
450 if (enc_level != QUIC_ENC_LEVEL_0RTT)
451 txp->args.crypto[ossl_quic_enc_level_to_pn_space(enc_level)] = NULL;
452
a73078b7
HL		 453 return 1;
454}
455
cda88baf
HL		 456void ossl_quic_tx_packetiser_notify_handshake_complete(OSSL_QUIC_TX_PACKETISER *txp)
457{
458 txp->handshake_complete = 1;
459}
460
a73078b7
HL		 461void ossl_quic_tx_packetiser_schedule_handshake_done(OSSL_QUIC_TX_PACKETISER *txp)
462{
463 txp->want_handshake_done = 1;
464}
465
466void ossl_quic_tx_packetiser_schedule_ack_eliciting(OSSL_QUIC_TX_PACKETISER *txp,
467 uint32_t pn_space)
468{
469 txp->force_ack_eliciting |= (1UL << pn_space);
470}
471
472#define TXP_ERR_INTERNAL 0 /* Internal (e.g. alloc) error */
473#define TXP_ERR_SUCCESS 1 /* Success */
474#define TXP_ERR_SPACE 2 /* Not enough room for another packet */
475#define TXP_ERR_INPUT 3 /* Invalid/malformed input */
476
04e5226f
HL		 477int ossl_quic_tx_packetiser_has_pending(OSSL_QUIC_TX_PACKETISER *txp,
478 uint32_t archetype,
479 uint32_t flags)
480{
7f9d1249 481 uint32_t enc_level, conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
04e5226f 482 int bypass_cc = ((flags & TX_PACKETISER_BYPASS_CC) != 0);
fee8f48e 483 int cc_can_send;
04e5226f 484
fee8f48e 485 cc_can_send
90699176
HL		 486 = (bypass_cc
487 || txp->args.cc_method->get_tx_allowance(txp->args.cc_data) > 0);
04e5226f
HL		 488
489 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
490 enc_level < QUIC_ENC_LEVEL_NUM;
491 ++enc_level)
fee8f48e
HL		 492 if (txp_el_pending(txp, enc_level, archetype, cc_can_send,
493 &conn_close_enc_level))
04e5226f
HL		 494 return 1;
495
496 return 0;
497}
498
a73078b7
HL		 499/*
500 * Generates a datagram by polling the various ELs to determine if they want to
501 * generate any frames, and generating a datagram which coalesces packets for
502 * any ELs which do.
503 */
504int ossl_quic_tx_packetiser_generate(OSSL_QUIC_TX_PACKETISER *txp,
134b79c0
HL		 505 uint32_t archetype,
506 int *sent_ack_eliciting)
a73078b7 507{
7f9d1249 508 uint32_t enc_level, conn_close_enc_level = QUIC_ENC_LEVEL_NUM;
fee8f48e 509 int have_pkt_for_el[QUIC_ENC_LEVEL_NUM], is_last_in_dgram, cc_can_send;
a73078b7
HL		 510 size_t num_el_in_dgram = 0, pkts_done = 0;
511 int rc;
512
fee8f48e
HL		 513 /*
514 * If CC says we cannot send we still may be able to send any queued probes.
515 */
90699176 516 cc_can_send = (txp->args.cc_method->get_tx_allowance(txp->args.cc_data) > 0);
a73078b7
HL		 517
518 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
519 enc_level < QUIC_ENC_LEVEL_NUM;
520 ++enc_level) {
7f9d1249 521 have_pkt_for_el[enc_level] = txp_el_pending(txp, enc_level, archetype,
fee8f48e 522 cc_can_send,
7f9d1249 523 &conn_close_enc_level);
a73078b7
HL		 524 if (have_pkt_for_el[enc_level])
525 ++num_el_in_dgram;
526 }
527
528 if (num_el_in_dgram == 0)
529 return TX_PACKETISER_RES_NO_PKT;
530
531 /*
532 * Should not be needed, but a sanity check in case anyone else has been
533 * using the QTX.
534 */
535 ossl_qtx_finish_dgram(txp->args.qtx);
536
537 for (enc_level = QUIC_ENC_LEVEL_INITIAL;
538 enc_level < QUIC_ENC_LEVEL_NUM;
539 ++enc_level) {
540 if (!have_pkt_for_el[enc_level])
541 continue;
542
543 is_last_in_dgram = (pkts_done + 1 == num_el_in_dgram);
fee8f48e
HL		 544 rc = txp_generate_for_el(txp, enc_level, archetype, cc_can_send,
545 is_last_in_dgram,
7f9d1249 546 have_pkt_for_el[QUIC_ENC_LEVEL_INITIAL],
134b79c0
HL		 547 enc_level == conn_close_enc_level,
548 sent_ack_eliciting);
a73078b7
HL		 549
550 if (rc != TXP_ERR_SUCCESS) {
551 /*
552 * If we already successfully did at least one, make sure we report
553 * this via the return code.
554 */
555 if (pkts_done > 0)
556 break;
557 else
558 return TX_PACKETISER_RES_FAILURE;
559 }
560
561 ++pkts_done;
562 }
563
564 ossl_qtx_finish_dgram(txp->args.qtx);
565 return TX_PACKETISER_RES_SENT_PKT;
566}
567
568struct archetype_data {
569 unsigned int allow_ack : 1;
570 unsigned int allow_ping : 1;
571 unsigned int allow_crypto : 1;
572 unsigned int allow_handshake_done : 1;
573 unsigned int allow_path_challenge : 1;
574 unsigned int allow_path_response : 1;
575 unsigned int allow_new_conn_id : 1;
576 unsigned int allow_retire_conn_id : 1;
577 unsigned int allow_stream_rel : 1;
578 unsigned int allow_conn_fc : 1;
579 unsigned int allow_conn_close : 1;
580 unsigned int allow_cfq_other : 1;
581 unsigned int allow_new_token : 1;
582 unsigned int allow_force_ack_eliciting : 1;
583};
584
585static const struct archetype_data archetypes[QUIC_ENC_LEVEL_NUM][TX_PACKETISER_ARCHETYPE_NUM] = {
586 /* EL 0(INITIAL) */
587 {
588 /* EL 0(INITIAL) - Archetype 0(NORMAL) */
589 {
590 /*allow_ack =*/ 1,
591 /*allow_ping =*/ 1,
592 /*allow_crypto =*/ 1,
593 /*allow_handshake_done =*/ 0,
594 /*allow_path_challenge =*/ 0,
595 /*allow_path_response =*/ 0,
596 /*allow_new_conn_id =*/ 0,
597 /*allow_retire_conn_id =*/ 0,
598 /*allow_stream_rel =*/ 0,
599 /*allow_conn_fc =*/ 0,
600 /*allow_conn_close =*/ 1,
601 /*allow_cfq_other =*/ 1,
602 /*allow_new_token =*/ 0,
603 /*allow_force_ack_eliciting =*/ 1,
604 },
605 /* EL 0(INITIAL) - Archetype 1(ACK_ONLY) */
606 {
607 /*allow_ack =*/ 1,
608 /*allow_ping =*/ 0,
609 /*allow_crypto =*/ 0,
610 /*allow_handshake_done =*/ 0,
611 /*allow_path_challenge =*/ 0,
612 /*allow_path_response =*/ 0,
613 /*allow_new_conn_id =*/ 0,
614 /*allow_retire_conn_id =*/ 0,
615 /*allow_stream_rel =*/ 0,
616 /*allow_conn_fc =*/ 0,
617 /*allow_conn_close =*/ 0,
618 /*allow_cfq_other =*/ 0,
619 /*allow_new_token =*/ 0,
620 /*allow_force_ack_eliciting =*/ 1,
621 },
622 },
623 /* EL 1(HANDSHAKE) */
624 {
625 /* EL 1(HANDSHAKE) - Archetype 0(NORMAL) */
626 {
627 /*allow_ack =*/ 1,
628 /*allow_ping =*/ 1,
629 /*allow_crypto =*/ 1,
630 /*allow_handshake_done =*/ 0,
631 /*allow_path_challenge =*/ 0,
632 /*allow_path_response =*/ 0,
633 /*allow_new_conn_id =*/ 0,
634 /*allow_retire_conn_id =*/ 0,
635 /*allow_stream_rel =*/ 0,
636 /*allow_conn_fc =*/ 0,
637 /*allow_conn_close =*/ 1,
638 /*allow_cfq_other =*/ 1,
639 /*allow_new_token =*/ 0,
640 /*allow_force_ack_eliciting =*/ 1,
641 },
642 /* EL 1(HANDSHAKE) - Archetype 1(ACK_ONLY) */
643 {
644 /*allow_ack =*/ 1,
645 /*allow_ping =*/ 0,
646 /*allow_crypto =*/ 0,
647 /*allow_handshake_done =*/ 0,
648 /*allow_path_challenge =*/ 0,
649 /*allow_path_response =*/ 0,
650 /*allow_new_conn_id =*/ 0,
651 /*allow_retire_conn_id =*/ 0,
652 /*allow_stream_rel =*/ 0,
653 /*allow_conn_fc =*/ 0,
654 /*allow_conn_close =*/ 0,
655 /*allow_cfq_other =*/ 0,
656 /*allow_new_token =*/ 0,
657 /*allow_force_ack_eliciting =*/ 1,
658 },
659 },
660 /* EL 2(0RTT) */
661 {
662 /* EL 2(0RTT) - Archetype 0(NORMAL) */
663 {
664 /*allow_ack =*/ 0,
665 /*allow_ping =*/ 1,
666 /*allow_crypto =*/ 0,
667 /*allow_handshake_done =*/ 0,
668 /*allow_path_challenge =*/ 0,
669 /*allow_path_response =*/ 0,
670 /*allow_new_conn_id =*/ 1,
671 /*allow_retire_conn_id =*/ 1,
672 /*allow_stream_rel =*/ 1,
673 /*allow_conn_fc =*/ 1,
674 /*allow_conn_close =*/ 1,
675 /*allow_cfq_other =*/ 0,
676 /*allow_new_token =*/ 0,
677 /*allow_force_ack_eliciting =*/ 0,
678 },
679 /* EL 2(0RTT) - Archetype 1(ACK_ONLY) */
680 {
681 /*allow_ack =*/ 0,
682 /*allow_ping =*/ 0,
683 /*allow_crypto =*/ 0,
684 /*allow_handshake_done =*/ 0,
685 /*allow_path_challenge =*/ 0,
686 /*allow_path_response =*/ 0,
687 /*allow_new_conn_id =*/ 0,
688 /*allow_retire_conn_id =*/ 0,
689 /*allow_stream_rel =*/ 0,
690 /*allow_conn_fc =*/ 0,
691 /*allow_conn_close =*/ 0,
692 /*allow_cfq_other =*/ 0,
693 /*allow_new_token =*/ 0,
694 /*allow_force_ack_eliciting =*/ 0,
695 },
696 },
697 /* EL 3(1RTT) */
698 {
699 /* EL 3(1RTT) - Archetype 0(NORMAL) */
700 {
701 /*allow_ack =*/ 1,
702 /*allow_ping =*/ 1,
703 /*allow_crypto =*/ 1,
704 /*allow_handshake_done =*/ 1,
705 /*allow_path_challenge =*/ 0,
706 /*allow_path_response =*/ 0,
707 /*allow_new_conn_id =*/ 1,
708 /*allow_retire_conn_id =*/ 1,
709 /*allow_stream_rel =*/ 1,
710 /*allow_conn_fc =*/ 1,
711 /*allow_conn_close =*/ 1,
712 /*allow_cfq_other =*/ 1,
713 /*allow_new_token =*/ 1,
714 /*allow_force_ack_eliciting =*/ 1,
715 },
716 /* EL 3(1RTT) - Archetype 1(ACK_ONLY) */
717 {
718 /*allow_ack =*/ 1,
719 /*allow_ping =*/ 0,
720 /*allow_crypto =*/ 0,
721 /*allow_handshake_done =*/ 0,
722 /*allow_path_challenge =*/ 0,
723 /*allow_path_response =*/ 0,
724 /*allow_new_conn_id =*/ 0,
725 /*allow_retire_conn_id =*/ 0,
726 /*allow_stream_rel =*/ 0,
727 /*allow_conn_fc =*/ 0,
728 /*allow_conn_close =*/ 0,
729 /*allow_cfq_other =*/ 0,
730 /*allow_new_token =*/ 0,
731 /*allow_force_ack_eliciting =*/ 1,
732 }
733 }
734};
735
736static int txp_get_archetype_data(uint32_t enc_level,
737 uint32_t archetype,
738 struct archetype_data *a)
739{
740 if (enc_level >= QUIC_ENC_LEVEL_NUM
741 || archetype >= TX_PACKETISER_ARCHETYPE_NUM)
742 return 0;
743
744 /* No need to avoid copying this as it should not exceed one int in size. */
745 *a = archetypes[enc_level][archetype];
746 return 1;
747}
748
749/*
750 * Returns 1 if the given EL wants to produce one or more frames.
751 * Always returns 0 if the given EL is discarded.
752 */
753static int txp_el_pending(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
7f9d1249 754 uint32_t archetype,
fee8f48e 755 int cc_can_send,
7f9d1249 756 uint32_t *conn_close_enc_level)
a73078b7
HL		 757{
758 struct archetype_data a;
759 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
760 QUIC_CFQ_ITEM *cfq_item;
761
762 if (!ossl_qtx_is_enc_level_provisioned(txp->args.qtx, enc_level))
763 return 0;
764
7f9d1249
HL		 765 if (*conn_close_enc_level > enc_level)
766 *conn_close_enc_level = enc_level;
767
a73078b7
HL		 768 if (!txp_get_archetype_data(enc_level, archetype, &a))
769 return 0;
770
fee8f48e
HL		 771 /* Do we need to send a PTO probe? */
772 if (a.allow_force_ack_eliciting) {
773 OSSL_ACKM_PROBE_INFO *probe_info
2477e99f 774 = ossl_ackm_get0_probe_request(txp->args.ackm);
fee8f48e
HL		 775
776 if ((enc_level == QUIC_ENC_LEVEL_INITIAL
777 && probe_info->anti_deadlock_initial > 0)
778 || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
779 && probe_info->anti_deadlock_handshake > 0)
780 || probe_info->pto[pn_space] > 0)
781 return 1;
782 }
783
784 if (!cc_can_send)
785 /* If CC says we cannot currently send, we can only send probes. */
786 return 0;
787
a73078b7
HL		 788 /* Does the crypto stream for this EL want to produce anything? */
789 if (a.allow_crypto && sstream_is_pending(txp->args.crypto[pn_space]))
790 return 1;
791
792 /* Does the ACKM for this PN space want to produce anything? */
793 if (a.allow_ack && (ossl_ackm_is_ack_desired(txp->args.ackm, pn_space)
794 || (txp->want_ack & (1UL << pn_space)) != 0))
795 return 1;
796
797 /* Do we need to force emission of an ACK-eliciting packet? */
798 if (a.allow_force_ack_eliciting
799 && (txp->force_ack_eliciting & (1UL << pn_space)) != 0)
800 return 1;
801
802 /* Does the connection-level RXFC want to produce a frame? */
803 if (a.allow_conn_fc && (txp->want_max_data
804 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0)))
805 return 1;
806
807 /* Do we want to produce a MAX_STREAMS frame? */
808 if (a.allow_conn_fc && (txp->want_max_streams_bidi
809 || txp->want_max_streams_uni))
810 return 1;
811
812 /* Do we want to produce a HANDSHAKE_DONE frame? */
813 if (a.allow_handshake_done && txp->want_handshake_done)
814 return 1;
815
816 /* Do we want to produce a CONNECTION_CLOSE frame? */
7f9d1249
HL		 817 if (a.allow_conn_close && txp->want_conn_close &&
818 *conn_close_enc_level == enc_level)
819 /*
820 * This is a bit of a special case since CONNECTION_CLOSE can appear in
821 * most packet types, and when we decide we want to send it this status
822 * isn't tied to a specific EL. So if we want to send it, we send it
823 * only on the lowest non-dropped EL.
824 */
a73078b7
HL		 825 return 1;
826
827 /* Does the CFQ have any frames queued for this PN space? */
828 if (enc_level != QUIC_ENC_LEVEL_0RTT)
829 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
830 cfq_item != NULL;
831 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
832 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
833
834 switch (frame_type) {
835 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
836 if (a.allow_new_conn_id)
837 return 1;
838 break;
839 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
840 if (a.allow_retire_conn_id)
841 return 1;
842 break;
843 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
844 if (a.allow_new_token)
845 return 1;
846 break;
847 default:
848 if (a.allow_cfq_other)
849 return 1;
850 break;
851 }
852 }
853
cda88baf 854 if (a.allow_stream_rel && txp->handshake_complete) {
a73078b7
HL		 855 QUIC_STREAM_ITER it;
856
857 /* If there are any active streams, 0/1-RTT wants to produce a packet.
858 * Whether a stream is on the active list is required to be precise
859 * (i.e., a stream is never on the active list if we cannot produce a
860 * frame for it), and all stream-related frames are governed by
861 * a.allow_stream_rel (i.e., if we can send one type of stream-related
862 * frame, we can send any of them), so we don't need to inspect
863 * individual streams on the active list, just confirm that the active
864 * list is non-empty.
865 */
866 ossl_quic_stream_iter_init(&it, txp->args.qsm, 0);
867 if (it.stream != NULL)
868 return 1;
869 }
870
871 return 0;
872}
873
874static int sstream_is_pending(QUIC_SSTREAM *sstream)
875{
876 OSSL_QUIC_FRAME_STREAM hdr;
877 OSSL_QTX_IOVEC iov[2];
878 size_t num_iov = OSSL_NELEM(iov);
879
880 return ossl_quic_sstream_get_stream_frame(sstream, 0, &hdr, iov, &num_iov);
881}
882
883/*
884 * Generates a packet for a given EL, coalescing it into the current datagram.
885 *
886 * is_last_in_dgram and dgram_contains_initial are used to determine padding
887 * requirements.
888 *
889 * Returns TXP_ERR_* value.
890 */
891static int txp_generate_for_el(OSSL_QUIC_TX_PACKETISER *txp, uint32_t enc_level,
892 uint32_t archetype,
fee8f48e 893 int cc_can_send,
091f532e
HL		 894 int is_last_in_dgram,
895 int dgram_contains_initial,
134b79c0
HL		 896 int chosen_for_conn_close,
897 int *sent_ack_eliciting)
a73078b7 898{
091f532e 899 int must_pad = dgram_contains_initial && is_last_in_dgram;
a73078b7
HL		 900 size_t min_dpl, min_pl, min_ppl, cmpl, cmppl, running_total;
901 size_t mdpl, hdr_len, pkt_overhead, cc_limit;
902 uint64_t cc_limit_;
903 QUIC_PKT_HDR phdr;
a73078b7
HL		 904
905 /* Determine the limit CC imposes on what we can send. */
fee8f48e
HL		 906 if (!cc_can_send) {
907 /*
908 * If we are called when we cannot send, this must be because we want
909 * to generate a probe. In this circumstance, don't clamp based on CC.
910 */
911 cc_limit = SIZE_MAX;
912 } else {
913 /* Allow CC to clamp how much we can send. */
90699176 914 cc_limit_ = txp->args.cc_method->get_tx_allowance(txp->args.cc_data);
fee8f48e
HL		 915 cc_limit = (cc_limit_ > SIZE_MAX ? SIZE_MAX : (size_t)cc_limit_);
916 }
a73078b7
HL		 917
918 /* Assemble packet header. */
919 phdr.type = ossl_quic_enc_level_to_pkt_type(enc_level);
920 phdr.spin_bit = 0;
921 phdr.pn_len = txp_determine_pn_len(txp);
922 phdr.partial = 0;
923 phdr.fixed = 1;
924 phdr.version = QUIC_VERSION_1;
925 phdr.dst_conn_id = txp->args.cur_dcid;
926 phdr.src_conn_id = txp->args.cur_scid;
927
928 /*
929 * We need to know the length of the payload to get an accurate header
930 * length for non-1RTT packets, because the Length field found in
931 * Initial/Handshake/0-RTT packets uses a variable-length encoding. However,
932 * we don't have a good idea of the length of our payload, because the
933 * length of the payload depends on the room in the datagram after fitting
934 * the header, which depends on the size of the header.
935 *
936 * In general, it does not matter if a packet is slightly shorter (because
937 * e.g. we predicted use of a 2-byte length field, but ended up only needing
938 * a 1-byte length field). However this does matter for Initial packets
939 * which must be at least 1200 bytes, which is also the assumed default MTU;
940 * therefore in many cases Initial packets will be padded to 1200 bytes,
941 * which means if we overestimated the header size, we will be short by a
942 * few bytes and the server will ignore the packet for being too short. In
943 * this case, however, such packets always *will* be padded to meet 1200
944 * bytes, which requires a 2-byte length field, so we don't actually need to
945 * worry about this. Thus we estimate the header length assuming a 2-byte
946 * length field here, which should in practice work well in all cases.
947 */
948 phdr.len = OSSL_QUIC_VLINT_2B_MAX - phdr.pn_len;
949
950 if (enc_level == QUIC_ENC_LEVEL_INITIAL) {
951 phdr.token = txp->initial_token;
952 phdr.token_len = txp->initial_token_len;
953 } else {
954 phdr.token = NULL;
955 phdr.token_len = 0;
956 }
957
958 hdr_len = ossl_quic_wire_get_encoded_pkt_hdr_len(phdr.dst_conn_id.id_len,
959 &phdr);
960 if (hdr_len == 0)
961 return TXP_ERR_INPUT;
962
963 /* MinDPL: Minimum total datagram payload length. */
964 min_dpl = must_pad ? QUIC_MIN_INITIAL_DGRAM_LEN : 0;
965
966 /* How much data is already in the current datagram? */
967 running_total = ossl_qtx_get_cur_dgram_len_bytes(txp->args.qtx);
968
969 /* MinPL: Minimum length of the fully encoded packet. */
970 min_pl = running_total < min_dpl ? min_dpl - running_total : 0;
971 if ((uint64_t)min_pl > cc_limit)
972 /*
973 * Congestion control does not allow us to send a packet of adequate
974 * size.
975 */
976 return TXP_ERR_SPACE;
977
978 /* MinPPL: Minimum plaintext payload length needed to meet MinPL. */
979 if (!txp_determine_ppl_from_pl(txp, min_pl, enc_level, hdr_len, &min_ppl))
980 /* MinPL is less than a valid packet size, so just use a MinPPL of 0. */
981 min_ppl = 0;
982
983 /* MDPL: Maximum datagram payload length. */
984 mdpl = txp_get_mdpl(txp);
985
986 /*
987 * CMPL: Maximum encoded packet size we can put into this datagram given any
988 * previous packets coalesced into it.
989 */
990 if (running_total > mdpl)
991 /* Should not be possible, but if it happens: */
992 cmpl = 0;
993 else
994 cmpl = mdpl - running_total;
995
996 /* Clamp CMPL based on congestion control limit. */
997 if (cmpl > cc_limit)
998 cmpl = cc_limit;
999
1000 /* CMPPL: Maximum amount we can put into the current datagram payload. */
1001 if (!txp_determine_ppl_from_pl(txp, cmpl, enc_level, hdr_len, &cmppl))
1002 return TXP_ERR_SPACE;
1003
1004 /* Packet overhead (size of headers, AEAD tag, etc.) */
1005 pkt_overhead = cmpl - cmppl;
1006
1007 return txp_generate_for_el_actual(txp, enc_level, archetype, min_ppl, cmppl,
7f9d1249 1008 pkt_overhead, &phdr,
134b79c0
HL		 1009 chosen_for_conn_close,
1010 sent_ack_eliciting);
a73078b7
HL		 1011}
1012
1013/* Determine how many bytes we should use for the encoded PN. */
1014static size_t txp_determine_pn_len(OSSL_QUIC_TX_PACKETISER *txp)
1015{
1016 return 4; /* TODO(QUIC) */
1017}
1018
1019/* Determine plaintext packet payload length from payload length. */
1020static int txp_determine_ppl_from_pl(OSSL_QUIC_TX_PACKETISER *txp,
1021 size_t pl,
1022 uint32_t enc_level,
1023 size_t hdr_len,
1024 size_t *r)
1025{
1026 if (pl < hdr_len)
1027 return 0;
1028
1029 pl -= hdr_len;
1030
1031 if (!ossl_qtx_calculate_plaintext_payload_len(txp->args.qtx, enc_level,
1032 pl, &pl))
1033 return 0;
1034
1035 *r = pl;
1036 return 1;
1037}
1038
1039static size_t txp_get_mdpl(OSSL_QUIC_TX_PACKETISER *txp)
1040{
1041 return ossl_qtx_get_mdpl(txp->args.qtx);
1042}
1043
1044static QUIC_SSTREAM *get_sstream_by_id(uint64_t stream_id, uint32_t pn_space,
1045 void *arg)
1046{
1047 OSSL_QUIC_TX_PACKETISER *txp = arg;
1048 QUIC_STREAM *s;
1049
1050 if (stream_id == UINT64_MAX)
1051 return txp->args.crypto[pn_space];
1052
1053 s = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1054 if (s == NULL)
1055 return NULL;
1056
1057 return s->sstream;
1058}
1059
1060static void on_regen_notify(uint64_t frame_type, uint64_t stream_id,
1061 QUIC_TXPIM_PKT *pkt, void *arg)
1062{
1063 OSSL_QUIC_TX_PACKETISER *txp = arg;
1064
1065 switch (frame_type) {
1066 case OSSL_QUIC_FRAME_TYPE_HANDSHAKE_DONE:
1067 txp->want_handshake_done = 1;
1068 break;
1069 case OSSL_QUIC_FRAME_TYPE_MAX_DATA:
1070 txp->want_max_data = 1;
1071 break;
1072 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_BIDI:
1073 txp->want_max_streams_bidi = 1;
1074 break;
1075 case OSSL_QUIC_FRAME_TYPE_MAX_STREAMS_UNI:
1076 txp->want_max_streams_uni = 1;
1077 break;
1078 case OSSL_QUIC_FRAME_TYPE_ACK_WITH_ECN:
1079 txp->want_ack |= (1UL << pkt->ackm_pkt.pkt_space);
1080 break;
1081 case OSSL_QUIC_FRAME_TYPE_MAX_STREAM_DATA:
1082 {
1083 QUIC_STREAM *s
1084 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1085
1086 if (s == NULL)
1087 return;
1088
1089 s->want_max_stream_data = 1;
1090 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1091 }
1092 break;
1093 case OSSL_QUIC_FRAME_TYPE_STOP_SENDING:
1094 {
1095 QUIC_STREAM *s
1096 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1097
1098 if (s == NULL)
1099 return;
1100
1101 s->want_stop_sending = 1;
1102 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1103 }
1104 break;
1105 case OSSL_QUIC_FRAME_TYPE_RESET_STREAM:
1106 {
1107 QUIC_STREAM *s
1108 = ossl_quic_stream_map_get_by_id(txp->args.qsm, stream_id);
1109
1110 if (s == NULL)
1111 return;
1112
1113 s->want_reset_stream = 1;
1114 ossl_quic_stream_map_update_state(txp->args.qsm, s);
1115 }
1116 break;
1117 default:
1118 assert(0);
1119 break;
1120 }
1121}
1122
1123static int txp_generate_pre_token(OSSL_QUIC_TX_PACKETISER *txp,
1124 struct tx_helper *h,
1125 QUIC_TXPIM_PKT *tpkt,
1126 uint32_t pn_space,
7f9d1249 1127 struct archetype_data *a,
091f532e 1128 int chosen_for_conn_close)
a73078b7
HL		 1129{
1130 const OSSL_QUIC_FRAME_ACK *ack;
1131 OSSL_QUIC_FRAME_ACK ack2;
1132
1133 tpkt->ackm_pkt.largest_acked = QUIC_PN_INVALID;
1134
1135 /* ACK Frames (Regenerate) */
1136 if (a->allow_ack
1137 && tx_helper_get_space_left(h) >= MIN_FRAME_SIZE_ACK
1138 && (txp->want_ack
1139 || ossl_ackm_is_ack_desired(txp->args.ackm, pn_space))
1140 && (ack = ossl_ackm_get_ack_frame(txp->args.ackm, pn_space)) != NULL) {
1141 WPACKET *wpkt = tx_helper_begin(h);
1142
1143 if (wpkt == NULL)
1144 return 0;
1145
1146 /* We do not currently support ECN */
1147 ack2 = *ack;
1148 ack2.ecn_present = 0;
1149
1150 if (ossl_quic_wire_encode_frame_ack(wpkt,
1151 txp->args.ack_delay_exponent,
1152 &ack2)) {
1153 if (!tx_helper_commit(h))
1154 return 0;
1155
1156 tpkt->had_ack_frame = 1;
1157
1158 if (ack->num_ack_ranges > 0)
1159 tpkt->ackm_pkt.largest_acked = ack->ack_ranges[0].end;
1160 } else {
1161 tx_helper_rollback(h);
1162 }
1163 }
1164
1165 /* CONNECTION_CLOSE Frames (Regenerate) */
7f9d1249 1166 if (a->allow_conn_close && txp->want_conn_close && chosen_for_conn_close) {
a73078b7
HL		 1167 WPACKET *wpkt = tx_helper_begin(h);
1168
1169 if (wpkt == NULL)
1170 return 0;
1171
1172 if (ossl_quic_wire_encode_frame_conn_close(wpkt,
1173 &txp->conn_close_frame)) {
1174 if (!tx_helper_commit(h))
1175 return 0;
1176 } else {
1177 tx_helper_rollback(h);
1178 }
1179 }
1180
1181 return 1;
1182}
1183
1184static int try_len(size_t space_left, size_t orig_len,
1185 size_t base_hdr_len, size_t lenbytes,
1186 uint64_t maxn, size_t *hdr_len, size_t *payload_len)
1187{
1188 size_t n;
1189 size_t maxn_ = maxn > SIZE_MAX ? SIZE_MAX : (size_t)maxn;
1190
1191 *hdr_len = base_hdr_len + lenbytes;
1192
cf06f347
HL		 1193 if (orig_len == 0 && space_left >= *hdr_len) {
1194 *payload_len = 0;
1195 return 1;
1196 }
1197
a73078b7
HL		 1198 n = orig_len;
1199 if (n > maxn_)
1200 n = maxn_;
1201 if (n + *hdr_len > space_left)
1202 n = (space_left >= *hdr_len) ? space_left - *hdr_len : 0;
1203
1204 *payload_len = n;
1205 return n > 0;
1206}
1207
cf06f347
HL		 1208static int determine_len(size_t space_left, size_t orig_len,
1209 size_t base_hdr_len,
1210 uint64_t *hlen, uint64_t *len)
a73078b7 1211{
cf06f347 1212 int ok = 0;
a73078b7
HL		 1213 size_t chosen_payload_len = 0;
1214 size_t chosen_hdr_len = 0;
1215 size_t payload_len[4], hdr_len[4];
1216 int i, valid[4] = {0};
1217
1218 valid[0] = try_len(space_left, orig_len, base_hdr_len,
1219 1, OSSL_QUIC_VLINT_1B_MAX,
1220 &hdr_len[0], &payload_len[0]);
1221 valid[1] = try_len(space_left, orig_len, base_hdr_len,
1222 2, OSSL_QUIC_VLINT_2B_MAX,
1223 &hdr_len[1], &payload_len[1]);
1224 valid[2] = try_len(space_left, orig_len, base_hdr_len,
1225 4, OSSL_QUIC_VLINT_4B_MAX,
1226 &hdr_len[2], &payload_len[2]);
1227 valid[3] = try_len(space_left, orig_len, base_hdr_len,
1228 8, OSSL_QUIC_VLINT_8B_MAX,
1229 &hdr_len[3], &payload_len[3]);
1230
1231 for (i = OSSL_NELEM(valid) - 1; i >= 0; --i)
1232 if (valid[i] && payload_len[i] >= chosen_payload_len) {
1233 chosen_payload_len = payload_len[i];
1234 chosen_hdr_len = hdr_len[i];
cf06f347 1235 ok = 1;
a73078b7
HL		 1236 }
1237
1238 *hlen = chosen_hdr_len;
1239 *len = chosen_payload_len;
cf06f347 1240 return ok;
a73078b7
HL		 1241}
1242
1243/*
1244 * Given a CRYPTO frame header with accurate chdr->len and a budget
1245 * (space_left), try to find the optimal value of chdr->len to fill as much of
1246 * the budget as possible. This is slightly hairy because larger values of
1247 * chdr->len cause larger encoded sizes of the length field of the frame, which
1248 * in turn mean less space available for payload data. We check all possible
1249 * encodings and choose the optimal encoding.
1250 */
1251static int determine_crypto_len(struct tx_helper *h,
1252 OSSL_QUIC_FRAME_CRYPTO *chdr,
1253 size_t space_left,
1254 uint64_t *hlen,
1255 uint64_t *len)
1256{
1257 size_t orig_len;
1258 size_t base_hdr_len; /* CRYPTO header length without length field */
1259
1260 if (chdr->len > SIZE_MAX)
1261 return 0;
1262
1263 orig_len = (size_t)chdr->len;
1264
1265 chdr->len = 0;
1266 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_crypto_hdr(chdr);
1267 chdr->len = orig_len;
1268 if (base_hdr_len == 0)
1269 return 0;
1270
1271 --base_hdr_len;
1272
cf06f347 1273 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
a73078b7
HL		 1274}
1275
1276static int determine_stream_len(struct tx_helper *h,
1277 OSSL_QUIC_FRAME_STREAM *shdr,
1278 size_t space_left,
1279 uint64_t *hlen,
1280 uint64_t *len)
1281{
1282 size_t orig_len;
1283 size_t base_hdr_len; /* STREAM header length without length field */
1284
1285 if (shdr->len > SIZE_MAX)
1286 return 0;
1287
1288 orig_len = (size_t)shdr->len;
1289
1290 shdr->len = 0;
1291 base_hdr_len = ossl_quic_wire_get_encoded_frame_len_stream_hdr(shdr);
1292 shdr->len = orig_len;
1293 if (base_hdr_len == 0)
1294 return 0;
1295
1296 if (shdr->has_explicit_len)
1297 --base_hdr_len;
1298
cf06f347 1299 return determine_len(space_left, orig_len, base_hdr_len, hlen, len);
a73078b7
HL		 1300}
1301
1302static int txp_generate_crypto_frames(OSSL_QUIC_TX_PACKETISER *txp,
1303 struct tx_helper *h,
1304 uint32_t pn_space,
1305 QUIC_TXPIM_PKT *tpkt,
091f532e 1306 int *have_ack_eliciting)
a73078b7
HL		 1307{
1308 size_t num_stream_iovec;
1309 OSSL_QUIC_FRAME_STREAM shdr = {0};
1310 OSSL_QUIC_FRAME_CRYPTO chdr = {0};
1311 OSSL_QTX_IOVEC iov[2];
1312 uint64_t hdr_bytes;
1313 WPACKET *wpkt;
24c1be5c 1314 QUIC_TXPIM_CHUNK chunk = {0};
a73078b7
HL		 1315 size_t i, space_left;
1316
1317 for (i = 0;; ++i) {
1318 space_left = tx_helper_get_space_left(h);
1319
1320 if (space_left < MIN_FRAME_SIZE_CRYPTO)
1321 return 1; /* no point trying */
1322
1323 /* Do we have any CRYPTO data waiting? */
1324 num_stream_iovec = OSSL_NELEM(iov);
1325 if (!ossl_quic_sstream_get_stream_frame(txp->args.crypto[pn_space],
1326 i, &shdr, iov,
1327 &num_stream_iovec))
1328 return 1; /* nothing to do */
1329
1330 /* Convert STREAM frame header to CRYPTO frame header */
1331 chdr.offset = shdr.offset;
1332 chdr.len = shdr.len;
1333
1334 if (chdr.len == 0)
1335 return 1; /* nothing to do */
1336
1337 /* Find best fit (header length, payload length) combination. */
1338 if (!determine_crypto_len(h, &chdr, space_left, &hdr_bytes,
cf06f347 1339 &chdr.len))
a73078b7 1340 return 1; /* can't fit anything */
a73078b7
HL		 1341
1342 /*
1343 * Truncate IOVs to match our chosen length.
1344 *
1345 * The length cannot be more than SIZE_MAX because this length comes
1346 * from our send stream buffer.
1347 */
1348 ossl_quic_sstream_adjust_iov((size_t)chdr.len, iov, num_stream_iovec);
1349
1350 /*
1351 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
1352 * the the stream data.)
1353 */
1354 if (!txp_ensure_iovec(txp, h->num_iovec + 3))
1355 return 0; /* alloc error */
1356
1357 /* Encode the header. */
1358 wpkt = tx_helper_begin(h);
1359 if (wpkt == NULL)
1360 return 0; /* alloc error */
1361
1362 if (!ossl_quic_wire_encode_frame_crypto_hdr(wpkt, &chdr)) {
1363 tx_helper_rollback(h);
1364 return 1; /* can't fit */
1365 }
1366
1367 if (!tx_helper_commit(h))
1368 return 0; /* alloc error */
1369
1370 /* Add payload iovecs to the helper (infallible). */
1371 for (i = 0; i < num_stream_iovec; ++i)
1372 tx_helper_append_iovec(h, iov[i].buf, iov[i].buf_len);
1373
1374 *have_ack_eliciting = 1;
1375 tx_helper_unrestrict(h); /* no longer need PING */
1376
1377 /* Log chunk to TXPIM. */
1378 chunk.stream_id = UINT64_MAX; /* crypto stream */
1379 chunk.start = chdr.offset;
1380 chunk.end = chdr.offset + chdr.len - 1;
1381 chunk.has_fin = 0; /* Crypto stream never ends */
1382 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
1383 return 0; /* alloc error */
1384 }
1385}
1386
1387struct chunk_info {
1388 OSSL_QUIC_FRAME_STREAM shdr;
1389 OSSL_QTX_IOVEC iov[2];
1390 size_t num_stream_iovec;
091f532e 1391 int valid;
a73078b7
HL		 1392};
1393
1394static int txp_plan_stream_chunk(OSSL_QUIC_TX_PACKETISER *txp,
1395 struct tx_helper *h,
1396 QUIC_SSTREAM *sstream,
1397 QUIC_TXFC *stream_txfc,
1398 size_t skip,
1399 struct chunk_info *chunk)
1400{
1401 uint64_t fc_credit, fc_swm, fc_limit;
1402
1403 chunk->num_stream_iovec = OSSL_NELEM(chunk->iov);
1404 chunk->valid = ossl_quic_sstream_get_stream_frame(sstream, skip,
1405 &chunk->shdr,
1406 chunk->iov,
1407 &chunk->num_stream_iovec);
1408 if (!chunk->valid)
1409 return 1;
1410
1411 if (!ossl_assert(chunk->shdr.len > 0 || chunk->shdr.is_fin))
1412 /* Should only have 0-length chunk if FIN */
1413 return 0;
1414
1415 /* Clamp according to connection and stream-level TXFC. */
1416 fc_credit = ossl_quic_txfc_get_credit(stream_txfc);
1417 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
1418 fc_limit = fc_swm + fc_credit;
1419
1420 if (chunk->shdr.len > 0 && chunk->shdr.offset + chunk->shdr.len > fc_limit) {
1421 chunk->shdr.len = (fc_limit <= chunk->shdr.offset)
1422 ? 0 : fc_limit - chunk->shdr.offset;
1423 chunk->shdr.is_fin = 0;
1424 }
1425
1426 if (chunk->shdr.len == 0 && !chunk->shdr.is_fin) {
1427 /*
1428 * Nothing to do due to TXFC. Since SSTREAM returns chunks in ascending
1429 * order of offset we don't need to check any later chunks, so stop
1430 * iterating here.
1431 */
1432 chunk->valid = 0;
1433 return 1;
1434 }
1435
1436 return 1;
1437}
1438
1439/*
1440 * Returns 0 on fatal error (e.g. allocation failure), 1 on success.
1441 * *packet_full is set to 1 if there is no longer enough room for another STREAM
1442 * frame, and *stream_drained is set to 1 if all stream buffers have now been
1443 * sent.
1444 */
1445static int txp_generate_stream_frames(OSSL_QUIC_TX_PACKETISER *txp,
1446 struct tx_helper *h,
1447 uint32_t pn_space,
1448 QUIC_TXPIM_PKT *tpkt,
1449 uint64_t id,
1450 QUIC_SSTREAM *sstream,
1451 QUIC_TXFC *stream_txfc,
1452 QUIC_STREAM *next_stream,
1453 size_t min_ppl,
091f532e
HL		 1454 int *have_ack_eliciting,
1455 int *packet_full,
1456 int *stream_drained,
a73078b7
HL		 1457 uint64_t *new_credit_consumed)
1458{
1459 int rc = 0;
1460 struct chunk_info chunks[2] = {0};
1461
1462 OSSL_QUIC_FRAME_STREAM *shdr;
1463 WPACKET *wpkt;
1464 QUIC_TXPIM_CHUNK chunk;
1465 size_t i, j, space_left;
1466 int needs_padding_if_implicit, can_fill_payload, use_explicit_len;
1467 int could_have_following_chunk;
05f97354 1468 uint64_t orig_len;
a73078b7
HL		 1469 uint64_t hdr_len_implicit, payload_len_implicit;
1470 uint64_t hdr_len_explicit, payload_len_explicit;
1471 uint64_t fc_swm, fc_new_hwm;
1472
1473 fc_swm = ossl_quic_txfc_get_swm(stream_txfc);
1474 fc_new_hwm = fc_swm;
1475
1476 /*
1477 * Load the first two chunks if any offered by the send stream. We retrieve
1478 * the next chunk in advance so we can determine if we need to send any more
1479 * chunks from the same stream after this one, which is needed when
1480 * determining when we can use an implicit length in a STREAM frame.
1481 */
1482 for (i = 0; i < 2; ++i) {
1483 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i, &chunks[i]))
1484 goto err;
1485
1486 if (i == 0 && !chunks[i].valid) {
1487 /* No chunks, nothing to do. */
1488 *stream_drained = 1;
1489 rc = 1;
1490 goto err;
1491 }
1492 }
1493
1494 for (i = 0;; ++i) {
1495 space_left = tx_helper_get_space_left(h);
1496
cf06f347
HL		 1497 if (!chunks[i % 2].valid) {
1498 /* Out of chunks; we're done. */
1499 *stream_drained = 1;
a73078b7
HL		 1500 rc = 1;
1501 goto err;
1502 }
1503
cf06f347
HL		 1504 if (space_left < MIN_FRAME_SIZE_STREAM) {
1505 *packet_full = 1;
a73078b7
HL		 1506 rc = 1;
1507 goto err;
1508 }
1509
1510 if (!ossl_assert(!h->done_implicit))
1511 /*
1512 * Logic below should have ensured we didn't append an
1513 * implicit-length unless we filled the packet or didn't have
1514 * another stream to handle, so this should not be possible.
1515 */
1516 goto err;
1517
1518 shdr = &chunks[i % 2].shdr;
05f97354 1519 orig_len = shdr->len;
a73078b7
HL		 1520 if (i > 0)
1521 /* Load next chunk for lookahead. */
1522 if (!txp_plan_stream_chunk(txp, h, sstream, stream_txfc, i + 1,
1523 &chunks[(i + 1) % 2]))
1524 goto err;
1525
1526 /*
1527 * Find best fit (header length, payload length) combination for if we
1528 * use an implicit length.
1529 */
1530 shdr->has_explicit_len = 0;
1531 hdr_len_implicit = payload_len_implicit = 0;
1532 if (!determine_stream_len(h, shdr, space_left,
cf06f347 1533 &hdr_len_implicit, &payload_len_implicit)) {
a73078b7
HL		 1534 *packet_full = 1;
1535 rc = 1;
1536 goto err; /* can't fit anything */
1537 }
1538
1539 /*
1540 * If using the implicit-length representation would need padding, we
1541 * can't use it.
1542 */
1543 needs_padding_if_implicit = (h->bytes_appended + hdr_len_implicit
1544 + payload_len_implicit < min_ppl);
1545
1546 /*
1547 * If there is a next stream, we don't use the implicit length so we can
1548 * add more STREAM frames after this one, unless there is enough data
1549 * for this STREAM frame to fill the packet.
1550 */
1551 can_fill_payload = (hdr_len_implicit + payload_len_implicit
1552 >= space_left);
1553
1554 /*
1555 * Is there is a stream after this one, or another chunk pending
1556 * transmission in this stream?
1557 */
1558 could_have_following_chunk
1559 = (next_stream != NULL || chunks[(i + 1) % 2].valid);
1560
1561 /* Choose between explicit or implicit length representations. */
1562 use_explicit_len = !((can_fill_payload || !could_have_following_chunk)
1563 && !needs_padding_if_implicit);
1564
1565 if (use_explicit_len) {
1566 /*
1567 * Find best fit (header length, payload length) combination for if
1568 * we use an explicit length.
1569 */
1570 shdr->has_explicit_len = 1;
1571 hdr_len_explicit = payload_len_explicit = 0;
1572 if (!determine_stream_len(h, shdr, space_left,
cf06f347 1573 &hdr_len_explicit, &payload_len_explicit)) {
a73078b7
HL		 1574 *packet_full = 1;
1575 rc = 1;
1576 goto err; /* can't fit anything */
1577 }
1578
1579 shdr->len = payload_len_explicit;
1580 } else {
1581 shdr->has_explicit_len = 0;
1582 shdr->len = payload_len_implicit;
1583 }
1584
cf06f347
HL		 1585 /* If this is a FIN, don't keep filling the packet with more FINs. */
1586 if (shdr->is_fin)
1587 chunks[(i + 1) % 2].valid = 0;
1588
a73078b7
HL		 1589 /* Truncate IOVs to match our chosen length. */
1590 ossl_quic_sstream_adjust_iov((size_t)shdr->len, chunks[i % 2].iov,
1591 chunks[i % 2].num_stream_iovec);
1592
1593 /*
1594 * Ensure we have enough iovecs allocated (1 for the header, up to 2 for
1595 * the the stream data.)
1596 */
1597 if (!txp_ensure_iovec(txp, h->num_iovec + 3))
1598 goto err; /* alloc error */
1599
1600 /* Encode the header. */
1601 wpkt = tx_helper_begin(h);
1602 if (wpkt == NULL)
1603 goto err; /* alloc error */
1604
1605 shdr->stream_id = id;
1606 if (!ossl_assert(ossl_quic_wire_encode_frame_stream_hdr(wpkt, shdr))) {
1607 /* (Should not be possible.) */
1608 tx_helper_rollback(h);
1609 *packet_full = 1;
1610 rc = 1;
1611 goto err; /* can't fit */
1612 }
1613
1614 if (!tx_helper_commit(h))
1615 goto err; /* alloc error */
1616
1617 /* Add payload iovecs to the helper (infallible). */
1618 for (j = 0; j < chunks[i % 2].num_stream_iovec; ++j)
1619 tx_helper_append_iovec(h, chunks[i % 2].iov[j].buf,
1620 chunks[i % 2].iov[j].buf_len);
1621
1622 *have_ack_eliciting = 1;
1623 tx_helper_unrestrict(h); /* no longer need PING */
1624 if (!shdr->has_explicit_len)
1625 h->done_implicit = 1;
1626
1627 /* Log new TXFC credit which was consumed. */
1628 if (shdr->len > 0 && shdr->offset + shdr->len > fc_new_hwm)
1629 fc_new_hwm = shdr->offset + shdr->len;
1630
1631 /* Log chunk to TXPIM. */
1632 chunk.stream_id = shdr->stream_id;
1633 chunk.start = shdr->offset;
1634 chunk.end = shdr->offset + shdr->len - 1;
1635 chunk.has_fin = shdr->is_fin;
1636 chunk.has_stop_sending = 0;
1637 chunk.has_reset_stream = 0;
1638 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
1639 goto err; /* alloc error */
05f97354
HL		 1640
1641 if (shdr->len < orig_len) {
1642 /*
1643 * If we did not serialize all of this chunk we definitely do not
1644 * want to try the next chunk (and we must not mark the stream
1645 * as drained).
1646 */
1647 rc = 1;
1648 goto err;
1649 }
a73078b7
HL		 1650 }
1651
1652err:
1653 *new_credit_consumed = fc_new_hwm - fc_swm;
1654 return rc;
1655}
1656
1657static void txp_enlink_tmp(QUIC_STREAM **tmp_head, QUIC_STREAM *stream)
1658{
1659 stream->txp_next = *tmp_head;
1660 *tmp_head = stream;
1661}
1662
1663static int txp_generate_stream_related(OSSL_QUIC_TX_PACKETISER *txp,
1664 struct tx_helper *h,
1665 uint32_t pn_space,
1666 QUIC_TXPIM_PKT *tpkt,
1667 size_t min_ppl,
091f532e 1668 int *have_ack_eliciting,
a73078b7
HL		 1669 QUIC_STREAM **tmp_head)
1670{
1671 QUIC_STREAM_ITER it;
1672 void *rstream;
1673 WPACKET *wpkt;
1674 uint64_t cwm;
1675 QUIC_STREAM *stream, *snext;
1676
1677 for (ossl_quic_stream_iter_init(&it, txp->args.qsm, 1);
1678 it.stream != NULL;) {
1679
1680 stream = it.stream;
1681 ossl_quic_stream_iter_next(&it);
1682 snext = it.stream;
1683
1684 stream->txp_sent_fc = 0;
1685 stream->txp_sent_stop_sending = 0;
1686 stream->txp_sent_reset_stream = 0;
1687 stream->txp_drained = 0;
1688 stream->txp_blocked = 0;
1689 stream->txp_txfc_new_credit_consumed = 0;
1690
1691 rstream = stream->rstream;
1692
1693 /* Stream Abort Frames (STOP_SENDING, RESET_STREAM) */
1694 if (stream->want_stop_sending) {
1695 OSSL_QUIC_FRAME_STOP_SENDING f;
1696
1697 wpkt = tx_helper_begin(h);
1698 if (wpkt == NULL)
1699 return 0; /* alloc error */
1700
1701 f.stream_id = stream->id;
1702 f.app_error_code = stream->stop_sending_aec;
1703 if (!ossl_quic_wire_encode_frame_stop_sending(wpkt, &f)) {
1704 tx_helper_rollback(h); /* can't fit */
1705 txp_enlink_tmp(tmp_head, stream);
1706 break;
1707 }
1708
1709 if (!tx_helper_commit(h))
1710 return 0; /* alloc error */
1711
1712 *have_ack_eliciting = 1;
1713 tx_helper_unrestrict(h); /* no longer need PING */
1714 stream->txp_sent_stop_sending = 1;
1715 }
1716
1717 if (stream->want_reset_stream) {
1718 OSSL_QUIC_FRAME_RESET_STREAM f;
1719
1720 wpkt = tx_helper_begin(h);
1721 if (wpkt == NULL)
1722 return 0; /* alloc error */
1723
1724 f.stream_id = stream->id;
1725 f.app_error_code = stream->reset_stream_aec;
1726 f.final_size = ossl_quic_sstream_get_cur_size(stream->sstream);
1727 if (!ossl_quic_wire_encode_frame_reset_stream(wpkt, &f)) {
1728 tx_helper_rollback(h); /* can't fit */
1729 txp_enlink_tmp(tmp_head, stream);
1730 break;
1731 }
1732
1733 if (!tx_helper_commit(h))
1734 return 0; /* alloc error */
1735
1736 *have_ack_eliciting = 1;
1737 tx_helper_unrestrict(h); /* no longer need PING */
1738 stream->txp_sent_reset_stream = 1;
1739 }
1740
1741 /* Stream Flow Control Frames (MAX_STREAM_DATA) */
1742 if (rstream != NULL
1743 && (stream->want_max_stream_data
1744 || ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 0))) {
1745
1746 wpkt = tx_helper_begin(h);
1747 if (wpkt == NULL)
1748 return 0; /* alloc error */
1749
1750 cwm = ossl_quic_rxfc_get_cwm(&stream->rxfc);
1751
1752 if (!ossl_quic_wire_encode_frame_max_stream_data(wpkt, stream->id,
1753 cwm)) {
1754 tx_helper_rollback(h); /* can't fit */
1755 txp_enlink_tmp(tmp_head, stream);
1756 break;
1757 }
1758
1759 if (!tx_helper_commit(h))
1760 return 0; /* alloc error */
1761
1762 *have_ack_eliciting = 1;
1763 tx_helper_unrestrict(h); /* no longer need PING */
1764 stream->txp_sent_fc = 1;
1765 }
1766
1767 /* Stream Data Frames (STREAM) */
1768 if (stream->sstream != NULL) {
091f532e 1769 int packet_full = 0, stream_drained = 0;
a73078b7
HL		 1770
1771 if (!txp_generate_stream_frames(txp, h, pn_space, tpkt,
1772 stream->id, stream->sstream,
1773 &stream->txfc,
1774 snext, min_ppl,
1775 have_ack_eliciting,
1776 &packet_full,
1777 &stream_drained,
1778 &stream->txp_txfc_new_credit_consumed)) {
1779 /* Fatal error (allocation, etc.) */
1780 txp_enlink_tmp(tmp_head, stream);
1781 return 0;
1782 }
1783
1784 if (stream_drained)
1785 stream->txp_drained = 1;
1786
1787 if (packet_full) {
1788 txp_enlink_tmp(tmp_head, stream);
1789 break;
1790 }
1791 }
1792
1793 txp_enlink_tmp(tmp_head, stream);
1794 }
1795
1796 return 1;
1797}
1798
1799/*
1800 * Generates a packet for a given EL with the given minimum and maximum
1801 * plaintext packet payload lengths. Returns TXP_ERR_* value.
1802 */
1803static int txp_generate_for_el_actual(OSSL_QUIC_TX_PACKETISER *txp,
1804 uint32_t enc_level,
1805 uint32_t archetype,
1806 size_t min_ppl,
1807 size_t max_ppl,
1808 size_t pkt_overhead,
7f9d1249 1809 QUIC_PKT_HDR *phdr,
134b79c0
HL		 1810 int chosen_for_conn_close,
1811 int *sent_ack_eliciting)
a73078b7
HL		 1812{
1813 int rc = TXP_ERR_SUCCESS;
1814 struct archetype_data a;
1815 uint32_t pn_space = ossl_quic_enc_level_to_pn_space(enc_level);
1816 struct tx_helper h;
091f532e 1817 int have_helper = 0, have_ack_eliciting = 0, done_pre_token = 0;
fee8f48e 1818 int require_ack_eliciting = 0;
a73078b7
HL		 1819 QUIC_CFQ_ITEM *cfq_item;
1820 QUIC_TXPIM_PKT *tpkt = NULL;
1821 OSSL_QTX_PKT pkt;
1822 QUIC_STREAM *tmp_head = NULL, *stream;
fee8f48e 1823 OSSL_ACKM_PROBE_INFO *probe_info
2477e99f 1824 = ossl_ackm_get0_probe_request(txp->args.ackm);
a73078b7
HL		 1825
1826 if (!txp_get_archetype_data(enc_level, archetype, &a))
1827 goto fatal_err;
1828
fee8f48e
HL		 1829 if (a.allow_force_ack_eliciting) {
1830 /*
1831 * Make this packet ACK-eliciting if it has been explicitly requested,
1832 * or if ACKM has requested a probe for this PN space.
1833 */
1834 if ((txp->force_ack_eliciting & (1UL << pn_space)) != 0
1835 || (enc_level == QUIC_ENC_LEVEL_INITIAL
1836 && probe_info->anti_deadlock_initial > 0)
1837 || (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
1838 && probe_info->anti_deadlock_handshake > 0)
1839 || probe_info->pto[pn_space] > 0)
1840 require_ack_eliciting = 1;
1841 }
a73078b7
HL		 1842
1843 /* Minimum cannot be bigger than maximum. */
1844 if (min_ppl > max_ppl)
1845 goto fatal_err;
1846
1847 /* Maximum PN reached? */
1848 if (txp->next_pn[pn_space] >= (((QUIC_PN)1) << 62))
1849 goto fatal_err;
1850
1851 if ((tpkt = ossl_quic_txpim_pkt_alloc(txp->args.txpim)) == NULL)
1852 goto fatal_err;
1853
1854 /*
1855 * Initialise TX helper. If we must be ACK eliciting, reserve 1 byte for
1856 * PING.
1857 */
1858 if (!tx_helper_init(&h, txp, max_ppl, require_ack_eliciting ? 1 : 0))
1859 goto fatal_err;
1860
1861 have_helper = 1;
1862
1863 /*
1864 * Frame Serialization
1865 * ===================
1866 *
1867 * We now serialize frames into the packet in descending order of priority.
1868 */
1869
1870 /* HANDSHAKE_DONE (Regenerate) */
1871 if (a.allow_handshake_done && txp->want_handshake_done
1872 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_HANDSHAKE_DONE) {
1873 WPACKET *wpkt = tx_helper_begin(&h);
1874
1875 if (wpkt == NULL)
1876 goto fatal_err;
1877
1878 if (ossl_quic_wire_encode_frame_handshake_done(wpkt)) {
1879 tpkt->had_handshake_done_frame = 1;
1880 have_ack_eliciting = 1;
1881
1882 if (!tx_helper_commit(&h))
1883 goto fatal_err;
1884
1885 tx_helper_unrestrict(&h); /* no longer need PING */
1886 } else {
1887 tx_helper_rollback(&h);
1888 }
1889 }
1890
1891 /* MAX_DATA (Regenerate) */
1892 if (a.allow_conn_fc
1893 && (txp->want_max_data
1894 || ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 0))
1895 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_DATA) {
1896 WPACKET *wpkt = tx_helper_begin(&h);
1897 uint64_t cwm = ossl_quic_rxfc_get_cwm(txp->args.conn_rxfc);
1898
1899 if (wpkt == NULL)
1900 goto fatal_err;
1901
1902 if (ossl_quic_wire_encode_frame_max_data(wpkt, cwm)) {
1903 tpkt->had_max_data_frame = 1;
1904 have_ack_eliciting = 1;
1905
1906 if (!tx_helper_commit(&h))
1907 goto fatal_err;
1908
1909 tx_helper_unrestrict(&h); /* no longer need PING */
1910 } else {
1911 tx_helper_rollback(&h);
1912 }
1913 }
1914
1915 /* MAX_STREAMS_BIDI (Regenerate) */
1916 /*
1917 * TODO(STREAMS): Once we support multiple streams, add stream count FC
1918 * and plug this in.
1919 */
1920 if (a.allow_conn_fc
1921 && txp->want_max_streams_bidi
1922 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_STREAMS_BIDI) {
1923 WPACKET *wpkt = tx_helper_begin(&h);
1924 uint64_t max_streams = 1; /* TODO */
1925
1926 if (wpkt == NULL)
1927 goto fatal_err;
1928
1929 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/0,
1930 max_streams)) {
1931 tpkt->had_max_streams_bidi_frame = 1;
1932 have_ack_eliciting = 1;
1933
1934 if (!tx_helper_commit(&h))
1935 goto fatal_err;
1936
1937 tx_helper_unrestrict(&h); /* no longer need PING */
1938 } else {
1939 tx_helper_rollback(&h);
1940 }
1941 }
1942
1943 /* MAX_STREAMS_UNI (Regenerate) */
1944 if (a.allow_conn_fc
1945 && txp->want_max_streams_uni
1946 && tx_helper_get_space_left(&h) >= MIN_FRAME_SIZE_MAX_STREAMS_UNI) {
1947 WPACKET *wpkt = tx_helper_begin(&h);
1948 uint64_t max_streams = 0; /* TODO */
1949
1950 if (wpkt == NULL)
1951 goto fatal_err;
1952
1953 if (ossl_quic_wire_encode_frame_max_streams(wpkt, /*is_uni=*/1,
1954 max_streams)) {
1955 tpkt->had_max_streams_uni_frame = 1;
1956 have_ack_eliciting = 1;
1957
1958 if (!tx_helper_commit(&h))
1959 goto fatal_err;
1960
1961 tx_helper_unrestrict(&h); /* no longer need PING */
1962 } else {
1963 tx_helper_rollback(&h);
1964 }
1965 }
1966
1967 /* GCR Frames */
1968 for (cfq_item = ossl_quic_cfq_get_priority_head(txp->args.cfq, pn_space);
1969 cfq_item != NULL;
1970 cfq_item = ossl_quic_cfq_item_get_priority_next(cfq_item, pn_space)) {
1971 uint64_t frame_type = ossl_quic_cfq_item_get_frame_type(cfq_item);
1972 const unsigned char *encoded = ossl_quic_cfq_item_get_encoded(cfq_item);
1973 size_t encoded_len = ossl_quic_cfq_item_get_encoded_len(cfq_item);
1974
1975 switch (frame_type) {
1976 case OSSL_QUIC_FRAME_TYPE_NEW_CONN_ID:
1977 if (!a.allow_new_conn_id)
1978 continue;
1979 break;
1980 case OSSL_QUIC_FRAME_TYPE_RETIRE_CONN_ID:
1981 if (!a.allow_retire_conn_id)
1982 continue;
1983 break;
1984 case OSSL_QUIC_FRAME_TYPE_NEW_TOKEN:
1985 if (!a.allow_new_token)
1986 continue;
1987
1988 /*
1989 * NEW_TOKEN frames are handled via GCR, but some
1990 * Regenerate-strategy frames should come before them (namely
1991 * ACK, CONNECTION_CLOSE, PATH_CHALLENGE and PATH_RESPONSE). If
1992 * we find a NEW_TOKEN frame, do these now. If there are no
1993 * NEW_TOKEN frames in the GCR queue we will handle these below.
1994 */
1995 if (!done_pre_token)
7f9d1249
HL		 1996 if (txp_generate_pre_token(txp, &h, tpkt, pn_space, &a,
1997 chosen_for_conn_close))
a73078b7
HL		 1998 done_pre_token = 1;
1999
2000 break;
2001 default:
2002 if (!a.allow_cfq_other)
2003 continue;
2004 break;
2005 }
2006
2007 /*
2008 * If the frame is too big, don't try to schedule any more GCR frames in
2009 * this packet rather than sending subsequent ones out of order.
2010 */
2011 if (encoded_len > tx_helper_get_space_left(&h))
2012 break;
2013
2014 if (!tx_helper_append_iovec(&h, encoded, encoded_len))
2015 goto fatal_err;
2016
2017 ossl_quic_txpim_pkt_add_cfq_item(tpkt, cfq_item);
2018
2019 if (ossl_quic_frame_type_is_ack_eliciting(frame_type)) {
2020 have_ack_eliciting = 1;
2021 tx_helper_unrestrict(&h); /* no longer need PING */
2022 }
2023 }
2024
2025 /*
2026 * If we didn't generate ACK, CONNECTION_CLOSE, PATH_CHALLENGE or
2027 * PATH_RESPONSE (as desired) before, do so now.
2028 */
2029 if (!done_pre_token)
7f9d1249
HL		 2030 if (txp_generate_pre_token(txp, &h, tpkt, pn_space, &a,
2031 chosen_for_conn_close))
a73078b7
HL		 2032 done_pre_token = 1;
2033
2034 /* CRYPTO Frames */
2035 if (a.allow_crypto)
2036 if (!txp_generate_crypto_frames(txp, &h, pn_space, tpkt,
2037 &have_ack_eliciting))
2038 goto fatal_err;
2039
2040 /* Stream-specific frames */
cda88baf 2041 if (a.allow_stream_rel && txp->handshake_complete)
a73078b7
HL		 2042 if (!txp_generate_stream_related(txp, &h, pn_space, tpkt, min_ppl,
2043 &have_ack_eliciting,
2044 &tmp_head))
2045 goto fatal_err;
2046
2047 /* PING */
2048 tx_helper_unrestrict(&h);
2049
2050 if (require_ack_eliciting && !have_ack_eliciting && a.allow_ping) {
2051 WPACKET *wpkt;
2052
2053 wpkt = tx_helper_begin(&h);
2054 if (wpkt == NULL)
2055 goto fatal_err;
2056
2057 if (!ossl_quic_wire_encode_frame_ping(wpkt)
2058 || !tx_helper_commit(&h))
2059 /*
2060 * We treat a request to be ACK-eliciting as a requirement, so this
2061 * is an error.
2062 */
2063 goto fatal_err;
2064
2065 have_ack_eliciting = 1;
2066 }
2067
2068 /* PADDING */
2069 if (h.bytes_appended < min_ppl) {
2070 WPACKET *wpkt = tx_helper_begin(&h);
2071 if (wpkt == NULL)
2072 goto fatal_err;
2073
2074 if (!ossl_quic_wire_encode_padding(wpkt, min_ppl - h.bytes_appended)
2075 || !tx_helper_commit(&h))
2076 goto fatal_err;
2077 }
2078
2079 /*
2080 * Dispatch
2081 * ========
2082 */
2083 /* ACKM Data */
2084 tpkt->ackm_pkt.num_bytes = h.bytes_appended + pkt_overhead;
2085 tpkt->ackm_pkt.pkt_num = txp->next_pn[pn_space];
2086 /* largest_acked is set in txp_generate_pre_token */
2087 tpkt->ackm_pkt.pkt_space = pn_space;
2088 tpkt->ackm_pkt.is_inflight = 1;
2089 tpkt->ackm_pkt.is_ack_eliciting = have_ack_eliciting;
2090 tpkt->ackm_pkt.is_pto_probe = 0;
2091 tpkt->ackm_pkt.is_mtu_probe = 0;
2092 tpkt->ackm_pkt.time = ossl_time_now();
2093
2094 /* Packet Information for QTX */
2095 pkt.hdr = phdr;
2096 pkt.iovec = txp->iovec;
2097 pkt.num_iovec = h.num_iovec;
2098 pkt.local = NULL;
2099 pkt.peer = BIO_ADDR_family(&txp->args.peer) == AF_UNSPEC
2100 ? NULL : &txp->args.peer;
2101 pkt.pn = txp->next_pn[pn_space];
2102 pkt.flags = OSSL_QTX_PKT_FLAG_COALESCE; /* always try to coalesce */
2103
2104 /* Do TX key update if needed. */
2105 if (enc_level == QUIC_ENC_LEVEL_1RTT) {
2106 uint64_t cur_pkt_count, max_pkt_count;
2107
2108 cur_pkt_count = ossl_qtx_get_cur_epoch_pkt_count(txp->args.qtx, enc_level);
2109 max_pkt_count = ossl_qtx_get_max_epoch_pkt_count(txp->args.qtx, enc_level);
2110
2111 if (cur_pkt_count >= max_pkt_count / 2)
2112 if (!ossl_qtx_trigger_key_update(txp->args.qtx))
2113 goto fatal_err;
2114 }
2115
2116 if (!ossl_assert(h.bytes_appended > 0))
2117 goto fatal_err;
2118
2119 /* Generate TXPIM chunks representing STOP_SENDING and RESET_STREAM frames. */
2120 for (stream = tmp_head; stream != NULL; stream = stream->txp_next)
2121 if (stream->txp_sent_stop_sending || stream->txp_sent_reset_stream) {
2122 /* Log STOP_SENDING chunk to TXPIM. */
2123 QUIC_TXPIM_CHUNK chunk;
2124
2125 chunk.stream_id = stream->id;
2126 chunk.start = UINT64_MAX;
2127 chunk.end = 0;
2128 chunk.has_fin = 0;
2129 chunk.has_stop_sending = stream->txp_sent_stop_sending;
2130 chunk.has_reset_stream = stream->txp_sent_reset_stream;
2131 if (!ossl_quic_txpim_pkt_append_chunk(tpkt, &chunk))
2132 return 0; /* alloc error */
2133 }
2134
2135 /* Dispatch to FIFD. */
2136 if (!ossl_quic_fifd_pkt_commit(&txp->fifd, tpkt))
2137 goto fatal_err;
2138
2139 /* Send the packet. */
2140 if (!ossl_qtx_write_pkt(txp->args.qtx, &pkt))
2141 goto fatal_err;
2142
2143 ++txp->next_pn[pn_space];
2144
2145 /*
2146 * Record FC and stream abort frames as sent; deactivate streams which no
2147 * longer have anything to do.
2148 */
2149 for (stream = tmp_head; stream != NULL; stream = stream->txp_next) {
2150 if (stream->txp_sent_fc) {
2151 stream->want_max_stream_data = 0;
2152 ossl_quic_rxfc_has_cwm_changed(&stream->rxfc, 1);
2153 }
2154
2155 if (stream->txp_sent_stop_sending)
2156 stream->want_stop_sending = 0;
2157
2158 if (stream->txp_sent_reset_stream)
2159 stream->want_reset_stream = 0;
2160
2161 if (stream->txp_txfc_new_credit_consumed > 0) {
2162 if (!ossl_assert(ossl_quic_txfc_consume_credit(&stream->txfc,
2163 stream->txp_txfc_new_credit_consumed)))
2164 /*
2165 * Should not be possible, but we should continue with our
2166 * bookkeeping as we have already committed the packet to the
2167 * FIFD. Just change the value we return.
2168 */
2169 rc = TXP_ERR_INTERNAL;
2170
2171 stream->txp_txfc_new_credit_consumed = 0;
2172 }
2173
2174 /*
2175 * If we no longer need to generate any flow control (MAX_STREAM_DATA),
2176 * STOP_SENDING or RESET_STREAM frames, nor any STREAM frames (because
2177 * the stream is drained of data or TXFC-blocked), we can mark the
2178 * stream as inactive.
2179 */
2180 ossl_quic_stream_map_update_state(txp->args.qsm, stream);
2181
05f97354
HL		 2182 if (stream->txp_drained)
2183 assert(!ossl_quic_sstream_has_pending(stream->sstream));
a73078b7
HL		 2184 }
2185
2186 /* We have now sent the packet, so update state accordingly. */
2187 if (have_ack_eliciting)
2188 txp->force_ack_eliciting &= ~(1UL << pn_space);
2189
2190 if (tpkt->had_handshake_done_frame)
2191 txp->want_handshake_done = 0;
2192
2193 if (tpkt->had_max_data_frame) {
2194 txp->want_max_data = 0;
2195 ossl_quic_rxfc_has_cwm_changed(txp->args.conn_rxfc, 1);
2196 }
2197
2198 if (tpkt->had_max_streams_bidi_frame)
2199 txp->want_max_streams_bidi = 0;
2200
2201 if (tpkt->had_max_streams_uni_frame)
2202 txp->want_max_streams_uni = 0;
2203
2204 if (tpkt->had_ack_frame)
2205 txp->want_ack &= ~(1UL << pn_space);
2206
fee8f48e
HL		 2207 /*
2208 * Decrement probe request counts if we have sent a packet that meets
2209 * the requirement of a probe, namely being ACK-eliciting.
2210 */
2211 if (have_ack_eliciting) {
2212 if (enc_level == QUIC_ENC_LEVEL_INITIAL
2213 && probe_info->anti_deadlock_initial > 0)
2214 --probe_info->anti_deadlock_initial;
2215
2216 if (enc_level == QUIC_ENC_LEVEL_HANDSHAKE
2217 && probe_info->anti_deadlock_handshake > 0)
2218 --probe_info->anti_deadlock_handshake;
2219
2220 if (a.allow_force_ack_eliciting /* (i.e., not for 0-RTT) */
2221 && probe_info->pto[pn_space] > 0)
2222 --probe_info->pto[pn_space];
2223 }
2224
134b79c0
HL		 2225 if (have_ack_eliciting)
2226 *sent_ack_eliciting = 1;
2227
a73078b7
HL		 2228 /* Done. */
2229 tx_helper_cleanup(&h);
2230 return rc;
2231
2232fatal_err:
2233 /*
2234 * Handler for fatal errors, i.e. errors causing us to abort the entire
2235 * packet rather than just one frame. Examples of such errors include
2236 * allocation errors.
2237 */
2238 if (have_helper)
2239 tx_helper_cleanup(&h);
2240 if (tpkt != NULL)
2241 ossl_quic_txpim_pkt_release(txp->args.txpim, tpkt);
2242 return TXP_ERR_INTERNAL;
2243}
2244
2245/* Ensure the iovec array is at least num elements long. */
2246static int txp_ensure_iovec(OSSL_QUIC_TX_PACKETISER *txp, size_t num)
2247{
2248 OSSL_QTX_IOVEC *iovec;
2249
2250 if (txp->alloc_iovec >= num)
2251 return 1;
2252
2253 num = txp->alloc_iovec != 0 ? txp->alloc_iovec * 2 : 8;
2254
2255 iovec = OPENSSL_realloc(txp->iovec, sizeof(OSSL_QTX_IOVEC) * num);
2256 if (iovec == NULL)
2257 return 0;
2258
2259 txp->iovec = iovec;
2260 txp->alloc_iovec = num;
2261 return 1;
2262}
2263
2264int ossl_quic_tx_packetiser_schedule_conn_close(OSSL_QUIC_TX_PACKETISER *txp,
2265 const OSSL_QUIC_FRAME_CONN_CLOSE *f)
2266{
2267 char *reason = NULL;
2268 size_t reason_len = f->reason_len;
2269 size_t max_reason_len = txp_get_mdpl(txp) / 2;
2270
2271 if (txp->want_conn_close)
2272 return 0;
2273
2274 /*
2275 * Arbitrarily limit the length of the reason length string to half of the
2276 * MDPL.
2277 */
2278 if (reason_len > max_reason_len)
2279 reason_len = max_reason_len;
2280
2281 if (reason_len > 0) {
2282 reason = OPENSSL_memdup(f->reason, reason_len);
2283 if (reason == NULL)
2284 return 0;
2285 }
2286
2287 txp->conn_close_frame = *f;
2288 txp->conn_close_frame.reason = reason;
2289 txp->conn_close_frame.reason_len = reason_len;
2290 txp->want_conn_close = 1;
2291 return 1;
2292}
A mirror of the official openssl repository