2 * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
12 #include "bio_local.h"
13 #include "internal/cryptlib.h"
14 #include "internal/safe_math.h"
16 #if !defined(OPENSSL_NO_DGRAM) && !defined(OPENSSL_NO_SOCK)
18 OSSL_SAFE_MATH_UNSIGNED(size_t, size_t)
20 /* ===========================================================================
21 * Byte-wise ring buffer which supports pushing and popping blocks of multiple
25 unsigned char *start
; /* start of buffer */
26 size_t len
; /* size of buffer allocation in bytes */
27 size_t count
; /* number of bytes currently pushed */
29 * These index into start. Where idx[0] == idx[1], the buffer is full
30 * (if count is nonzero) and empty otherwise.
32 size_t idx
[2]; /* 0: head, 1: tail */
35 static int ring_buf_init(struct ring_buf
*r
, size_t nbytes
)
37 r
->start
= OPENSSL_malloc(nbytes
);
42 r
->idx
[0] = r
->idx
[1] = r
->count
= 0;
46 static void ring_buf_destroy(struct ring_buf
*r
)
48 OPENSSL_free(r
->start
);
55 * Get a pointer to the next place to write data to be pushed to the ring buffer
56 * (idx=0), or the next data to be popped from the ring buffer (idx=1). The
57 * pointer is written to *buf and the maximum number of bytes which can be
58 * read/written are written to *len. After writing data to the buffer, call
59 * ring_buf_push/pop() with the number of bytes actually read/written, which
60 * must not exceed the returned length.
62 static void ring_buf_head_tail(struct ring_buf
*r
, int idx
, uint8_t **buf
, size_t *len
)
64 size_t max_len
= r
->len
- r
->idx
[idx
];
66 if (idx
== 0 && max_len
> r
->len
- r
->count
)
67 max_len
= r
->len
- r
->count
;
68 if (idx
== 1 && max_len
> r
->count
)
71 *buf
= (uint8_t *)r
->start
+ r
->idx
[idx
];
75 #define ring_buf_head(r, buf, len) ring_buf_head_tail((r), 0, (buf), (len))
76 #define ring_buf_tail(r, buf, len) ring_buf_head_tail((r), 1, (buf), (len))
79 * Commit bytes to the ring buffer previously filled after a call to
82 static void ring_buf_push_pop(struct ring_buf
*r
, int idx
, size_t num_bytes
)
86 /* A single push/pop op cannot wrap around, though it can reach the end.
87 * If the caller adheres to the convention of using the length returned
88 * by ring_buf_head/tail(), this cannot happen.
90 if (!ossl_assert(num_bytes
<= r
->len
- r
->idx
[idx
]))
94 * Must not overfill the buffer, or pop more than is in the buffer either.
96 if (!ossl_assert(idx
!= 0 ? num_bytes
<= r
->count
97 : num_bytes
+ r
->count
<= r
->len
))
100 /* Update the index. */
101 new_idx
= r
->idx
[idx
] + num_bytes
;
102 if (new_idx
== r
->len
)
105 r
->idx
[idx
] = new_idx
;
107 r
->count
-= num_bytes
;
109 r
->count
+= num_bytes
;
112 #define ring_buf_push(r, num_bytes) ring_buf_push_pop((r), 0, (num_bytes))
113 #define ring_buf_pop(r, num_bytes) ring_buf_push_pop((r), 1, (num_bytes))
115 static void ring_buf_clear(struct ring_buf
*r
)
117 r
->idx
[0] = r
->idx
[1] = r
->count
= 0;
120 static int ring_buf_resize(struct ring_buf
*r
, size_t nbytes
)
122 unsigned char *new_start
;
124 if (r
->start
== NULL
)
125 return ring_buf_init(r
, nbytes
);
127 if (nbytes
== r
->len
)
130 if (r
->count
> 0 && nbytes
< r
->len
)
131 /* fail shrinking the ring buffer when there is any data in it */
134 new_start
= OPENSSL_realloc(r
->start
, nbytes
);
135 if (new_start
== NULL
)
138 /* Moving tail if it is after (or equal to) head */
140 if (r
->idx
[0] <= r
->idx
[1]) {
141 size_t offset
= nbytes
- r
->len
;
143 memmove(new_start
+ r
->idx
[1] + offset
, new_start
+ r
->idx
[1],
148 /* just reset the head/tail because it might be pointing outside */
149 r
->idx
[0] = r
->idx
[1] = 0;
152 r
->start
= new_start
;
158 /* ===========================================================================
159 * BIO_s_dgram_pair is documented in BIO_s_dgram_pair(3).
161 * INTERNAL DATA STRUCTURE
163 * This is managed internally by using a bytewise ring buffer which supports
164 * pushing and popping spans of multiple bytes at once. The ring buffer stores
165 * internal packets which look like this:
167 * struct dgram_hdr hdr;
170 * The header contains the length of the data and metadata such as
171 * source/destination addresses.
173 * The datagram pair BIO is designed to support both traditional
174 * BIO_read/BIO_write (likely to be used by applications) as well as
175 * BIO_recvmmsg/BIO_sendmmsg.
177 struct bio_dgram_pair_st
;
178 static int dgram_pair_write(BIO
*bio
, const char *buf
, int sz_
);
179 static int dgram_pair_read(BIO
*bio
, char *buf
, int sz_
);
180 static int dgram_mem_read(BIO
*bio
, char *buf
, int sz_
);
181 static long dgram_pair_ctrl(BIO
*bio
, int cmd
, long num
, void *ptr
);
182 static long dgram_mem_ctrl(BIO
*bio
, int cmd
, long num
, void *ptr
);
183 static int dgram_pair_init(BIO
*bio
);
184 static int dgram_mem_init(BIO
*bio
);
185 static int dgram_pair_free(BIO
*bio
);
186 static int dgram_pair_sendmmsg(BIO
*b
, BIO_MSG
*msg
, size_t stride
,
187 size_t num_msg
, uint64_t flags
,
188 size_t *num_processed
);
189 static int dgram_pair_recvmmsg(BIO
*b
, BIO_MSG
*msg
, size_t stride
,
190 size_t num_msg
, uint64_t flags
,
191 size_t *num_processed
);
193 static int dgram_pair_ctrl_destroy_bio_pair(BIO
*bio1
);
194 static size_t dgram_pair_read_inner(struct bio_dgram_pair_st
*b
, uint8_t *buf
,
197 #define BIO_MSG_N(array, n) (*(BIO_MSG *)((char *)(array) + (n)*stride))
199 static const BIO_METHOD dgram_pair_method
= {
206 NULL
, /* dgram_pair_puts */
207 NULL
, /* dgram_pair_gets */
211 NULL
, /* dgram_pair_callback_ctrl */
216 static const BIO_METHOD dgram_mem_method
= {
223 NULL
, /* dgram_pair_puts */
224 NULL
, /* dgram_pair_gets */
228 NULL
, /* dgram_pair_callback_ctrl */
233 const BIO_METHOD
*BIO_s_dgram_pair(void)
235 return &dgram_pair_method
;
238 const BIO_METHOD
*BIO_s_dgram_mem(void)
240 return &dgram_mem_method
;
244 size_t len
; /* payload length in bytes, not including this struct */
245 BIO_ADDR src_addr
, dst_addr
; /* family == 0: not present */
248 struct bio_dgram_pair_st
{
249 /* The other half of the BIO pair. NULL for dgram_mem. */
251 /* Writes are directed to our own ringbuf and reads to our peer. */
252 struct ring_buf rbuf
;
253 /* Requested size of rbuf buffer in bytes once we initialize. */
255 /* Largest possible datagram size */
257 /* Capability flags. */
260 * This lock protects updates to our rbuf. Since writes are directed to our
261 * own rbuf, this means we use this lock for writes and our peer's lock for
265 unsigned int no_trunc
: 1; /* Reads fail if they would truncate */
266 unsigned int local_addr_enable
: 1; /* Can use BIO_MSG->local? */
267 unsigned int role
: 1; /* Determines lock order */
268 unsigned int grows_on_write
: 1; /* Set for BIO_s_dgram_mem only */
271 #define MIN_BUF_LEN (1024)
273 #define is_dgram_pair(b) (b->peer != NULL)
275 static int dgram_pair_init(BIO
*bio
)
277 struct bio_dgram_pair_st
*b
= OPENSSL_zalloc(sizeof(*b
));
282 b
->req_buf_len
= 17*1024; /* default buffer size */
283 b
->mtu
= 1472; /* conservative default MTU */
285 b
->lock
= CRYPTO_THREAD_lock_new();
286 if (b
->lock
== NULL
) {
295 static int dgram_mem_init(BIO
*bio
)
297 struct bio_dgram_pair_st
*b
;
299 if (!dgram_pair_init(bio
))
304 if (ring_buf_init(&b
->rbuf
, b
->req_buf_len
) == 0) {
305 ERR_raise(ERR_LIB_BIO
, ERR_R_BIO_LIB
);
309 b
->grows_on_write
= 1;
315 static int dgram_pair_free(BIO
*bio
)
317 struct bio_dgram_pair_st
*b
;
323 if (!ossl_assert(b
!= NULL
))
326 /* We are being freed. Disconnect any peer and destroy buffers. */
327 dgram_pair_ctrl_destroy_bio_pair(bio
);
329 CRYPTO_THREAD_lock_free(b
->lock
);
334 /* BIO_make_bio_pair (BIO_C_MAKE_BIO_PAIR) */
335 static int dgram_pair_ctrl_make_bio_pair(BIO
*bio1
, BIO
*bio2
)
337 struct bio_dgram_pair_st
*b1
, *b2
;
339 /* peer must be non-NULL. */
340 if (bio1
== NULL
|| bio2
== NULL
) {
341 ERR_raise(ERR_LIB_BIO
, BIO_R_INVALID_ARGUMENT
);
345 /* Ensure the BIO we have been passed is actually a dgram pair BIO. */
346 if (bio1
->method
!= &dgram_pair_method
|| bio2
->method
!= &dgram_pair_method
) {
347 ERR_raise_data(ERR_LIB_BIO
, BIO_R_INVALID_ARGUMENT
,
348 "both BIOs must be BIO_dgram_pair");
355 if (!ossl_assert(b1
!= NULL
&& b2
!= NULL
)) {
356 ERR_raise(ERR_LIB_BIO
, BIO_R_UNINITIALIZED
);
361 * This ctrl cannot be used to associate a BIO pair half which is already
364 if (b1
->peer
!= NULL
|| b2
->peer
!= NULL
) {
365 ERR_raise_data(ERR_LIB_BIO
, BIO_R_IN_USE
,
366 "cannot associate a BIO_dgram_pair which is already in use");
370 if (!ossl_assert(b1
->req_buf_len
>= MIN_BUF_LEN
371 && b2
->req_buf_len
>= MIN_BUF_LEN
)) {
372 ERR_raise(ERR_LIB_BIO
, BIO_R_UNINITIALIZED
);
376 if (b1
->rbuf
.len
!= b1
->req_buf_len
)
377 if (ring_buf_init(&b1
->rbuf
, b1
->req_buf_len
) == 0) {
378 ERR_raise(ERR_LIB_BIO
, ERR_R_BIO_LIB
);
382 if (b2
->rbuf
.len
!= b2
->req_buf_len
)
383 if (ring_buf_init(&b2
->rbuf
, b2
->req_buf_len
) == 0) {
384 ERR_raise(ERR_LIB_BIO
, ERR_R_BIO_LIB
);
385 ring_buf_destroy(&b1
->rbuf
);
398 /* BIO_destroy_bio_pair (BIO_C_DESTROY_BIO_PAIR) */
399 static int dgram_pair_ctrl_destroy_bio_pair(BIO
*bio1
)
402 struct bio_dgram_pair_st
*b1
= bio1
->ptr
, *b2
;
404 ring_buf_destroy(&b1
->rbuf
);
407 /* Early return if we don't have a peer. */
408 if (b1
->peer
== NULL
)
415 if (!ossl_assert(b2
->peer
== bio1
))
419 ring_buf_destroy(&b2
->rbuf
);
427 /* BIO_eof (BIO_CTRL_EOF) */
428 static int dgram_pair_ctrl_eof(BIO
*bio
)
430 struct bio_dgram_pair_st
*b
= bio
->ptr
, *peerb
;
432 if (!ossl_assert(b
!= NULL
))
435 /* If we aren't initialized, we can never read anything */
438 if (!is_dgram_pair(b
))
442 peerb
= b
->peer
->ptr
;
443 if (!ossl_assert(peerb
!= NULL
))
447 * Since we are emulating datagram semantics, never indicate EOF so long as
453 /* BIO_set_write_buf_size (BIO_C_SET_WRITE_BUF_SIZE) */
454 static int dgram_pair_ctrl_set_write_buf_size(BIO
*bio
, size_t len
)
456 struct bio_dgram_pair_st
*b
= bio
->ptr
;
458 /* Changing buffer sizes is not permitted while a peer is connected. */
459 if (b
->peer
!= NULL
) {
460 ERR_raise(ERR_LIB_BIO
, BIO_R_IN_USE
);
464 /* Enforce minimum size. */
465 if (len
< MIN_BUF_LEN
)
468 if (b
->rbuf
.start
!= NULL
) {
469 if (!ring_buf_resize(&b
->rbuf
, len
))
473 b
->req_buf_len
= len
;
474 b
->grows_on_write
= 0;
478 /* BIO_reset (BIO_CTRL_RESET) */
479 static int dgram_pair_ctrl_reset(BIO
*bio
)
481 struct bio_dgram_pair_st
*b
= bio
->ptr
;
483 ring_buf_clear(&b
->rbuf
);
487 /* BIO_pending (BIO_CTRL_PENDING) (Threadsafe) */
488 static size_t dgram_pair_ctrl_pending(BIO
*bio
)
490 size_t saved_idx
, saved_count
;
491 struct bio_dgram_pair_st
*b
= bio
->ptr
, *readb
;
492 struct dgram_hdr hdr
;
495 /* Safe to check; init may not change during this call */
498 if (is_dgram_pair(b
))
499 readb
= b
->peer
->ptr
;
503 if (CRYPTO_THREAD_write_lock(readb
->lock
) == 0)
506 saved_idx
= readb
->rbuf
.idx
[1];
507 saved_count
= readb
->rbuf
.count
;
509 l
= dgram_pair_read_inner(readb
, (uint8_t *)&hdr
, sizeof(hdr
));
511 readb
->rbuf
.idx
[1] = saved_idx
;
512 readb
->rbuf
.count
= saved_count
;
514 CRYPTO_THREAD_unlock(readb
->lock
);
516 if (!ossl_assert(l
== 0 || l
== sizeof(hdr
)))
519 return l
> 0 ? hdr
.len
: 0;
522 /* BIO_get_write_guarantee (BIO_C_GET_WRITE_GUARANTEE) (Threadsafe) */
523 static size_t dgram_pair_ctrl_get_write_guarantee(BIO
*bio
)
526 struct bio_dgram_pair_st
*b
= bio
->ptr
;
528 if (CRYPTO_THREAD_read_lock(b
->lock
) == 0)
531 l
= b
->rbuf
.len
- b
->rbuf
.count
;
532 if (l
>= sizeof(struct dgram_hdr
))
533 l
-= sizeof(struct dgram_hdr
);
536 * If the amount of buffer space would not be enough to accommodate the
537 * worst-case size of a datagram, report no space available.
542 CRYPTO_THREAD_unlock(b
->lock
);
546 /* BIO_dgram_get_local_addr_cap (BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP) */
547 static int dgram_pair_ctrl_get_local_addr_cap(BIO
*bio
)
549 struct bio_dgram_pair_st
*b
= bio
->ptr
, *readb
;
554 if (is_dgram_pair(b
))
555 readb
= b
->peer
->ptr
;
559 return (~readb
->cap
& (BIO_DGRAM_CAP_HANDLES_SRC_ADDR
560 | BIO_DGRAM_CAP_PROVIDES_DST_ADDR
)) == 0;
563 /* BIO_dgram_get_effective_caps (BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS) */
564 static int dgram_pair_ctrl_get_effective_caps(BIO
*bio
)
566 struct bio_dgram_pair_st
*b
= bio
->ptr
, *peerb
;
571 peerb
= b
->peer
->ptr
;
576 /* BIO_dgram_get_caps (BIO_CTRL_DGRAM_GET_CAPS) */
577 static uint32_t dgram_pair_ctrl_get_caps(BIO
*bio
)
579 struct bio_dgram_pair_st
*b
= bio
->ptr
;
584 /* BIO_dgram_set_caps (BIO_CTRL_DGRAM_SET_CAPS) */
585 static int dgram_pair_ctrl_set_caps(BIO
*bio
, uint32_t caps
)
587 struct bio_dgram_pair_st
*b
= bio
->ptr
;
593 /* BIO_dgram_get_local_addr_enable (BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE) */
594 static int dgram_pair_ctrl_get_local_addr_enable(BIO
*bio
)
596 struct bio_dgram_pair_st
*b
= bio
->ptr
;
598 return b
->local_addr_enable
;
601 /* BIO_dgram_set_local_addr_enable (BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE) */
602 static int dgram_pair_ctrl_set_local_addr_enable(BIO
*bio
, int enable
)
604 struct bio_dgram_pair_st
*b
= bio
->ptr
;
606 if (dgram_pair_ctrl_get_local_addr_cap(bio
) == 0)
609 b
->local_addr_enable
= (enable
!= 0 ? 1 : 0);
613 /* BIO_dgram_get_mtu (BIO_CTRL_DGRAM_GET_MTU) */
614 static int dgram_pair_ctrl_get_mtu(BIO
*bio
)
616 struct bio_dgram_pair_st
*b
= bio
->ptr
;
621 /* BIO_dgram_set_mtu (BIO_CTRL_DGRAM_SET_MTU) */
622 static int dgram_pair_ctrl_set_mtu(BIO
*bio
, size_t mtu
)
624 struct bio_dgram_pair_st
*b
= bio
->ptr
, *peerb
;
628 if (b
->peer
!= NULL
) {
629 peerb
= b
->peer
->ptr
;
636 /* Partially threadsafe (some commands) */
637 static long dgram_mem_ctrl(BIO
*bio
, int cmd
, long num
, void *ptr
)
640 struct bio_dgram_pair_st
*b
= bio
->ptr
;
642 if (!ossl_assert(b
!= NULL
))
647 * BIO_set_write_buf_size: Set the size of the ring buffer used for storing
648 * datagrams. No more writes can be performed once the buffer is filled up,
649 * until reads are performed. This cannot be used after a peer is connected.
651 case BIO_C_SET_WRITE_BUF_SIZE
: /* Non-threadsafe */
652 ret
= (long)dgram_pair_ctrl_set_write_buf_size(bio
, (size_t)num
);
656 * BIO_get_write_buf_size: Get ring buffer size.
658 case BIO_C_GET_WRITE_BUF_SIZE
: /* Non-threadsafe */
659 ret
= (long)b
->req_buf_len
;
663 * BIO_reset: Clear all data which was written to this side of the pair.
665 case BIO_CTRL_RESET
: /* Non-threadsafe */
666 dgram_pair_ctrl_reset(bio
);
670 * BIO_get_write_guarantee: Any BIO_write providing a buffer less than or
671 * equal to this value is guaranteed to succeed.
673 case BIO_C_GET_WRITE_GUARANTEE
: /* Threadsafe */
674 ret
= (long)dgram_pair_ctrl_get_write_guarantee(bio
);
677 /* BIO_pending: Bytes available to read. */
678 case BIO_CTRL_PENDING
: /* Threadsafe */
679 ret
= (long)dgram_pair_ctrl_pending(bio
);
682 /* BIO_flush: No-op. */
683 case BIO_CTRL_FLUSH
: /* Threadsafe */
686 /* BIO_dgram_get_no_trunc */
687 case BIO_CTRL_DGRAM_GET_NO_TRUNC
: /* Non-threadsafe */
688 ret
= (long)b
->no_trunc
;
691 /* BIO_dgram_set_no_trunc */
692 case BIO_CTRL_DGRAM_SET_NO_TRUNC
: /* Non-threadsafe */
693 b
->no_trunc
= (num
> 0);
696 /* BIO_dgram_get_local_addr_enable */
697 case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_ENABLE
: /* Non-threadsafe */
698 *(int *)ptr
= (int)dgram_pair_ctrl_get_local_addr_enable(bio
);
701 /* BIO_dgram_set_local_addr_enable */
702 case BIO_CTRL_DGRAM_SET_LOCAL_ADDR_ENABLE
: /* Non-threadsafe */
703 ret
= (long)dgram_pair_ctrl_set_local_addr_enable(bio
, num
);
706 /* BIO_dgram_get_local_addr_cap: Can local addresses be supported? */
707 case BIO_CTRL_DGRAM_GET_LOCAL_ADDR_CAP
: /* Non-threadsafe */
708 ret
= (long)dgram_pair_ctrl_get_local_addr_cap(bio
);
711 /* BIO_dgram_get_effective_caps */
712 case BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS
: /* Non-threadsafe */
713 /* BIO_dgram_get_caps */
714 case BIO_CTRL_DGRAM_GET_CAPS
: /* Non-threadsafe */
715 ret
= (long)dgram_pair_ctrl_get_caps(bio
);
718 /* BIO_dgram_set_caps */
719 case BIO_CTRL_DGRAM_SET_CAPS
: /* Non-threadsafe */
720 ret
= (long)dgram_pair_ctrl_set_caps(bio
, (uint32_t)num
);
723 /* BIO_dgram_get_mtu */
724 case BIO_CTRL_DGRAM_GET_MTU
: /* Non-threadsafe */
725 ret
= (long)dgram_pair_ctrl_get_mtu(bio
);
728 /* BIO_dgram_set_mtu */
729 case BIO_CTRL_DGRAM_SET_MTU
: /* Non-threadsafe */
730 ret
= (long)dgram_pair_ctrl_set_mtu(bio
, (uint32_t)num
);
734 * BIO_eof: Returns whether this half of the BIO pair is empty of data to
737 case BIO_CTRL_EOF
: /* Non-threadsafe */
738 ret
= (long)dgram_pair_ctrl_eof(bio
);
749 static long dgram_pair_ctrl(BIO
*bio
, int cmd
, long num
, void *ptr
)
755 * BIO_make_bio_pair: this is usually used by BIO_new_dgram_pair, though it
756 * may be used manually after manually creating each half of a BIO pair
757 * using BIO_new. This only needs to be called on one of the BIOs.
759 case BIO_C_MAKE_BIO_PAIR
: /* Non-threadsafe */
760 ret
= (long)dgram_pair_ctrl_make_bio_pair(bio
, (BIO
*)ptr
);
764 * BIO_destroy_bio_pair: Manually disconnect two halves of a BIO pair so
765 * that they are no longer peers.
767 case BIO_C_DESTROY_BIO_PAIR
: /* Non-threadsafe */
768 dgram_pair_ctrl_destroy_bio_pair(bio
);
771 /* BIO_dgram_get_effective_caps */
772 case BIO_CTRL_DGRAM_GET_EFFECTIVE_CAPS
: /* Non-threadsafe */
773 ret
= (long)dgram_pair_ctrl_get_effective_caps(bio
);
777 ret
= dgram_mem_ctrl(bio
, cmd
, num
, ptr
);
784 int BIO_new_bio_dgram_pair(BIO
**pbio1
, size_t writebuf1
,
785 BIO
**pbio2
, size_t writebuf2
)
789 BIO
*bio1
= NULL
, *bio2
= NULL
;
791 bio1
= BIO_new(BIO_s_dgram_pair());
795 bio2
= BIO_new(BIO_s_dgram_pair());
800 r
= BIO_set_write_buf_size(bio1
, writebuf1
);
806 r
= BIO_set_write_buf_size(bio2
, writebuf2
);
811 r
= BIO_make_bio_pair(bio1
, bio2
);
829 /* Must hold peer write lock */
830 static size_t dgram_pair_read_inner(struct bio_dgram_pair_st
*b
, uint8_t *buf
, size_t sz
)
832 size_t total_read
= 0;
835 * We repeat pops from the ring buffer for as long as we have more
836 * application *buffer to fill until we fail. We may not be able to pop
837 * enough data to fill the buffer in one operation if the ring buffer wraps
838 * around, but there may still be more data available.
841 uint8_t *src_buf
= NULL
;
845 * There are two BIO instances, each with a ringbuf. We read from the
846 * peer ringbuf and write to our own ringbuf.
848 ring_buf_tail(&b
->rbuf
, &src_buf
, &src_len
);
856 memcpy(buf
, src_buf
, src_len
);
858 ring_buf_pop(&b
->rbuf
, src_len
);
862 total_read
+= src_len
;
870 * Must hold peer write lock. Returns number of bytes processed or negated BIO
873 static ossl_ssize_t
dgram_pair_read_actual(BIO
*bio
, char *buf
, size_t sz
,
874 BIO_ADDR
*local
, BIO_ADDR
*peer
,
877 size_t l
, trunc
= 0, saved_idx
, saved_count
;
878 struct bio_dgram_pair_st
*b
= bio
->ptr
, *readb
;
879 struct dgram_hdr hdr
;
882 BIO_clear_retry_flags(bio
);
885 return -BIO_R_UNINITIALIZED
;
887 if (!ossl_assert(b
!= NULL
))
888 return -BIO_R_TRANSFER_ERROR
;
890 if (is_dgram_pair(b
))
891 readb
= b
->peer
->ptr
;
894 if (!ossl_assert(readb
!= NULL
&& readb
->rbuf
.start
!= NULL
))
895 return -BIO_R_TRANSFER_ERROR
;
897 if (sz
> 0 && buf
== NULL
)
898 return -BIO_R_INVALID_ARGUMENT
;
900 /* If the caller wants to know the local address, it must be enabled */
901 if (local
!= NULL
&& b
->local_addr_enable
== 0)
902 return -BIO_R_LOCAL_ADDR_NOT_AVAILABLE
;
904 /* Read the header. */
905 saved_idx
= readb
->rbuf
.idx
[1];
906 saved_count
= readb
->rbuf
.count
;
907 l
= dgram_pair_read_inner(readb
, (uint8_t *)&hdr
, sizeof(hdr
));
909 /* Buffer was empty. */
911 BIO_set_retry_read(bio
);
912 return -BIO_R_NON_FATAL
;
915 if (!ossl_assert(l
== sizeof(hdr
)))
917 * This should not be possible as headers (and their following payloads)
918 * should always be written atomically.
920 return -BIO_R_BROKEN_PIPE
;
924 } else if (sz
< hdr
.len
) {
925 /* Truncation is occurring. */
926 trunc
= hdr
.len
- sz
;
928 /* Restore original state. */
929 readb
->rbuf
.idx
[1] = saved_idx
;
930 readb
->rbuf
.count
= saved_count
;
931 return -BIO_R_NON_FATAL
;
935 l
= dgram_pair_read_inner(readb
, (uint8_t *)buf
, sz
);
936 if (!ossl_assert(l
== sz
))
937 /* We were somehow not able to read the entire datagram. */
938 return -BIO_R_TRANSFER_ERROR
;
941 * If the datagram was truncated due to an inadequate buffer, discard the
944 if (trunc
> 0 && !ossl_assert(dgram_pair_read_inner(readb
, NULL
, trunc
) == trunc
))
945 /* We were somehow not able to read/skip the entire datagram. */
946 return -BIO_R_TRANSFER_ERROR
;
949 *local
= hdr
.dst_addr
;
951 *peer
= hdr
.src_addr
;
953 return (ossl_ssize_t
)l
;
957 static int dgram_pair_lock_both_write(struct bio_dgram_pair_st
*a
,
958 struct bio_dgram_pair_st
*b
)
960 struct bio_dgram_pair_st
*x
, *y
;
962 x
= (a
->role
== 1) ? a
: b
;
963 y
= (a
->role
== 1) ? b
: a
;
965 if (!ossl_assert(a
->role
!= b
->role
))
968 if (!ossl_assert(a
!= b
&& x
!= y
))
971 if (CRYPTO_THREAD_write_lock(x
->lock
) == 0)
974 if (CRYPTO_THREAD_write_lock(y
->lock
) == 0) {
975 CRYPTO_THREAD_unlock(x
->lock
);
982 static void dgram_pair_unlock_both(struct bio_dgram_pair_st
*a
,
983 struct bio_dgram_pair_st
*b
)
985 CRYPTO_THREAD_unlock(a
->lock
);
986 CRYPTO_THREAD_unlock(b
->lock
);
990 static int dgram_pair_read(BIO
*bio
, char *buf
, int sz_
)
994 struct bio_dgram_pair_st
*b
= bio
->ptr
, *peerb
;
997 ERR_raise(ERR_LIB_BIO
, BIO_R_INVALID_ARGUMENT
);
1001 if (b
->peer
== NULL
) {
1002 ERR_raise(ERR_LIB_BIO
, BIO_R_BROKEN_PIPE
);
1006 peerb
= b
->peer
->ptr
;
1009 * For BIO_read we have to acquire both locks because we touch the retry
1010 * flags on the local bio. (This is avoided in the recvmmsg case as it does
1011 * not touch the retry flags.)
1013 if (dgram_pair_lock_both_write(peerb
, b
) == 0) {
1014 ERR_raise(ERR_LIB_BIO
, ERR_R_UNABLE_TO_GET_WRITE_LOCK
);
1018 l
= dgram_pair_read_actual(bio
, buf
, (size_t)sz_
, NULL
, NULL
, 0);
1020 if (l
!= -BIO_R_NON_FATAL
)
1021 ERR_raise(ERR_LIB_BIO
, -l
);
1027 dgram_pair_unlock_both(peerb
, b
);
1032 static int dgram_pair_recvmmsg(BIO
*bio
, BIO_MSG
*msg
,
1033 size_t stride
, size_t num_msg
,
1035 size_t *num_processed
)
1041 struct bio_dgram_pair_st
*b
= bio
->ptr
, *readb
;
1049 ERR_raise(ERR_LIB_BIO
, BIO_R_BROKEN_PIPE
);
1054 if (is_dgram_pair(b
))
1055 readb
= b
->peer
->ptr
;
1059 if (CRYPTO_THREAD_write_lock(readb
->lock
) == 0) {
1060 ERR_raise(ERR_LIB_BIO
, ERR_R_UNABLE_TO_GET_WRITE_LOCK
);
1065 for (i
= 0; i
< num_msg
; ++i
) {
1066 m
= &BIO_MSG_N(msg
, i
);
1067 l
= dgram_pair_read_actual(bio
, m
->data
, m
->data_len
,
1068 m
->local
, m
->peer
, 1);
1074 ERR_raise(ERR_LIB_BIO
, -l
);
1087 CRYPTO_THREAD_unlock(readb
->lock
);
1092 static int dgram_mem_read(BIO
*bio
, char *buf
, int sz_
)
1096 struct bio_dgram_pair_st
*b
= bio
->ptr
;
1099 ERR_raise(ERR_LIB_BIO
, BIO_R_INVALID_ARGUMENT
);
1103 if (CRYPTO_THREAD_write_lock(b
->lock
) == 0) {
1104 ERR_raise(ERR_LIB_BIO
, ERR_R_UNABLE_TO_GET_WRITE_LOCK
);
1108 l
= dgram_pair_read_actual(bio
, buf
, (size_t)sz_
, NULL
, NULL
, 0);
1110 if (l
!= -BIO_R_NON_FATAL
)
1111 ERR_raise(ERR_LIB_BIO
, -l
);
1117 CRYPTO_THREAD_unlock(b
->lock
);
1122 * Calculate the array growth based on the target size.
1124 * The growth factor is a rational number and is defined by a numerator
1125 * and a denominator. According to Andrew Koenig in his paper "Why Are
1126 * Vectors Efficient?" from JOOP 11(5) 1998, this factor should be less
1127 * than the golden ratio (1.618...).
1129 * We use an expansion factor of 8 / 5 = 1.6
1131 static const size_t max_rbuf_size
= SIZE_MAX
/ 2; /* unlimited in practice */
1132 static ossl_inline
size_t compute_rbuf_growth(size_t target
, size_t current
)
1136 while (current
< target
) {
1137 if (current
>= max_rbuf_size
)
1140 current
= safe_muldiv_size_t(current
, 8, 5, &err
);
1143 if (current
>= max_rbuf_size
)
1144 current
= max_rbuf_size
;
1149 /* Must hold local write lock */
1150 static size_t dgram_pair_write_inner(struct bio_dgram_pair_st
*b
,
1151 const uint8_t *buf
, size_t sz
)
1153 size_t total_written
= 0;
1156 * We repeat pushes to the ring buffer for as long as we have data until we
1157 * fail. We may not be able to push in one operation if the ring buffer
1158 * wraps around, but there may still be more room for data.
1165 * There are two BIO instances, each with a ringbuf. We write to our own
1166 * ringbuf and read from the peer ringbuf.
1168 ring_buf_head(&b
->rbuf
, &dst_buf
, &dst_len
);
1172 if (!b
->grows_on_write
) /* resize only if size not set explicitly */
1174 /* increase the size */
1175 new_len
= compute_rbuf_growth(b
->req_buf_len
+ sz
, b
->req_buf_len
);
1176 if (new_len
== 0 || !ring_buf_resize(&b
->rbuf
, new_len
))
1178 b
->req_buf_len
= new_len
;
1184 memcpy(dst_buf
, buf
, dst_len
);
1185 ring_buf_push(&b
->rbuf
, dst_len
);
1189 total_written
+= dst_len
;
1192 return total_written
;
1196 * Must hold local write lock. Returns number of bytes processed or negated BIO
1199 static ossl_ssize_t
dgram_pair_write_actual(BIO
*bio
, const char *buf
, size_t sz
,
1200 const BIO_ADDR
*local
, const BIO_ADDR
*peer
,
1203 static const BIO_ADDR zero_addr
;
1204 size_t saved_idx
, saved_count
;
1205 struct bio_dgram_pair_st
*b
= bio
->ptr
, *readb
;
1206 struct dgram_hdr hdr
= {0};
1209 BIO_clear_retry_flags(bio
);
1212 return -BIO_R_UNINITIALIZED
;
1214 if (!ossl_assert(b
!= NULL
&& b
->rbuf
.start
!= NULL
))
1215 return -BIO_R_TRANSFER_ERROR
;
1217 if (sz
> 0 && buf
== NULL
)
1218 return -BIO_R_INVALID_ARGUMENT
;
1220 if (local
!= NULL
&& b
->local_addr_enable
== 0)
1221 return -BIO_R_LOCAL_ADDR_NOT_AVAILABLE
;
1223 if (is_dgram_pair(b
))
1224 readb
= b
->peer
->ptr
;
1227 if (peer
!= NULL
&& (readb
->cap
& BIO_DGRAM_CAP_HANDLES_DST_ADDR
) == 0)
1228 return -BIO_R_PEER_ADDR_NOT_AVAILABLE
;
1231 hdr
.dst_addr
= (peer
!= NULL
? *peer
: zero_addr
);
1232 hdr
.src_addr
= (local
!= NULL
? *local
: zero_addr
);
1234 saved_idx
= b
->rbuf
.idx
[0];
1235 saved_count
= b
->rbuf
.count
;
1236 if (dgram_pair_write_inner(b
, (const uint8_t *)&hdr
, sizeof(hdr
)) != sizeof(hdr
)
1237 || dgram_pair_write_inner(b
, (const uint8_t *)buf
, sz
) != sz
) {
1239 * We were not able to push the header and the entirety of the payload
1240 * onto the ring buffer, so abort and roll back the ring buffer state.
1242 b
->rbuf
.idx
[0] = saved_idx
;
1243 b
->rbuf
.count
= saved_count
;
1245 BIO_set_retry_write(bio
);
1246 return -BIO_R_NON_FATAL
;
1253 static int dgram_pair_write(BIO
*bio
, const char *buf
, int sz_
)
1257 struct bio_dgram_pair_st
*b
= bio
->ptr
;
1260 ERR_raise(ERR_LIB_BIO
, BIO_R_INVALID_ARGUMENT
);
1264 if (CRYPTO_THREAD_write_lock(b
->lock
) == 0) {
1265 ERR_raise(ERR_LIB_BIO
, ERR_R_UNABLE_TO_GET_WRITE_LOCK
);
1269 l
= dgram_pair_write_actual(bio
, buf
, (size_t)sz_
, NULL
, NULL
, 0);
1271 ERR_raise(ERR_LIB_BIO
, -l
);
1277 CRYPTO_THREAD_unlock(b
->lock
);
1282 static int dgram_pair_sendmmsg(BIO
*bio
, BIO_MSG
*msg
,
1283 size_t stride
, size_t num_msg
,
1284 uint64_t flags
, size_t *num_processed
)
1286 ossl_ssize_t ret
, l
;
1289 struct bio_dgram_pair_st
*b
= bio
->ptr
;
1296 if (CRYPTO_THREAD_write_lock(b
->lock
) == 0) {
1297 ERR_raise(ERR_LIB_BIO
, ERR_R_UNABLE_TO_GET_WRITE_LOCK
);
1302 for (i
= 0; i
< num_msg
; ++i
) {
1303 m
= &BIO_MSG_N(msg
, i
);
1304 l
= dgram_pair_write_actual(bio
, m
->data
, m
->data_len
,
1305 m
->local
, m
->peer
, 1);
1311 ERR_raise(ERR_LIB_BIO
, -l
);
1323 CRYPTO_THREAD_unlock(b
->lock
);