1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/slab.h>
7 #include <linux/compat.h>
8 #include <net/compat.h>
9 #include <linux/io_uring.h>
11 #include <uapi/linux/io_uring.h>
15 #include "alloc_cache.h"
20 #if defined(CONFIG_NET)
28 struct sockaddr __user
*addr
;
47 struct sockaddr __user
*addr
;
50 bool seen_econnaborted
;
56 struct compat_msghdr __user
*umsg_compat
;
57 struct user_msghdr __user
*umsg
;
63 unsigned nr_multishot_loops
;
65 /* initialised and used only by !msg send variants */
69 void __user
*msg_control
;
70 /* used only for send zerocopy */
71 struct io_kiocb
*notif
;
75 * Number of times we'll try and do receives if there's more data. If we
76 * exceed this limit, then add us to the back of the queue and retry from
77 * there. This helps fairness between flooding clients.
79 #define MULTISHOT_MAX_RETRY 32
81 static inline bool io_check_multishot(struct io_kiocb
*req
,
82 unsigned int issue_flags
)
85 * When ->locked_cq is set we only allow to post CQEs from the original
86 * task context. Usual request completions will be handled in other
87 * generic paths but multipoll may decide to post extra cqes.
89 return !(issue_flags
& IO_URING_F_IOWQ
) ||
90 !(issue_flags
& IO_URING_F_MULTISHOT
) ||
91 !req
->ctx
->task_complete
;
94 int io_shutdown_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
96 struct io_shutdown
*shutdown
= io_kiocb_to_cmd(req
, struct io_shutdown
);
98 if (unlikely(sqe
->off
|| sqe
->addr
|| sqe
->rw_flags
||
99 sqe
->buf_index
|| sqe
->splice_fd_in
))
102 shutdown
->how
= READ_ONCE(sqe
->len
);
103 req
->flags
|= REQ_F_FORCE_ASYNC
;
107 int io_shutdown(struct io_kiocb
*req
, unsigned int issue_flags
)
109 struct io_shutdown
*shutdown
= io_kiocb_to_cmd(req
, struct io_shutdown
);
113 WARN_ON_ONCE(issue_flags
& IO_URING_F_NONBLOCK
);
115 sock
= sock_from_file(req
->file
);
119 ret
= __sys_shutdown_sock(sock
, shutdown
->how
);
120 io_req_set_res(req
, ret
, 0);
124 static bool io_net_retry(struct socket
*sock
, int flags
)
126 if (!(flags
& MSG_WAITALL
))
128 return sock
->type
== SOCK_STREAM
|| sock
->type
== SOCK_SEQPACKET
;
131 static void io_netmsg_recycle(struct io_kiocb
*req
, unsigned int issue_flags
)
133 struct io_async_msghdr
*hdr
= req
->async_data
;
135 if (!req_has_async_data(req
) || issue_flags
& IO_URING_F_UNLOCKED
)
138 /* Let normal cleanup path reap it if we fail adding to the cache */
139 if (io_alloc_cache_put(&req
->ctx
->netmsg_cache
, &hdr
->cache
)) {
140 req
->async_data
= NULL
;
141 req
->flags
&= ~REQ_F_ASYNC_DATA
;
145 static struct io_async_msghdr
*io_msg_alloc_async(struct io_kiocb
*req
,
146 unsigned int issue_flags
)
148 struct io_ring_ctx
*ctx
= req
->ctx
;
149 struct io_cache_entry
*entry
;
150 struct io_async_msghdr
*hdr
;
152 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
153 entry
= io_alloc_cache_get(&ctx
->netmsg_cache
);
155 hdr
= container_of(entry
, struct io_async_msghdr
, cache
);
156 hdr
->free_iov
= NULL
;
157 req
->flags
|= REQ_F_ASYNC_DATA
;
158 req
->async_data
= hdr
;
163 if (!io_alloc_async_data(req
)) {
164 hdr
= req
->async_data
;
165 hdr
->free_iov
= NULL
;
171 static inline struct io_async_msghdr
*io_msg_alloc_async_prep(struct io_kiocb
*req
)
173 /* ->prep_async is always called from the submission context */
174 return io_msg_alloc_async(req
, 0);
177 static int io_setup_async_msg(struct io_kiocb
*req
,
178 struct io_async_msghdr
*kmsg
,
179 unsigned int issue_flags
)
181 struct io_async_msghdr
*async_msg
;
183 if (req_has_async_data(req
))
185 async_msg
= io_msg_alloc_async(req
, issue_flags
);
187 kfree(kmsg
->free_iov
);
190 req
->flags
|= REQ_F_NEED_CLEANUP
;
191 memcpy(async_msg
, kmsg
, sizeof(*kmsg
));
192 if (async_msg
->msg
.msg_name
)
193 async_msg
->msg
.msg_name
= &async_msg
->addr
;
195 if ((req
->flags
& REQ_F_BUFFER_SELECT
) && !async_msg
->msg
.msg_iter
.nr_segs
)
198 /* if were using fast_iov, set it to the new one */
199 if (iter_is_iovec(&kmsg
->msg
.msg_iter
) && !kmsg
->free_iov
) {
200 size_t fast_idx
= iter_iov(&kmsg
->msg
.msg_iter
) - kmsg
->fast_iov
;
201 async_msg
->msg
.msg_iter
.__iov
= &async_msg
->fast_iov
[fast_idx
];
207 static int io_sendmsg_copy_hdr(struct io_kiocb
*req
,
208 struct io_async_msghdr
*iomsg
)
210 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
213 iomsg
->msg
.msg_name
= &iomsg
->addr
;
214 iomsg
->free_iov
= iomsg
->fast_iov
;
215 ret
= sendmsg_copy_msghdr(&iomsg
->msg
, sr
->umsg
, sr
->msg_flags
,
217 /* save msg_control as sys_sendmsg() overwrites it */
218 sr
->msg_control
= iomsg
->msg
.msg_control_user
;
222 int io_send_prep_async(struct io_kiocb
*req
)
224 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
225 struct io_async_msghdr
*io
;
228 if (!zc
->addr
|| req_has_async_data(req
))
230 io
= io_msg_alloc_async_prep(req
);
233 ret
= move_addr_to_kernel(zc
->addr
, zc
->addr_len
, &io
->addr
);
237 static int io_setup_async_addr(struct io_kiocb
*req
,
238 struct sockaddr_storage
*addr_storage
,
239 unsigned int issue_flags
)
241 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
242 struct io_async_msghdr
*io
;
244 if (!sr
->addr
|| req_has_async_data(req
))
246 io
= io_msg_alloc_async(req
, issue_flags
);
249 memcpy(&io
->addr
, addr_storage
, sizeof(io
->addr
));
253 int io_sendmsg_prep_async(struct io_kiocb
*req
)
257 if (!io_msg_alloc_async_prep(req
))
259 ret
= io_sendmsg_copy_hdr(req
, req
->async_data
);
261 req
->flags
|= REQ_F_NEED_CLEANUP
;
265 void io_sendmsg_recvmsg_cleanup(struct io_kiocb
*req
)
267 struct io_async_msghdr
*io
= req
->async_data
;
272 int io_sendmsg_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
274 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
276 if (req
->opcode
== IORING_OP_SEND
) {
277 if (READ_ONCE(sqe
->__pad3
[0]))
279 sr
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
280 sr
->addr_len
= READ_ONCE(sqe
->addr_len
);
281 } else if (sqe
->addr2
|| sqe
->file_index
) {
285 sr
->umsg
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
286 sr
->len
= READ_ONCE(sqe
->len
);
287 sr
->flags
= READ_ONCE(sqe
->ioprio
);
288 if (sr
->flags
& ~IORING_RECVSEND_POLL_FIRST
)
290 sr
->msg_flags
= READ_ONCE(sqe
->msg_flags
) | MSG_NOSIGNAL
;
291 if (sr
->msg_flags
& MSG_DONTWAIT
)
292 req
->flags
|= REQ_F_NOWAIT
;
295 if (req
->ctx
->compat
)
296 sr
->msg_flags
|= MSG_CMSG_COMPAT
;
302 int io_sendmsg(struct io_kiocb
*req
, unsigned int issue_flags
)
304 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
305 struct io_async_msghdr iomsg
, *kmsg
;
311 sock
= sock_from_file(req
->file
);
315 if (req_has_async_data(req
)) {
316 kmsg
= req
->async_data
;
317 kmsg
->msg
.msg_control_user
= sr
->msg_control
;
319 ret
= io_sendmsg_copy_hdr(req
, &iomsg
);
325 if (!(req
->flags
& REQ_F_POLLED
) &&
326 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
327 return io_setup_async_msg(req
, kmsg
, issue_flags
);
329 flags
= sr
->msg_flags
;
330 if (issue_flags
& IO_URING_F_NONBLOCK
)
331 flags
|= MSG_DONTWAIT
;
332 if (flags
& MSG_WAITALL
)
333 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
335 ret
= __sys_sendmsg_sock(sock
, &kmsg
->msg
, flags
);
338 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
339 return io_setup_async_msg(req
, kmsg
, issue_flags
);
340 if (ret
> 0 && io_net_retry(sock
, flags
)) {
341 kmsg
->msg
.msg_controllen
= 0;
342 kmsg
->msg
.msg_control
= NULL
;
344 req
->flags
|= REQ_F_PARTIAL_IO
;
345 return io_setup_async_msg(req
, kmsg
, issue_flags
);
347 if (ret
== -ERESTARTSYS
)
351 /* fast path, check for non-NULL to avoid function call */
353 kfree(kmsg
->free_iov
);
354 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
355 io_netmsg_recycle(req
, issue_flags
);
358 else if (sr
->done_io
)
360 io_req_set_res(req
, ret
, 0);
364 int io_send(struct io_kiocb
*req
, unsigned int issue_flags
)
366 struct sockaddr_storage __address
;
367 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
375 msg
.msg_control
= NULL
;
376 msg
.msg_controllen
= 0;
381 if (req_has_async_data(req
)) {
382 struct io_async_msghdr
*io
= req
->async_data
;
384 msg
.msg_name
= &io
->addr
;
386 ret
= move_addr_to_kernel(sr
->addr
, sr
->addr_len
, &__address
);
387 if (unlikely(ret
< 0))
389 msg
.msg_name
= (struct sockaddr
*)&__address
;
391 msg
.msg_namelen
= sr
->addr_len
;
394 if (!(req
->flags
& REQ_F_POLLED
) &&
395 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
396 return io_setup_async_addr(req
, &__address
, issue_flags
);
398 sock
= sock_from_file(req
->file
);
402 ret
= import_ubuf(ITER_SOURCE
, sr
->buf
, sr
->len
, &msg
.msg_iter
);
406 flags
= sr
->msg_flags
;
407 if (issue_flags
& IO_URING_F_NONBLOCK
)
408 flags
|= MSG_DONTWAIT
;
409 if (flags
& MSG_WAITALL
)
410 min_ret
= iov_iter_count(&msg
.msg_iter
);
412 flags
&= ~MSG_INTERNAL_SENDMSG_FLAGS
;
413 msg
.msg_flags
= flags
;
414 ret
= sock_sendmsg(sock
, &msg
);
416 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
417 return io_setup_async_addr(req
, &__address
, issue_flags
);
419 if (ret
> 0 && io_net_retry(sock
, flags
)) {
423 req
->flags
|= REQ_F_PARTIAL_IO
;
424 return io_setup_async_addr(req
, &__address
, issue_flags
);
426 if (ret
== -ERESTARTSYS
)
432 else if (sr
->done_io
)
434 io_req_set_res(req
, ret
, 0);
438 static bool io_recvmsg_multishot_overflow(struct io_async_msghdr
*iomsg
)
442 if (iomsg
->namelen
< 0)
444 if (check_add_overflow((int)sizeof(struct io_uring_recvmsg_out
),
445 iomsg
->namelen
, &hdr
))
447 if (check_add_overflow(hdr
, (int)iomsg
->controllen
, &hdr
))
453 static int __io_recvmsg_copy_hdr(struct io_kiocb
*req
,
454 struct io_async_msghdr
*iomsg
)
456 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
457 struct user_msghdr msg
;
460 if (copy_from_user(&msg
, sr
->umsg
, sizeof(*sr
->umsg
)))
463 ret
= __copy_msghdr(&iomsg
->msg
, &msg
, &iomsg
->uaddr
);
467 if (req
->flags
& REQ_F_BUFFER_SELECT
) {
468 if (msg
.msg_iovlen
== 0) {
469 sr
->len
= iomsg
->fast_iov
[0].iov_len
= 0;
470 iomsg
->fast_iov
[0].iov_base
= NULL
;
471 iomsg
->free_iov
= NULL
;
472 } else if (msg
.msg_iovlen
> 1) {
475 if (copy_from_user(iomsg
->fast_iov
, msg
.msg_iov
, sizeof(*msg
.msg_iov
)))
477 sr
->len
= iomsg
->fast_iov
[0].iov_len
;
478 iomsg
->free_iov
= NULL
;
481 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
482 iomsg
->namelen
= msg
.msg_namelen
;
483 iomsg
->controllen
= msg
.msg_controllen
;
484 if (io_recvmsg_multishot_overflow(iomsg
))
488 iomsg
->free_iov
= iomsg
->fast_iov
;
489 ret
= __import_iovec(ITER_DEST
, msg
.msg_iov
, msg
.msg_iovlen
, UIO_FASTIOV
,
490 &iomsg
->free_iov
, &iomsg
->msg
.msg_iter
,
500 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb
*req
,
501 struct io_async_msghdr
*iomsg
)
503 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
504 struct compat_msghdr msg
;
505 struct compat_iovec __user
*uiov
;
508 if (copy_from_user(&msg
, sr
->umsg_compat
, sizeof(msg
)))
511 ret
= __get_compat_msghdr(&iomsg
->msg
, &msg
, &iomsg
->uaddr
);
515 uiov
= compat_ptr(msg
.msg_iov
);
516 if (req
->flags
& REQ_F_BUFFER_SELECT
) {
519 iomsg
->free_iov
= NULL
;
520 if (msg
.msg_iovlen
== 0) {
522 } else if (msg
.msg_iovlen
> 1) {
525 if (!access_ok(uiov
, sizeof(*uiov
)))
527 if (__get_user(clen
, &uiov
->iov_len
))
534 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
535 iomsg
->namelen
= msg
.msg_namelen
;
536 iomsg
->controllen
= msg
.msg_controllen
;
537 if (io_recvmsg_multishot_overflow(iomsg
))
541 iomsg
->free_iov
= iomsg
->fast_iov
;
542 ret
= __import_iovec(ITER_DEST
, (struct iovec __user
*)uiov
, msg
.msg_iovlen
,
543 UIO_FASTIOV
, &iomsg
->free_iov
,
544 &iomsg
->msg
.msg_iter
, true);
553 static int io_recvmsg_copy_hdr(struct io_kiocb
*req
,
554 struct io_async_msghdr
*iomsg
)
556 iomsg
->msg
.msg_name
= &iomsg
->addr
;
557 iomsg
->msg
.msg_iter
.nr_segs
= 0;
560 if (req
->ctx
->compat
)
561 return __io_compat_recvmsg_copy_hdr(req
, iomsg
);
564 return __io_recvmsg_copy_hdr(req
, iomsg
);
567 int io_recvmsg_prep_async(struct io_kiocb
*req
)
571 if (!io_msg_alloc_async_prep(req
))
573 ret
= io_recvmsg_copy_hdr(req
, req
->async_data
);
575 req
->flags
|= REQ_F_NEED_CLEANUP
;
579 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)
581 int io_recvmsg_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
583 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
585 if (unlikely(sqe
->file_index
|| sqe
->addr2
))
588 sr
->umsg
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
589 sr
->len
= READ_ONCE(sqe
->len
);
590 sr
->flags
= READ_ONCE(sqe
->ioprio
);
591 if (sr
->flags
& ~(RECVMSG_FLAGS
))
593 sr
->msg_flags
= READ_ONCE(sqe
->msg_flags
);
594 if (sr
->msg_flags
& MSG_DONTWAIT
)
595 req
->flags
|= REQ_F_NOWAIT
;
596 if (sr
->msg_flags
& MSG_ERRQUEUE
)
597 req
->flags
|= REQ_F_CLEAR_POLLIN
;
598 if (sr
->flags
& IORING_RECV_MULTISHOT
) {
599 if (!(req
->flags
& REQ_F_BUFFER_SELECT
))
601 if (sr
->msg_flags
& MSG_WAITALL
)
603 if (req
->opcode
== IORING_OP_RECV
&& sr
->len
)
605 req
->flags
|= REQ_F_APOLL_MULTISHOT
;
607 * Store the buffer group for this multishot receive separately,
608 * as if we end up doing an io-wq based issue that selects a
609 * buffer, it has to be committed immediately and that will
610 * clear ->buf_list. This means we lose the link to the buffer
611 * list, and the eventual buffer put on completion then cannot
614 sr
->buf_group
= req
->buf_index
;
618 if (req
->ctx
->compat
)
619 sr
->msg_flags
|= MSG_CMSG_COMPAT
;
622 sr
->nr_multishot_loops
= 0;
626 static inline void io_recv_prep_retry(struct io_kiocb
*req
)
628 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
631 sr
->len
= 0; /* get from the provided buffer */
632 req
->buf_index
= sr
->buf_group
;
636 * Finishes io_recv and io_recvmsg.
638 * Returns true if it is actually finished, or false if it should run
639 * again (for multishot).
641 static inline bool io_recv_finish(struct io_kiocb
*req
, int *ret
,
642 struct msghdr
*msg
, bool mshot_finished
,
643 unsigned issue_flags
)
647 cflags
= io_put_kbuf(req
, issue_flags
);
648 if (msg
->msg_inq
&& msg
->msg_inq
!= -1)
649 cflags
|= IORING_CQE_F_SOCK_NONEMPTY
;
651 if (!(req
->flags
& REQ_F_APOLL_MULTISHOT
)) {
652 io_req_set_res(req
, *ret
, cflags
);
661 * Fill CQE for this receive and see if we should keep trying to
662 * receive from this socket.
664 if (io_fill_cqe_req_aux(req
, issue_flags
& IO_URING_F_COMPLETE_DEFER
,
665 *ret
, cflags
| IORING_CQE_F_MORE
)) {
666 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
667 int mshot_retry_ret
= IOU_ISSUE_SKIP_COMPLETE
;
669 io_recv_prep_retry(req
);
670 /* Known not-empty or unknown state, retry */
671 if (cflags
& IORING_CQE_F_SOCK_NONEMPTY
|| msg
->msg_inq
== -1) {
672 if (sr
->nr_multishot_loops
++ < MULTISHOT_MAX_RETRY
)
674 /* mshot retries exceeded, force a requeue */
675 sr
->nr_multishot_loops
= 0;
676 mshot_retry_ret
= IOU_REQUEUE
;
678 if (issue_flags
& IO_URING_F_MULTISHOT
)
679 *ret
= mshot_retry_ret
;
684 /* Otherwise stop multishot but use the current result. */
686 io_req_set_res(req
, *ret
, cflags
);
688 if (issue_flags
& IO_URING_F_MULTISHOT
)
689 *ret
= IOU_STOP_MULTISHOT
;
695 static int io_recvmsg_prep_multishot(struct io_async_msghdr
*kmsg
,
696 struct io_sr_msg
*sr
, void __user
**buf
,
699 unsigned long ubuf
= (unsigned long) *buf
;
702 hdr
= sizeof(struct io_uring_recvmsg_out
) + kmsg
->namelen
+
707 if (kmsg
->controllen
) {
708 unsigned long control
= ubuf
+ hdr
- kmsg
->controllen
;
710 kmsg
->msg
.msg_control_user
= (void __user
*) control
;
711 kmsg
->msg
.msg_controllen
= kmsg
->controllen
;
714 sr
->buf
= *buf
; /* stash for later copy */
715 *buf
= (void __user
*) (ubuf
+ hdr
);
716 kmsg
->payloadlen
= *len
= *len
- hdr
;
720 struct io_recvmsg_multishot_hdr
{
721 struct io_uring_recvmsg_out msg
;
722 struct sockaddr_storage addr
;
725 static int io_recvmsg_multishot(struct socket
*sock
, struct io_sr_msg
*io
,
726 struct io_async_msghdr
*kmsg
,
727 unsigned int flags
, bool *finished
)
731 struct io_recvmsg_multishot_hdr hdr
;
734 kmsg
->msg
.msg_name
= &hdr
.addr
;
735 kmsg
->msg
.msg_flags
= flags
& (MSG_CMSG_CLOEXEC
|MSG_CMSG_COMPAT
);
736 kmsg
->msg
.msg_namelen
= 0;
738 if (sock
->file
->f_flags
& O_NONBLOCK
)
739 flags
|= MSG_DONTWAIT
;
741 err
= sock_recvmsg(sock
, &kmsg
->msg
, flags
);
742 *finished
= err
<= 0;
746 hdr
.msg
= (struct io_uring_recvmsg_out
) {
747 .controllen
= kmsg
->controllen
- kmsg
->msg
.msg_controllen
,
748 .flags
= kmsg
->msg
.msg_flags
& ~MSG_CMSG_COMPAT
751 hdr
.msg
.payloadlen
= err
;
752 if (err
> kmsg
->payloadlen
)
753 err
= kmsg
->payloadlen
;
755 copy_len
= sizeof(struct io_uring_recvmsg_out
);
756 if (kmsg
->msg
.msg_namelen
> kmsg
->namelen
)
757 copy_len
+= kmsg
->namelen
;
759 copy_len
+= kmsg
->msg
.msg_namelen
;
762 * "fromlen shall refer to the value before truncation.."
765 hdr
.msg
.namelen
= kmsg
->msg
.msg_namelen
;
767 /* ensure that there is no gap between hdr and sockaddr_storage */
768 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr
, addr
) !=
769 sizeof(struct io_uring_recvmsg_out
));
770 if (copy_to_user(io
->buf
, &hdr
, copy_len
)) {
775 return sizeof(struct io_uring_recvmsg_out
) + kmsg
->namelen
+
776 kmsg
->controllen
+ err
;
779 int io_recvmsg(struct io_kiocb
*req
, unsigned int issue_flags
)
781 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
782 struct io_async_msghdr iomsg
, *kmsg
;
785 int ret
, min_ret
= 0;
786 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
787 bool mshot_finished
= true;
789 sock
= sock_from_file(req
->file
);
793 if (req_has_async_data(req
)) {
794 kmsg
= req
->async_data
;
796 ret
= io_recvmsg_copy_hdr(req
, &iomsg
);
802 if (!(req
->flags
& REQ_F_POLLED
) &&
803 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
804 return io_setup_async_msg(req
, kmsg
, issue_flags
);
806 if (!io_check_multishot(req
, issue_flags
))
807 return io_setup_async_msg(req
, kmsg
, issue_flags
);
810 if (io_do_buffer_select(req
)) {
812 size_t len
= sr
->len
;
814 buf
= io_buffer_select(req
, &len
, issue_flags
);
818 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
819 ret
= io_recvmsg_prep_multishot(kmsg
, sr
, &buf
, &len
);
821 io_kbuf_recycle(req
, issue_flags
);
826 iov_iter_ubuf(&kmsg
->msg
.msg_iter
, ITER_DEST
, buf
, len
);
829 flags
= sr
->msg_flags
;
831 flags
|= MSG_DONTWAIT
;
833 kmsg
->msg
.msg_get_inq
= 1;
834 kmsg
->msg
.msg_inq
= -1;
835 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
836 ret
= io_recvmsg_multishot(sock
, sr
, kmsg
, flags
,
839 /* disable partial retry for recvmsg with cmsg attached */
840 if (flags
& MSG_WAITALL
&& !kmsg
->msg
.msg_controllen
)
841 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
843 ret
= __sys_recvmsg_sock(sock
, &kmsg
->msg
, sr
->umsg
,
848 if (ret
== -EAGAIN
&& force_nonblock
) {
849 ret
= io_setup_async_msg(req
, kmsg
, issue_flags
);
850 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_MULTISHOT
)) {
851 io_kbuf_recycle(req
, issue_flags
);
852 return IOU_ISSUE_SKIP_COMPLETE
;
856 if (ret
> 0 && io_net_retry(sock
, flags
)) {
858 req
->flags
|= REQ_F_PARTIAL_IO
;
859 return io_setup_async_msg(req
, kmsg
, issue_flags
);
861 if (ret
== -ERESTARTSYS
)
864 } else if ((flags
& MSG_WAITALL
) && (kmsg
->msg
.msg_flags
& (MSG_TRUNC
| MSG_CTRUNC
))) {
870 else if (sr
->done_io
)
873 io_kbuf_recycle(req
, issue_flags
);
875 if (!io_recv_finish(req
, &ret
, &kmsg
->msg
, mshot_finished
, issue_flags
))
876 goto retry_multishot
;
878 if (mshot_finished
) {
879 /* fast path, check for non-NULL to avoid function call */
881 kfree(kmsg
->free_iov
);
882 io_netmsg_recycle(req
, issue_flags
);
883 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
889 int io_recv(struct io_kiocb
*req
, unsigned int issue_flags
)
891 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
895 int ret
, min_ret
= 0;
896 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
897 size_t len
= sr
->len
;
899 if (!(req
->flags
& REQ_F_POLLED
) &&
900 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
903 if (!io_check_multishot(req
, issue_flags
))
906 sock
= sock_from_file(req
->file
);
912 msg
.msg_control
= NULL
;
914 msg
.msg_controllen
= 0;
919 if (io_do_buffer_select(req
)) {
922 buf
= io_buffer_select(req
, &len
, issue_flags
);
929 ret
= import_ubuf(ITER_DEST
, sr
->buf
, len
, &msg
.msg_iter
);
936 flags
= sr
->msg_flags
;
938 flags
|= MSG_DONTWAIT
;
939 if (flags
& MSG_WAITALL
)
940 min_ret
= iov_iter_count(&msg
.msg_iter
);
942 ret
= sock_recvmsg(sock
, &msg
, flags
);
944 if (ret
== -EAGAIN
&& force_nonblock
) {
945 if (issue_flags
& IO_URING_F_MULTISHOT
) {
946 io_kbuf_recycle(req
, issue_flags
);
947 return IOU_ISSUE_SKIP_COMPLETE
;
952 if (ret
> 0 && io_net_retry(sock
, flags
)) {
956 req
->flags
|= REQ_F_PARTIAL_IO
;
959 if (ret
== -ERESTARTSYS
)
962 } else if ((flags
& MSG_WAITALL
) && (msg
.msg_flags
& (MSG_TRUNC
| MSG_CTRUNC
))) {
969 else if (sr
->done_io
)
972 io_kbuf_recycle(req
, issue_flags
);
974 if (!io_recv_finish(req
, &ret
, &msg
, ret
<= 0, issue_flags
))
975 goto retry_multishot
;
980 void io_send_zc_cleanup(struct io_kiocb
*req
)
982 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
983 struct io_async_msghdr
*io
;
985 if (req_has_async_data(req
)) {
986 io
= req
->async_data
;
987 /* might be ->fast_iov if *msg_copy_hdr failed */
988 if (io
->free_iov
!= io
->fast_iov
)
992 io_notif_flush(zc
->notif
);
997 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
998 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE)
1000 int io_send_zc_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1002 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1003 struct io_ring_ctx
*ctx
= req
->ctx
;
1004 struct io_kiocb
*notif
;
1006 if (unlikely(READ_ONCE(sqe
->__pad2
[0]) || READ_ONCE(sqe
->addr3
)))
1008 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1009 if (req
->flags
& REQ_F_CQE_SKIP
)
1012 notif
= zc
->notif
= io_alloc_notif(ctx
);
1015 notif
->cqe
.user_data
= req
->cqe
.user_data
;
1017 notif
->cqe
.flags
= IORING_CQE_F_NOTIF
;
1018 req
->flags
|= REQ_F_NEED_CLEANUP
;
1020 zc
->flags
= READ_ONCE(sqe
->ioprio
);
1021 if (unlikely(zc
->flags
& ~IO_ZC_FLAGS_COMMON
)) {
1022 if (zc
->flags
& ~IO_ZC_FLAGS_VALID
)
1024 if (zc
->flags
& IORING_SEND_ZC_REPORT_USAGE
) {
1025 io_notif_set_extended(notif
);
1026 io_notif_to_data(notif
)->zc_report
= true;
1030 if (zc
->flags
& IORING_RECVSEND_FIXED_BUF
) {
1031 unsigned idx
= READ_ONCE(sqe
->buf_index
);
1033 if (unlikely(idx
>= ctx
->nr_user_bufs
))
1035 idx
= array_index_nospec(idx
, ctx
->nr_user_bufs
);
1036 req
->imu
= READ_ONCE(ctx
->user_bufs
[idx
]);
1037 io_req_set_rsrc_node(notif
, ctx
, 0);
1040 if (req
->opcode
== IORING_OP_SEND_ZC
) {
1041 if (READ_ONCE(sqe
->__pad3
[0]))
1043 zc
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
1044 zc
->addr_len
= READ_ONCE(sqe
->addr_len
);
1046 if (unlikely(sqe
->addr2
|| sqe
->file_index
))
1048 if (unlikely(zc
->flags
& IORING_RECVSEND_FIXED_BUF
))
1052 zc
->buf
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1053 zc
->len
= READ_ONCE(sqe
->len
);
1054 zc
->msg_flags
= READ_ONCE(sqe
->msg_flags
) | MSG_NOSIGNAL
;
1055 if (zc
->msg_flags
& MSG_DONTWAIT
)
1056 req
->flags
|= REQ_F_NOWAIT
;
1060 #ifdef CONFIG_COMPAT
1061 if (req
->ctx
->compat
)
1062 zc
->msg_flags
|= MSG_CMSG_COMPAT
;
1067 static int io_sg_from_iter_iovec(struct sock
*sk
, struct sk_buff
*skb
,
1068 struct iov_iter
*from
, size_t length
)
1070 skb_zcopy_downgrade_managed(skb
);
1071 return __zerocopy_sg_from_iter(NULL
, sk
, skb
, from
, length
);
1074 static int io_sg_from_iter(struct sock
*sk
, struct sk_buff
*skb
,
1075 struct iov_iter
*from
, size_t length
)
1077 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1078 int frag
= shinfo
->nr_frags
;
1080 struct bvec_iter bi
;
1082 unsigned long truesize
= 0;
1085 shinfo
->flags
|= SKBFL_MANAGED_FRAG_REFS
;
1086 else if (unlikely(!skb_zcopy_managed(skb
)))
1087 return __zerocopy_sg_from_iter(NULL
, sk
, skb
, from
, length
);
1089 bi
.bi_size
= min(from
->count
, length
);
1090 bi
.bi_bvec_done
= from
->iov_offset
;
1093 while (bi
.bi_size
&& frag
< MAX_SKB_FRAGS
) {
1094 struct bio_vec v
= mp_bvec_iter_bvec(from
->bvec
, bi
);
1097 truesize
+= PAGE_ALIGN(v
.bv_len
+ v
.bv_offset
);
1098 __skb_fill_page_desc_noacc(shinfo
, frag
++, v
.bv_page
,
1099 v
.bv_offset
, v
.bv_len
);
1100 bvec_iter_advance_single(from
->bvec
, &bi
, v
.bv_len
);
1105 shinfo
->nr_frags
= frag
;
1106 from
->bvec
+= bi
.bi_idx
;
1107 from
->nr_segs
-= bi
.bi_idx
;
1108 from
->count
-= copied
;
1109 from
->iov_offset
= bi
.bi_bvec_done
;
1111 skb
->data_len
+= copied
;
1113 skb
->truesize
+= truesize
;
1115 if (sk
&& sk
->sk_type
== SOCK_STREAM
) {
1116 sk_wmem_queued_add(sk
, truesize
);
1117 if (!skb_zcopy_pure(skb
))
1118 sk_mem_charge(sk
, truesize
);
1120 refcount_add(truesize
, &skb
->sk
->sk_wmem_alloc
);
1125 int io_send_zc(struct io_kiocb
*req
, unsigned int issue_flags
)
1127 struct sockaddr_storage __address
;
1128 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1130 struct socket
*sock
;
1132 int ret
, min_ret
= 0;
1134 sock
= sock_from_file(req
->file
);
1135 if (unlikely(!sock
))
1137 if (!test_bit(SOCK_SUPPORT_ZC
, &sock
->flags
))
1140 msg
.msg_name
= NULL
;
1141 msg
.msg_control
= NULL
;
1142 msg
.msg_controllen
= 0;
1143 msg
.msg_namelen
= 0;
1146 if (req_has_async_data(req
)) {
1147 struct io_async_msghdr
*io
= req
->async_data
;
1149 msg
.msg_name
= &io
->addr
;
1151 ret
= move_addr_to_kernel(zc
->addr
, zc
->addr_len
, &__address
);
1152 if (unlikely(ret
< 0))
1154 msg
.msg_name
= (struct sockaddr
*)&__address
;
1156 msg
.msg_namelen
= zc
->addr_len
;
1159 if (!(req
->flags
& REQ_F_POLLED
) &&
1160 (zc
->flags
& IORING_RECVSEND_POLL_FIRST
))
1161 return io_setup_async_addr(req
, &__address
, issue_flags
);
1163 if (zc
->flags
& IORING_RECVSEND_FIXED_BUF
) {
1164 ret
= io_import_fixed(ITER_SOURCE
, &msg
.msg_iter
, req
->imu
,
1165 (u64
)(uintptr_t)zc
->buf
, zc
->len
);
1168 msg
.sg_from_iter
= io_sg_from_iter
;
1170 io_notif_set_extended(zc
->notif
);
1171 ret
= import_ubuf(ITER_SOURCE
, zc
->buf
, zc
->len
, &msg
.msg_iter
);
1174 ret
= io_notif_account_mem(zc
->notif
, zc
->len
);
1177 msg
.sg_from_iter
= io_sg_from_iter_iovec
;
1180 msg_flags
= zc
->msg_flags
| MSG_ZEROCOPY
;
1181 if (issue_flags
& IO_URING_F_NONBLOCK
)
1182 msg_flags
|= MSG_DONTWAIT
;
1183 if (msg_flags
& MSG_WAITALL
)
1184 min_ret
= iov_iter_count(&msg
.msg_iter
);
1185 msg_flags
&= ~MSG_INTERNAL_SENDMSG_FLAGS
;
1187 msg
.msg_flags
= msg_flags
;
1188 msg
.msg_ubuf
= &io_notif_to_data(zc
->notif
)->uarg
;
1189 ret
= sock_sendmsg(sock
, &msg
);
1191 if (unlikely(ret
< min_ret
)) {
1192 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1193 return io_setup_async_addr(req
, &__address
, issue_flags
);
1195 if (ret
> 0 && io_net_retry(sock
, msg
.msg_flags
)) {
1199 req
->flags
|= REQ_F_PARTIAL_IO
;
1200 return io_setup_async_addr(req
, &__address
, issue_flags
);
1202 if (ret
== -ERESTARTSYS
)
1209 else if (zc
->done_io
)
1213 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1214 * flushing notif to io_send_zc_cleanup()
1216 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
1217 io_notif_flush(zc
->notif
);
1218 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
1220 io_req_set_res(req
, ret
, IORING_CQE_F_MORE
);
1224 int io_sendmsg_zc(struct io_kiocb
*req
, unsigned int issue_flags
)
1226 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1227 struct io_async_msghdr iomsg
, *kmsg
;
1228 struct socket
*sock
;
1230 int ret
, min_ret
= 0;
1232 io_notif_set_extended(sr
->notif
);
1234 sock
= sock_from_file(req
->file
);
1235 if (unlikely(!sock
))
1237 if (!test_bit(SOCK_SUPPORT_ZC
, &sock
->flags
))
1240 if (req_has_async_data(req
)) {
1241 kmsg
= req
->async_data
;
1243 ret
= io_sendmsg_copy_hdr(req
, &iomsg
);
1249 if (!(req
->flags
& REQ_F_POLLED
) &&
1250 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
1251 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1253 flags
= sr
->msg_flags
| MSG_ZEROCOPY
;
1254 if (issue_flags
& IO_URING_F_NONBLOCK
)
1255 flags
|= MSG_DONTWAIT
;
1256 if (flags
& MSG_WAITALL
)
1257 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
1259 kmsg
->msg
.msg_ubuf
= &io_notif_to_data(sr
->notif
)->uarg
;
1260 kmsg
->msg
.sg_from_iter
= io_sg_from_iter_iovec
;
1261 ret
= __sys_sendmsg_sock(sock
, &kmsg
->msg
, flags
);
1263 if (unlikely(ret
< min_ret
)) {
1264 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1265 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1267 if (ret
> 0 && io_net_retry(sock
, flags
)) {
1269 req
->flags
|= REQ_F_PARTIAL_IO
;
1270 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1272 if (ret
== -ERESTARTSYS
)
1276 /* fast path, check for non-NULL to avoid function call */
1277 if (kmsg
->free_iov
) {
1278 kfree(kmsg
->free_iov
);
1279 kmsg
->free_iov
= NULL
;
1282 io_netmsg_recycle(req
, issue_flags
);
1285 else if (sr
->done_io
)
1289 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1290 * flushing notif to io_send_zc_cleanup()
1292 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
1293 io_notif_flush(sr
->notif
);
1294 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
1296 io_req_set_res(req
, ret
, IORING_CQE_F_MORE
);
1300 void io_sendrecv_fail(struct io_kiocb
*req
)
1302 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1304 if (req
->flags
& REQ_F_PARTIAL_IO
)
1305 req
->cqe
.res
= sr
->done_io
;
1307 if ((req
->flags
& REQ_F_NEED_CLEANUP
) &&
1308 (req
->opcode
== IORING_OP_SEND_ZC
|| req
->opcode
== IORING_OP_SENDMSG_ZC
))
1309 req
->cqe
.flags
|= IORING_CQE_F_MORE
;
1312 int io_accept_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1314 struct io_accept
*accept
= io_kiocb_to_cmd(req
, struct io_accept
);
1317 if (sqe
->len
|| sqe
->buf_index
)
1320 accept
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1321 accept
->addr_len
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
1322 accept
->flags
= READ_ONCE(sqe
->accept_flags
);
1323 accept
->nofile
= rlimit(RLIMIT_NOFILE
);
1324 flags
= READ_ONCE(sqe
->ioprio
);
1325 if (flags
& ~IORING_ACCEPT_MULTISHOT
)
1328 accept
->file_slot
= READ_ONCE(sqe
->file_index
);
1329 if (accept
->file_slot
) {
1330 if (accept
->flags
& SOCK_CLOEXEC
)
1332 if (flags
& IORING_ACCEPT_MULTISHOT
&&
1333 accept
->file_slot
!= IORING_FILE_INDEX_ALLOC
)
1336 if (accept
->flags
& ~(SOCK_CLOEXEC
| SOCK_NONBLOCK
))
1338 if (SOCK_NONBLOCK
!= O_NONBLOCK
&& (accept
->flags
& SOCK_NONBLOCK
))
1339 accept
->flags
= (accept
->flags
& ~SOCK_NONBLOCK
) | O_NONBLOCK
;
1340 if (flags
& IORING_ACCEPT_MULTISHOT
)
1341 req
->flags
|= REQ_F_APOLL_MULTISHOT
;
1345 int io_accept(struct io_kiocb
*req
, unsigned int issue_flags
)
1347 struct io_accept
*accept
= io_kiocb_to_cmd(req
, struct io_accept
);
1348 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
1349 unsigned int file_flags
= force_nonblock
? O_NONBLOCK
: 0;
1350 bool fixed
= !!accept
->file_slot
;
1354 if (!io_check_multishot(req
, issue_flags
))
1358 fd
= __get_unused_fd_flags(accept
->flags
, accept
->nofile
);
1359 if (unlikely(fd
< 0))
1362 file
= do_accept(req
->file
, file_flags
, accept
->addr
, accept
->addr_len
,
1367 ret
= PTR_ERR(file
);
1368 if (ret
== -EAGAIN
&& force_nonblock
) {
1370 * if it's multishot and polled, we don't need to
1371 * return EAGAIN to arm the poll infra since it
1372 * has already been done
1374 if (issue_flags
& IO_URING_F_MULTISHOT
)
1375 ret
= IOU_ISSUE_SKIP_COMPLETE
;
1378 if (ret
== -ERESTARTSYS
)
1381 } else if (!fixed
) {
1382 fd_install(fd
, file
);
1385 ret
= io_fixed_fd_install(req
, issue_flags
, file
,
1389 if (!(req
->flags
& REQ_F_APOLL_MULTISHOT
)) {
1390 io_req_set_res(req
, ret
, 0);
1396 if (io_fill_cqe_req_aux(req
, issue_flags
& IO_URING_F_COMPLETE_DEFER
,
1397 ret
, IORING_CQE_F_MORE
))
1403 int io_socket_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1405 struct io_socket
*sock
= io_kiocb_to_cmd(req
, struct io_socket
);
1407 if (sqe
->addr
|| sqe
->rw_flags
|| sqe
->buf_index
)
1410 sock
->domain
= READ_ONCE(sqe
->fd
);
1411 sock
->type
= READ_ONCE(sqe
->off
);
1412 sock
->protocol
= READ_ONCE(sqe
->len
);
1413 sock
->file_slot
= READ_ONCE(sqe
->file_index
);
1414 sock
->nofile
= rlimit(RLIMIT_NOFILE
);
1416 sock
->flags
= sock
->type
& ~SOCK_TYPE_MASK
;
1417 if (sock
->file_slot
&& (sock
->flags
& SOCK_CLOEXEC
))
1419 if (sock
->flags
& ~(SOCK_CLOEXEC
| SOCK_NONBLOCK
))
1424 int io_socket(struct io_kiocb
*req
, unsigned int issue_flags
)
1426 struct io_socket
*sock
= io_kiocb_to_cmd(req
, struct io_socket
);
1427 bool fixed
= !!sock
->file_slot
;
1432 fd
= __get_unused_fd_flags(sock
->flags
, sock
->nofile
);
1433 if (unlikely(fd
< 0))
1436 file
= __sys_socket_file(sock
->domain
, sock
->type
, sock
->protocol
);
1440 ret
= PTR_ERR(file
);
1441 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1443 if (ret
== -ERESTARTSYS
)
1446 } else if (!fixed
) {
1447 fd_install(fd
, file
);
1450 ret
= io_fixed_fd_install(req
, issue_flags
, file
,
1453 io_req_set_res(req
, ret
, 0);
1457 int io_connect_prep_async(struct io_kiocb
*req
)
1459 struct io_async_connect
*io
= req
->async_data
;
1460 struct io_connect
*conn
= io_kiocb_to_cmd(req
, struct io_connect
);
1462 return move_addr_to_kernel(conn
->addr
, conn
->addr_len
, &io
->address
);
1465 int io_connect_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1467 struct io_connect
*conn
= io_kiocb_to_cmd(req
, struct io_connect
);
1469 if (sqe
->len
|| sqe
->buf_index
|| sqe
->rw_flags
|| sqe
->splice_fd_in
)
1472 conn
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1473 conn
->addr_len
= READ_ONCE(sqe
->addr2
);
1474 conn
->in_progress
= conn
->seen_econnaborted
= false;
1478 int io_connect(struct io_kiocb
*req
, unsigned int issue_flags
)
1480 struct io_connect
*connect
= io_kiocb_to_cmd(req
, struct io_connect
);
1481 struct io_async_connect __io
, *io
;
1482 unsigned file_flags
;
1484 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
1486 if (req_has_async_data(req
)) {
1487 io
= req
->async_data
;
1489 ret
= move_addr_to_kernel(connect
->addr
,
1497 file_flags
= force_nonblock
? O_NONBLOCK
: 0;
1499 ret
= __sys_connect_file(req
->file
, &io
->address
,
1500 connect
->addr_len
, file_flags
);
1501 if ((ret
== -EAGAIN
|| ret
== -EINPROGRESS
|| ret
== -ECONNABORTED
)
1502 && force_nonblock
) {
1503 if (ret
== -EINPROGRESS
) {
1504 connect
->in_progress
= true;
1505 } else if (ret
== -ECONNABORTED
) {
1506 if (connect
->seen_econnaborted
)
1508 connect
->seen_econnaborted
= true;
1510 if (req_has_async_data(req
))
1512 if (io_alloc_async_data(req
)) {
1516 memcpy(req
->async_data
, &__io
, sizeof(__io
));
1519 if (connect
->in_progress
) {
1521 * At least bluetooth will return -EBADFD on a re-connect
1522 * attempt, and it's (supposedly) also valid to get -EISCONN
1523 * which means the previous result is good. For both of these,
1524 * grab the sock_error() and use that for the completion.
1526 if (ret
== -EBADFD
|| ret
== -EISCONN
)
1527 ret
= sock_error(sock_from_file(req
->file
)->sk
);
1529 if (ret
== -ERESTARTSYS
)
1534 io_req_set_res(req
, ret
, 0);
1538 void io_netmsg_cache_free(struct io_cache_entry
*entry
)
1540 kfree(container_of(entry
, struct io_async_msghdr
, cache
));