1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/slab.h>
7 #include <linux/compat.h>
8 #include <net/compat.h>
9 #include <linux/io_uring.h>
11 #include <uapi/linux/io_uring.h>
15 #include "alloc_cache.h"
20 #if defined(CONFIG_NET)
28 struct sockaddr __user
*addr
;
47 struct sockaddr __user
*addr
;
50 bool seen_econnaborted
;
56 struct compat_msghdr __user
*umsg_compat
;
57 struct user_msghdr __user
*umsg
;
63 unsigned nr_multishot_loops
;
65 /* initialised and used only by !msg send variants */
69 void __user
*msg_control
;
70 /* used only for send zerocopy */
71 struct io_kiocb
*notif
;
75 * Number of times we'll try and do receives if there's more data. If we
76 * exceed this limit, then add us to the back of the queue and retry from
77 * there. This helps fairness between flooding clients.
79 #define MULTISHOT_MAX_RETRY 32
81 int io_shutdown_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
83 struct io_shutdown
*shutdown
= io_kiocb_to_cmd(req
, struct io_shutdown
);
85 if (unlikely(sqe
->off
|| sqe
->addr
|| sqe
->rw_flags
||
86 sqe
->buf_index
|| sqe
->splice_fd_in
))
89 shutdown
->how
= READ_ONCE(sqe
->len
);
90 req
->flags
|= REQ_F_FORCE_ASYNC
;
94 int io_shutdown(struct io_kiocb
*req
, unsigned int issue_flags
)
96 struct io_shutdown
*shutdown
= io_kiocb_to_cmd(req
, struct io_shutdown
);
100 WARN_ON_ONCE(issue_flags
& IO_URING_F_NONBLOCK
);
102 sock
= sock_from_file(req
->file
);
106 ret
= __sys_shutdown_sock(sock
, shutdown
->how
);
107 io_req_set_res(req
, ret
, 0);
111 static bool io_net_retry(struct socket
*sock
, int flags
)
113 if (!(flags
& MSG_WAITALL
))
115 return sock
->type
== SOCK_STREAM
|| sock
->type
== SOCK_SEQPACKET
;
118 static void io_netmsg_recycle(struct io_kiocb
*req
, unsigned int issue_flags
)
120 struct io_async_msghdr
*hdr
= req
->async_data
;
122 if (!req_has_async_data(req
) || issue_flags
& IO_URING_F_UNLOCKED
)
125 /* Let normal cleanup path reap it if we fail adding to the cache */
126 if (io_alloc_cache_put(&req
->ctx
->netmsg_cache
, &hdr
->cache
)) {
127 req
->async_data
= NULL
;
128 req
->flags
&= ~REQ_F_ASYNC_DATA
;
132 static struct io_async_msghdr
*io_msg_alloc_async(struct io_kiocb
*req
,
133 unsigned int issue_flags
)
135 struct io_ring_ctx
*ctx
= req
->ctx
;
136 struct io_cache_entry
*entry
;
137 struct io_async_msghdr
*hdr
;
139 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
140 entry
= io_alloc_cache_get(&ctx
->netmsg_cache
);
142 hdr
= container_of(entry
, struct io_async_msghdr
, cache
);
143 hdr
->free_iov
= NULL
;
144 req
->flags
|= REQ_F_ASYNC_DATA
;
145 req
->async_data
= hdr
;
150 if (!io_alloc_async_data(req
)) {
151 hdr
= req
->async_data
;
152 hdr
->free_iov
= NULL
;
158 static inline struct io_async_msghdr
*io_msg_alloc_async_prep(struct io_kiocb
*req
)
160 /* ->prep_async is always called from the submission context */
161 return io_msg_alloc_async(req
, 0);
164 static int io_setup_async_msg(struct io_kiocb
*req
,
165 struct io_async_msghdr
*kmsg
,
166 unsigned int issue_flags
)
168 struct io_async_msghdr
*async_msg
;
170 if (req_has_async_data(req
))
172 async_msg
= io_msg_alloc_async(req
, issue_flags
);
174 kfree(kmsg
->free_iov
);
177 req
->flags
|= REQ_F_NEED_CLEANUP
;
178 memcpy(async_msg
, kmsg
, sizeof(*kmsg
));
179 if (async_msg
->msg
.msg_name
)
180 async_msg
->msg
.msg_name
= &async_msg
->addr
;
182 if ((req
->flags
& REQ_F_BUFFER_SELECT
) && !async_msg
->msg
.msg_iter
.nr_segs
)
185 /* if were using fast_iov, set it to the new one */
186 if (iter_is_iovec(&kmsg
->msg
.msg_iter
) && !kmsg
->free_iov
) {
187 size_t fast_idx
= iter_iov(&kmsg
->msg
.msg_iter
) - kmsg
->fast_iov
;
188 async_msg
->msg
.msg_iter
.__iov
= &async_msg
->fast_iov
[fast_idx
];
195 static int io_compat_msg_copy_hdr(struct io_kiocb
*req
,
196 struct io_async_msghdr
*iomsg
,
197 struct compat_msghdr
*msg
, int ddir
)
199 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
200 struct compat_iovec __user
*uiov
;
203 if (copy_from_user(msg
, sr
->umsg_compat
, sizeof(*msg
)))
206 uiov
= compat_ptr(msg
->msg_iov
);
207 if (req
->flags
& REQ_F_BUFFER_SELECT
) {
210 iomsg
->free_iov
= NULL
;
211 if (msg
->msg_iovlen
== 0) {
213 } else if (msg
->msg_iovlen
> 1) {
216 if (!access_ok(uiov
, sizeof(*uiov
)))
218 if (__get_user(clen
, &uiov
->iov_len
))
228 iomsg
->free_iov
= iomsg
->fast_iov
;
229 ret
= __import_iovec(ddir
, (struct iovec __user
*)uiov
, msg
->msg_iovlen
,
230 UIO_FASTIOV
, &iomsg
->free_iov
,
231 &iomsg
->msg
.msg_iter
, true);
232 if (unlikely(ret
< 0))
239 static int io_msg_copy_hdr(struct io_kiocb
*req
, struct io_async_msghdr
*iomsg
,
240 struct user_msghdr
*msg
, int ddir
)
242 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
245 if (!user_access_begin(sr
->umsg
, sizeof(*sr
->umsg
)))
249 unsafe_get_user(msg
->msg_name
, &sr
->umsg
->msg_name
, ua_end
);
250 unsafe_get_user(msg
->msg_namelen
, &sr
->umsg
->msg_namelen
, ua_end
);
251 unsafe_get_user(msg
->msg_iov
, &sr
->umsg
->msg_iov
, ua_end
);
252 unsafe_get_user(msg
->msg_iovlen
, &sr
->umsg
->msg_iovlen
, ua_end
);
253 unsafe_get_user(msg
->msg_control
, &sr
->umsg
->msg_control
, ua_end
);
254 unsafe_get_user(msg
->msg_controllen
, &sr
->umsg
->msg_controllen
, ua_end
);
257 if (req
->flags
& REQ_F_BUFFER_SELECT
) {
258 if (msg
->msg_iovlen
== 0) {
259 sr
->len
= iomsg
->fast_iov
[0].iov_len
= 0;
260 iomsg
->fast_iov
[0].iov_base
= NULL
;
261 iomsg
->free_iov
= NULL
;
262 } else if (msg
->msg_iovlen
> 1) {
266 /* we only need the length for provided buffers */
267 if (!access_ok(&msg
->msg_iov
[0].iov_len
, sizeof(__kernel_size_t
)))
269 unsafe_get_user(iomsg
->fast_iov
[0].iov_len
,
270 &msg
->msg_iov
[0].iov_len
, ua_end
);
271 sr
->len
= iomsg
->fast_iov
[0].iov_len
;
272 iomsg
->free_iov
= NULL
;
281 iomsg
->free_iov
= iomsg
->fast_iov
;
282 ret
= __import_iovec(ddir
, msg
->msg_iov
, msg
->msg_iovlen
, UIO_FASTIOV
,
283 &iomsg
->free_iov
, &iomsg
->msg
.msg_iter
, false);
284 if (unlikely(ret
< 0))
290 static int io_sendmsg_copy_hdr(struct io_kiocb
*req
,
291 struct io_async_msghdr
*iomsg
)
293 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
294 struct user_msghdr msg
;
297 iomsg
->msg
.msg_name
= &iomsg
->addr
;
298 iomsg
->msg
.msg_iter
.nr_segs
= 0;
301 if (unlikely(req
->ctx
->compat
)) {
302 struct compat_msghdr cmsg
;
304 ret
= io_compat_msg_copy_hdr(req
, iomsg
, &cmsg
, ITER_SOURCE
);
308 return __get_compat_msghdr(&iomsg
->msg
, &cmsg
, NULL
);
312 ret
= io_msg_copy_hdr(req
, iomsg
, &msg
, ITER_SOURCE
);
316 ret
= __copy_msghdr(&iomsg
->msg
, &msg
, NULL
);
318 /* save msg_control as sys_sendmsg() overwrites it */
319 sr
->msg_control
= iomsg
->msg
.msg_control_user
;
323 int io_send_prep_async(struct io_kiocb
*req
)
325 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
326 struct io_async_msghdr
*io
;
329 if (req_has_async_data(req
))
334 io
= io_msg_alloc_async_prep(req
);
337 ret
= move_addr_to_kernel(zc
->addr
, zc
->addr_len
, &io
->addr
);
341 static int io_setup_async_addr(struct io_kiocb
*req
,
342 struct sockaddr_storage
*addr_storage
,
343 unsigned int issue_flags
)
345 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
346 struct io_async_msghdr
*io
;
348 if (!sr
->addr
|| req_has_async_data(req
))
350 io
= io_msg_alloc_async(req
, issue_flags
);
353 memcpy(&io
->addr
, addr_storage
, sizeof(io
->addr
));
357 int io_sendmsg_prep_async(struct io_kiocb
*req
)
359 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
363 if (!io_msg_alloc_async_prep(req
))
365 ret
= io_sendmsg_copy_hdr(req
, req
->async_data
);
367 req
->flags
|= REQ_F_NEED_CLEANUP
;
371 void io_sendmsg_recvmsg_cleanup(struct io_kiocb
*req
)
373 struct io_async_msghdr
*io
= req
->async_data
;
378 int io_sendmsg_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
380 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
384 if (req
->opcode
== IORING_OP_SEND
) {
385 if (READ_ONCE(sqe
->__pad3
[0]))
387 sr
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
388 sr
->addr_len
= READ_ONCE(sqe
->addr_len
);
389 } else if (sqe
->addr2
|| sqe
->file_index
) {
393 sr
->umsg
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
394 sr
->len
= READ_ONCE(sqe
->len
);
395 sr
->flags
= READ_ONCE(sqe
->ioprio
);
396 if (sr
->flags
& ~IORING_RECVSEND_POLL_FIRST
)
398 sr
->msg_flags
= READ_ONCE(sqe
->msg_flags
) | MSG_NOSIGNAL
;
399 if (sr
->msg_flags
& MSG_DONTWAIT
)
400 req
->flags
|= REQ_F_NOWAIT
;
403 if (req
->ctx
->compat
)
404 sr
->msg_flags
|= MSG_CMSG_COMPAT
;
409 static void io_req_msg_cleanup(struct io_kiocb
*req
,
410 struct io_async_msghdr
*kmsg
,
411 unsigned int issue_flags
)
413 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
414 /* fast path, check for non-NULL to avoid function call */
416 kfree(kmsg
->free_iov
);
417 io_netmsg_recycle(req
, issue_flags
);
420 int io_sendmsg(struct io_kiocb
*req
, unsigned int issue_flags
)
422 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
423 struct io_async_msghdr iomsg
, *kmsg
;
429 sock
= sock_from_file(req
->file
);
433 if (req_has_async_data(req
)) {
434 kmsg
= req
->async_data
;
435 kmsg
->msg
.msg_control_user
= sr
->msg_control
;
437 ret
= io_sendmsg_copy_hdr(req
, &iomsg
);
443 if (!(req
->flags
& REQ_F_POLLED
) &&
444 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
445 return io_setup_async_msg(req
, kmsg
, issue_flags
);
447 flags
= sr
->msg_flags
;
448 if (issue_flags
& IO_URING_F_NONBLOCK
)
449 flags
|= MSG_DONTWAIT
;
450 if (flags
& MSG_WAITALL
)
451 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
453 ret
= __sys_sendmsg_sock(sock
, &kmsg
->msg
, flags
);
456 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
457 return io_setup_async_msg(req
, kmsg
, issue_flags
);
458 if (ret
> 0 && io_net_retry(sock
, flags
)) {
459 kmsg
->msg
.msg_controllen
= 0;
460 kmsg
->msg
.msg_control
= NULL
;
462 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
463 return io_setup_async_msg(req
, kmsg
, issue_flags
);
465 if (ret
== -ERESTARTSYS
)
469 io_req_msg_cleanup(req
, kmsg
, issue_flags
);
472 else if (sr
->done_io
)
474 io_req_set_res(req
, ret
, 0);
478 int io_send(struct io_kiocb
*req
, unsigned int issue_flags
)
480 struct sockaddr_storage __address
;
481 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
489 msg
.msg_control
= NULL
;
490 msg
.msg_controllen
= 0;
495 if (req_has_async_data(req
)) {
496 struct io_async_msghdr
*io
= req
->async_data
;
498 msg
.msg_name
= &io
->addr
;
500 ret
= move_addr_to_kernel(sr
->addr
, sr
->addr_len
, &__address
);
501 if (unlikely(ret
< 0))
503 msg
.msg_name
= (struct sockaddr
*)&__address
;
505 msg
.msg_namelen
= sr
->addr_len
;
508 if (!(req
->flags
& REQ_F_POLLED
) &&
509 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
510 return io_setup_async_addr(req
, &__address
, issue_flags
);
512 sock
= sock_from_file(req
->file
);
516 ret
= import_ubuf(ITER_SOURCE
, sr
->buf
, sr
->len
, &msg
.msg_iter
);
520 flags
= sr
->msg_flags
;
521 if (issue_flags
& IO_URING_F_NONBLOCK
)
522 flags
|= MSG_DONTWAIT
;
523 if (flags
& MSG_WAITALL
)
524 min_ret
= iov_iter_count(&msg
.msg_iter
);
526 flags
&= ~MSG_INTERNAL_SENDMSG_FLAGS
;
527 msg
.msg_flags
= flags
;
528 ret
= sock_sendmsg(sock
, &msg
);
530 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
531 return io_setup_async_addr(req
, &__address
, issue_flags
);
533 if (ret
> 0 && io_net_retry(sock
, flags
)) {
537 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
538 return io_setup_async_addr(req
, &__address
, issue_flags
);
540 if (ret
== -ERESTARTSYS
)
546 else if (sr
->done_io
)
548 io_req_set_res(req
, ret
, 0);
552 static int io_recvmsg_mshot_prep(struct io_kiocb
*req
,
553 struct io_async_msghdr
*iomsg
,
554 int namelen
, size_t controllen
)
556 if ((req
->flags
& (REQ_F_APOLL_MULTISHOT
|REQ_F_BUFFER_SELECT
)) ==
557 (REQ_F_APOLL_MULTISHOT
|REQ_F_BUFFER_SELECT
)) {
560 if (unlikely(namelen
< 0))
562 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out
),
565 if (check_add_overflow(hdr
, controllen
, &hdr
))
568 iomsg
->namelen
= namelen
;
569 iomsg
->controllen
= controllen
;
576 static int io_recvmsg_copy_hdr(struct io_kiocb
*req
,
577 struct io_async_msghdr
*iomsg
)
579 struct user_msghdr msg
;
582 iomsg
->msg
.msg_name
= &iomsg
->addr
;
583 iomsg
->msg
.msg_iter
.nr_segs
= 0;
586 if (unlikely(req
->ctx
->compat
)) {
587 struct compat_msghdr cmsg
;
589 ret
= io_compat_msg_copy_hdr(req
, iomsg
, &cmsg
, ITER_DEST
);
593 ret
= __get_compat_msghdr(&iomsg
->msg
, &cmsg
, &iomsg
->uaddr
);
597 return io_recvmsg_mshot_prep(req
, iomsg
, cmsg
.msg_namelen
,
598 cmsg
.msg_controllen
);
602 ret
= io_msg_copy_hdr(req
, iomsg
, &msg
, ITER_DEST
);
606 ret
= __copy_msghdr(&iomsg
->msg
, &msg
, &iomsg
->uaddr
);
610 return io_recvmsg_mshot_prep(req
, iomsg
, msg
.msg_namelen
,
614 int io_recvmsg_prep_async(struct io_kiocb
*req
)
616 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
617 struct io_async_msghdr
*iomsg
;
621 if (!io_msg_alloc_async_prep(req
))
623 iomsg
= req
->async_data
;
624 ret
= io_recvmsg_copy_hdr(req
, iomsg
);
626 req
->flags
|= REQ_F_NEED_CLEANUP
;
630 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)
632 int io_recvmsg_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
634 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
638 if (unlikely(sqe
->file_index
|| sqe
->addr2
))
641 sr
->umsg
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
642 sr
->len
= READ_ONCE(sqe
->len
);
643 sr
->flags
= READ_ONCE(sqe
->ioprio
);
644 if (sr
->flags
& ~(RECVMSG_FLAGS
))
646 sr
->msg_flags
= READ_ONCE(sqe
->msg_flags
);
647 if (sr
->msg_flags
& MSG_DONTWAIT
)
648 req
->flags
|= REQ_F_NOWAIT
;
649 if (sr
->msg_flags
& MSG_ERRQUEUE
)
650 req
->flags
|= REQ_F_CLEAR_POLLIN
;
651 if (sr
->flags
& IORING_RECV_MULTISHOT
) {
652 if (!(req
->flags
& REQ_F_BUFFER_SELECT
))
654 if (sr
->msg_flags
& MSG_WAITALL
)
656 if (req
->opcode
== IORING_OP_RECV
&& sr
->len
)
658 req
->flags
|= REQ_F_APOLL_MULTISHOT
;
660 * Store the buffer group for this multishot receive separately,
661 * as if we end up doing an io-wq based issue that selects a
662 * buffer, it has to be committed immediately and that will
663 * clear ->buf_list. This means we lose the link to the buffer
664 * list, and the eventual buffer put on completion then cannot
667 sr
->buf_group
= req
->buf_index
;
671 if (req
->ctx
->compat
)
672 sr
->msg_flags
|= MSG_CMSG_COMPAT
;
674 sr
->nr_multishot_loops
= 0;
678 static inline void io_recv_prep_retry(struct io_kiocb
*req
)
680 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
682 req
->flags
&= ~REQ_F_BL_EMPTY
;
684 sr
->len
= 0; /* get from the provided buffer */
685 req
->buf_index
= sr
->buf_group
;
689 * Finishes io_recv and io_recvmsg.
691 * Returns true if it is actually finished, or false if it should run
692 * again (for multishot).
694 static inline bool io_recv_finish(struct io_kiocb
*req
, int *ret
,
695 struct msghdr
*msg
, bool mshot_finished
,
696 unsigned issue_flags
)
700 cflags
= io_put_kbuf(req
, issue_flags
);
701 if (msg
->msg_inq
> 0)
702 cflags
|= IORING_CQE_F_SOCK_NONEMPTY
;
705 * Fill CQE for this receive and see if we should keep trying to
706 * receive from this socket.
708 if ((req
->flags
& REQ_F_APOLL_MULTISHOT
) && !mshot_finished
&&
709 io_fill_cqe_req_aux(req
, issue_flags
& IO_URING_F_COMPLETE_DEFER
,
710 *ret
, cflags
| IORING_CQE_F_MORE
)) {
711 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
712 int mshot_retry_ret
= IOU_ISSUE_SKIP_COMPLETE
;
714 io_recv_prep_retry(req
);
715 /* Known not-empty or unknown state, retry */
716 if (cflags
& IORING_CQE_F_SOCK_NONEMPTY
|| msg
->msg_inq
< 0) {
717 if (sr
->nr_multishot_loops
++ < MULTISHOT_MAX_RETRY
)
719 /* mshot retries exceeded, force a requeue */
720 sr
->nr_multishot_loops
= 0;
721 mshot_retry_ret
= IOU_REQUEUE
;
723 if (issue_flags
& IO_URING_F_MULTISHOT
)
724 *ret
= mshot_retry_ret
;
730 /* Finish the request / stop multishot. */
731 io_req_set_res(req
, *ret
, cflags
);
733 if (issue_flags
& IO_URING_F_MULTISHOT
)
734 *ret
= IOU_STOP_MULTISHOT
;
740 static int io_recvmsg_prep_multishot(struct io_async_msghdr
*kmsg
,
741 struct io_sr_msg
*sr
, void __user
**buf
,
744 unsigned long ubuf
= (unsigned long) *buf
;
747 hdr
= sizeof(struct io_uring_recvmsg_out
) + kmsg
->namelen
+
752 if (kmsg
->controllen
) {
753 unsigned long control
= ubuf
+ hdr
- kmsg
->controllen
;
755 kmsg
->msg
.msg_control_user
= (void __user
*) control
;
756 kmsg
->msg
.msg_controllen
= kmsg
->controllen
;
759 sr
->buf
= *buf
; /* stash for later copy */
760 *buf
= (void __user
*) (ubuf
+ hdr
);
761 kmsg
->payloadlen
= *len
= *len
- hdr
;
765 struct io_recvmsg_multishot_hdr
{
766 struct io_uring_recvmsg_out msg
;
767 struct sockaddr_storage addr
;
770 static int io_recvmsg_multishot(struct socket
*sock
, struct io_sr_msg
*io
,
771 struct io_async_msghdr
*kmsg
,
772 unsigned int flags
, bool *finished
)
776 struct io_recvmsg_multishot_hdr hdr
;
779 kmsg
->msg
.msg_name
= &hdr
.addr
;
780 kmsg
->msg
.msg_flags
= flags
& (MSG_CMSG_CLOEXEC
|MSG_CMSG_COMPAT
);
781 kmsg
->msg
.msg_namelen
= 0;
783 if (sock
->file
->f_flags
& O_NONBLOCK
)
784 flags
|= MSG_DONTWAIT
;
786 err
= sock_recvmsg(sock
, &kmsg
->msg
, flags
);
787 *finished
= err
<= 0;
791 hdr
.msg
= (struct io_uring_recvmsg_out
) {
792 .controllen
= kmsg
->controllen
- kmsg
->msg
.msg_controllen
,
793 .flags
= kmsg
->msg
.msg_flags
& ~MSG_CMSG_COMPAT
796 hdr
.msg
.payloadlen
= err
;
797 if (err
> kmsg
->payloadlen
)
798 err
= kmsg
->payloadlen
;
800 copy_len
= sizeof(struct io_uring_recvmsg_out
);
801 if (kmsg
->msg
.msg_namelen
> kmsg
->namelen
)
802 copy_len
+= kmsg
->namelen
;
804 copy_len
+= kmsg
->msg
.msg_namelen
;
807 * "fromlen shall refer to the value before truncation.."
810 hdr
.msg
.namelen
= kmsg
->msg
.msg_namelen
;
812 /* ensure that there is no gap between hdr and sockaddr_storage */
813 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr
, addr
) !=
814 sizeof(struct io_uring_recvmsg_out
));
815 if (copy_to_user(io
->buf
, &hdr
, copy_len
)) {
820 return sizeof(struct io_uring_recvmsg_out
) + kmsg
->namelen
+
821 kmsg
->controllen
+ err
;
824 int io_recvmsg(struct io_kiocb
*req
, unsigned int issue_flags
)
826 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
827 struct io_async_msghdr iomsg
, *kmsg
;
830 int ret
, min_ret
= 0;
831 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
832 bool mshot_finished
= true;
834 sock
= sock_from_file(req
->file
);
838 if (req_has_async_data(req
)) {
839 kmsg
= req
->async_data
;
841 ret
= io_recvmsg_copy_hdr(req
, &iomsg
);
847 if (!(req
->flags
& REQ_F_POLLED
) &&
848 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
849 return io_setup_async_msg(req
, kmsg
, issue_flags
);
851 flags
= sr
->msg_flags
;
853 flags
|= MSG_DONTWAIT
;
856 if (io_do_buffer_select(req
)) {
858 size_t len
= sr
->len
;
860 buf
= io_buffer_select(req
, &len
, issue_flags
);
864 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
865 ret
= io_recvmsg_prep_multishot(kmsg
, sr
, &buf
, &len
);
867 io_kbuf_recycle(req
, issue_flags
);
872 iov_iter_ubuf(&kmsg
->msg
.msg_iter
, ITER_DEST
, buf
, len
);
875 kmsg
->msg
.msg_get_inq
= 1;
876 kmsg
->msg
.msg_inq
= -1;
877 if (req
->flags
& REQ_F_APOLL_MULTISHOT
) {
878 ret
= io_recvmsg_multishot(sock
, sr
, kmsg
, flags
,
881 /* disable partial retry for recvmsg with cmsg attached */
882 if (flags
& MSG_WAITALL
&& !kmsg
->msg
.msg_controllen
)
883 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
885 ret
= __sys_recvmsg_sock(sock
, &kmsg
->msg
, sr
->umsg
,
890 if (ret
== -EAGAIN
&& force_nonblock
) {
891 ret
= io_setup_async_msg(req
, kmsg
, issue_flags
);
892 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_MULTISHOT
)) {
893 io_kbuf_recycle(req
, issue_flags
);
894 return IOU_ISSUE_SKIP_COMPLETE
;
898 if (ret
> 0 && io_net_retry(sock
, flags
)) {
900 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
901 return io_setup_async_msg(req
, kmsg
, issue_flags
);
903 if (ret
== -ERESTARTSYS
)
906 } else if ((flags
& MSG_WAITALL
) && (kmsg
->msg
.msg_flags
& (MSG_TRUNC
| MSG_CTRUNC
))) {
912 else if (sr
->done_io
)
915 io_kbuf_recycle(req
, issue_flags
);
917 if (!io_recv_finish(req
, &ret
, &kmsg
->msg
, mshot_finished
, issue_flags
))
918 goto retry_multishot
;
921 io_req_msg_cleanup(req
, kmsg
, issue_flags
);
922 else if (ret
== -EAGAIN
)
923 return io_setup_async_msg(req
, kmsg
, issue_flags
);
928 int io_recv(struct io_kiocb
*req
, unsigned int issue_flags
)
930 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
934 int ret
, min_ret
= 0;
935 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
936 size_t len
= sr
->len
;
938 if (!(req
->flags
& REQ_F_POLLED
) &&
939 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
942 sock
= sock_from_file(req
->file
);
948 msg
.msg_control
= NULL
;
950 msg
.msg_controllen
= 0;
954 flags
= sr
->msg_flags
;
956 flags
|= MSG_DONTWAIT
;
959 if (io_do_buffer_select(req
)) {
962 buf
= io_buffer_select(req
, &len
, issue_flags
);
969 ret
= import_ubuf(ITER_DEST
, sr
->buf
, len
, &msg
.msg_iter
);
976 if (flags
& MSG_WAITALL
)
977 min_ret
= iov_iter_count(&msg
.msg_iter
);
979 ret
= sock_recvmsg(sock
, &msg
, flags
);
981 if (ret
== -EAGAIN
&& force_nonblock
) {
982 if (issue_flags
& IO_URING_F_MULTISHOT
) {
983 io_kbuf_recycle(req
, issue_flags
);
984 return IOU_ISSUE_SKIP_COMPLETE
;
989 if (ret
> 0 && io_net_retry(sock
, flags
)) {
993 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
996 if (ret
== -ERESTARTSYS
)
999 } else if ((flags
& MSG_WAITALL
) && (msg
.msg_flags
& (MSG_TRUNC
| MSG_CTRUNC
))) {
1006 else if (sr
->done_io
)
1009 io_kbuf_recycle(req
, issue_flags
);
1011 if (!io_recv_finish(req
, &ret
, &msg
, ret
<= 0, issue_flags
))
1012 goto retry_multishot
;
1017 void io_send_zc_cleanup(struct io_kiocb
*req
)
1019 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1020 struct io_async_msghdr
*io
;
1022 if (req_has_async_data(req
)) {
1023 io
= req
->async_data
;
1024 /* might be ->fast_iov if *msg_copy_hdr failed */
1025 if (io
->free_iov
!= io
->fast_iov
)
1026 kfree(io
->free_iov
);
1029 io_notif_flush(zc
->notif
);
1034 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF)
1035 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE)
1037 int io_send_zc_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1039 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1040 struct io_ring_ctx
*ctx
= req
->ctx
;
1041 struct io_kiocb
*notif
;
1045 if (unlikely(READ_ONCE(sqe
->__pad2
[0]) || READ_ONCE(sqe
->addr3
)))
1047 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */
1048 if (req
->flags
& REQ_F_CQE_SKIP
)
1051 notif
= zc
->notif
= io_alloc_notif(ctx
);
1054 notif
->cqe
.user_data
= req
->cqe
.user_data
;
1056 notif
->cqe
.flags
= IORING_CQE_F_NOTIF
;
1057 req
->flags
|= REQ_F_NEED_CLEANUP
;
1059 zc
->flags
= READ_ONCE(sqe
->ioprio
);
1060 if (unlikely(zc
->flags
& ~IO_ZC_FLAGS_COMMON
)) {
1061 if (zc
->flags
& ~IO_ZC_FLAGS_VALID
)
1063 if (zc
->flags
& IORING_SEND_ZC_REPORT_USAGE
) {
1064 io_notif_set_extended(notif
);
1065 io_notif_to_data(notif
)->zc_report
= true;
1069 if (zc
->flags
& IORING_RECVSEND_FIXED_BUF
) {
1070 unsigned idx
= READ_ONCE(sqe
->buf_index
);
1072 if (unlikely(idx
>= ctx
->nr_user_bufs
))
1074 idx
= array_index_nospec(idx
, ctx
->nr_user_bufs
);
1075 req
->imu
= READ_ONCE(ctx
->user_bufs
[idx
]);
1076 io_req_set_rsrc_node(notif
, ctx
, 0);
1079 if (req
->opcode
== IORING_OP_SEND_ZC
) {
1080 if (READ_ONCE(sqe
->__pad3
[0]))
1082 zc
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
1083 zc
->addr_len
= READ_ONCE(sqe
->addr_len
);
1085 if (unlikely(sqe
->addr2
|| sqe
->file_index
))
1087 if (unlikely(zc
->flags
& IORING_RECVSEND_FIXED_BUF
))
1091 zc
->buf
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1092 zc
->len
= READ_ONCE(sqe
->len
);
1093 zc
->msg_flags
= READ_ONCE(sqe
->msg_flags
) | MSG_NOSIGNAL
;
1094 if (zc
->msg_flags
& MSG_DONTWAIT
)
1095 req
->flags
|= REQ_F_NOWAIT
;
1097 #ifdef CONFIG_COMPAT
1098 if (req
->ctx
->compat
)
1099 zc
->msg_flags
|= MSG_CMSG_COMPAT
;
1104 static int io_sg_from_iter_iovec(struct sock
*sk
, struct sk_buff
*skb
,
1105 struct iov_iter
*from
, size_t length
)
1107 skb_zcopy_downgrade_managed(skb
);
1108 return __zerocopy_sg_from_iter(NULL
, sk
, skb
, from
, length
);
1111 static int io_sg_from_iter(struct sock
*sk
, struct sk_buff
*skb
,
1112 struct iov_iter
*from
, size_t length
)
1114 struct skb_shared_info
*shinfo
= skb_shinfo(skb
);
1115 int frag
= shinfo
->nr_frags
;
1117 struct bvec_iter bi
;
1119 unsigned long truesize
= 0;
1122 shinfo
->flags
|= SKBFL_MANAGED_FRAG_REFS
;
1123 else if (unlikely(!skb_zcopy_managed(skb
)))
1124 return __zerocopy_sg_from_iter(NULL
, sk
, skb
, from
, length
);
1126 bi
.bi_size
= min(from
->count
, length
);
1127 bi
.bi_bvec_done
= from
->iov_offset
;
1130 while (bi
.bi_size
&& frag
< MAX_SKB_FRAGS
) {
1131 struct bio_vec v
= mp_bvec_iter_bvec(from
->bvec
, bi
);
1134 truesize
+= PAGE_ALIGN(v
.bv_len
+ v
.bv_offset
);
1135 __skb_fill_page_desc_noacc(shinfo
, frag
++, v
.bv_page
,
1136 v
.bv_offset
, v
.bv_len
);
1137 bvec_iter_advance_single(from
->bvec
, &bi
, v
.bv_len
);
1142 shinfo
->nr_frags
= frag
;
1143 from
->bvec
+= bi
.bi_idx
;
1144 from
->nr_segs
-= bi
.bi_idx
;
1145 from
->count
-= copied
;
1146 from
->iov_offset
= bi
.bi_bvec_done
;
1148 skb
->data_len
+= copied
;
1150 skb
->truesize
+= truesize
;
1152 if (sk
&& sk
->sk_type
== SOCK_STREAM
) {
1153 sk_wmem_queued_add(sk
, truesize
);
1154 if (!skb_zcopy_pure(skb
))
1155 sk_mem_charge(sk
, truesize
);
1157 refcount_add(truesize
, &skb
->sk
->sk_wmem_alloc
);
1162 int io_send_zc(struct io_kiocb
*req
, unsigned int issue_flags
)
1164 struct sockaddr_storage __address
;
1165 struct io_sr_msg
*zc
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1167 struct socket
*sock
;
1169 int ret
, min_ret
= 0;
1171 sock
= sock_from_file(req
->file
);
1172 if (unlikely(!sock
))
1174 if (!test_bit(SOCK_SUPPORT_ZC
, &sock
->flags
))
1177 msg
.msg_name
= NULL
;
1178 msg
.msg_control
= NULL
;
1179 msg
.msg_controllen
= 0;
1180 msg
.msg_namelen
= 0;
1183 if (req_has_async_data(req
)) {
1184 struct io_async_msghdr
*io
= req
->async_data
;
1186 msg
.msg_name
= &io
->addr
;
1188 ret
= move_addr_to_kernel(zc
->addr
, zc
->addr_len
, &__address
);
1189 if (unlikely(ret
< 0))
1191 msg
.msg_name
= (struct sockaddr
*)&__address
;
1193 msg
.msg_namelen
= zc
->addr_len
;
1196 if (!(req
->flags
& REQ_F_POLLED
) &&
1197 (zc
->flags
& IORING_RECVSEND_POLL_FIRST
))
1198 return io_setup_async_addr(req
, &__address
, issue_flags
);
1200 if (zc
->flags
& IORING_RECVSEND_FIXED_BUF
) {
1201 ret
= io_import_fixed(ITER_SOURCE
, &msg
.msg_iter
, req
->imu
,
1202 (u64
)(uintptr_t)zc
->buf
, zc
->len
);
1205 msg
.sg_from_iter
= io_sg_from_iter
;
1207 io_notif_set_extended(zc
->notif
);
1208 ret
= import_ubuf(ITER_SOURCE
, zc
->buf
, zc
->len
, &msg
.msg_iter
);
1211 ret
= io_notif_account_mem(zc
->notif
, zc
->len
);
1214 msg
.sg_from_iter
= io_sg_from_iter_iovec
;
1217 msg_flags
= zc
->msg_flags
| MSG_ZEROCOPY
;
1218 if (issue_flags
& IO_URING_F_NONBLOCK
)
1219 msg_flags
|= MSG_DONTWAIT
;
1220 if (msg_flags
& MSG_WAITALL
)
1221 min_ret
= iov_iter_count(&msg
.msg_iter
);
1222 msg_flags
&= ~MSG_INTERNAL_SENDMSG_FLAGS
;
1224 msg
.msg_flags
= msg_flags
;
1225 msg
.msg_ubuf
= &io_notif_to_data(zc
->notif
)->uarg
;
1226 ret
= sock_sendmsg(sock
, &msg
);
1228 if (unlikely(ret
< min_ret
)) {
1229 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1230 return io_setup_async_addr(req
, &__address
, issue_flags
);
1232 if (ret
> 0 && io_net_retry(sock
, msg
.msg_flags
)) {
1236 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
1237 return io_setup_async_addr(req
, &__address
, issue_flags
);
1239 if (ret
== -ERESTARTSYS
)
1246 else if (zc
->done_io
)
1250 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1251 * flushing notif to io_send_zc_cleanup()
1253 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
1254 io_notif_flush(zc
->notif
);
1255 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
1257 io_req_set_res(req
, ret
, IORING_CQE_F_MORE
);
1261 int io_sendmsg_zc(struct io_kiocb
*req
, unsigned int issue_flags
)
1263 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1264 struct io_async_msghdr iomsg
, *kmsg
;
1265 struct socket
*sock
;
1267 int ret
, min_ret
= 0;
1269 io_notif_set_extended(sr
->notif
);
1271 sock
= sock_from_file(req
->file
);
1272 if (unlikely(!sock
))
1274 if (!test_bit(SOCK_SUPPORT_ZC
, &sock
->flags
))
1277 if (req_has_async_data(req
)) {
1278 kmsg
= req
->async_data
;
1280 ret
= io_sendmsg_copy_hdr(req
, &iomsg
);
1286 if (!(req
->flags
& REQ_F_POLLED
) &&
1287 (sr
->flags
& IORING_RECVSEND_POLL_FIRST
))
1288 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1290 flags
= sr
->msg_flags
| MSG_ZEROCOPY
;
1291 if (issue_flags
& IO_URING_F_NONBLOCK
)
1292 flags
|= MSG_DONTWAIT
;
1293 if (flags
& MSG_WAITALL
)
1294 min_ret
= iov_iter_count(&kmsg
->msg
.msg_iter
);
1296 kmsg
->msg
.msg_ubuf
= &io_notif_to_data(sr
->notif
)->uarg
;
1297 kmsg
->msg
.sg_from_iter
= io_sg_from_iter_iovec
;
1298 ret
= __sys_sendmsg_sock(sock
, &kmsg
->msg
, flags
);
1300 if (unlikely(ret
< min_ret
)) {
1301 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1302 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1304 if (ret
> 0 && io_net_retry(sock
, flags
)) {
1306 req
->flags
|= REQ_F_BL_NO_RECYCLE
;
1307 return io_setup_async_msg(req
, kmsg
, issue_flags
);
1309 if (ret
== -ERESTARTSYS
)
1313 /* fast path, check for non-NULL to avoid function call */
1314 if (kmsg
->free_iov
) {
1315 kfree(kmsg
->free_iov
);
1316 kmsg
->free_iov
= NULL
;
1319 io_netmsg_recycle(req
, issue_flags
);
1322 else if (sr
->done_io
)
1326 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1327 * flushing notif to io_send_zc_cleanup()
1329 if (!(issue_flags
& IO_URING_F_UNLOCKED
)) {
1330 io_notif_flush(sr
->notif
);
1331 req
->flags
&= ~REQ_F_NEED_CLEANUP
;
1333 io_req_set_res(req
, ret
, IORING_CQE_F_MORE
);
1337 void io_sendrecv_fail(struct io_kiocb
*req
)
1339 struct io_sr_msg
*sr
= io_kiocb_to_cmd(req
, struct io_sr_msg
);
1342 req
->cqe
.res
= sr
->done_io
;
1344 if ((req
->flags
& REQ_F_NEED_CLEANUP
) &&
1345 (req
->opcode
== IORING_OP_SEND_ZC
|| req
->opcode
== IORING_OP_SENDMSG_ZC
))
1346 req
->cqe
.flags
|= IORING_CQE_F_MORE
;
1349 int io_accept_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1351 struct io_accept
*accept
= io_kiocb_to_cmd(req
, struct io_accept
);
1354 if (sqe
->len
|| sqe
->buf_index
)
1357 accept
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1358 accept
->addr_len
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
1359 accept
->flags
= READ_ONCE(sqe
->accept_flags
);
1360 accept
->nofile
= rlimit(RLIMIT_NOFILE
);
1361 flags
= READ_ONCE(sqe
->ioprio
);
1362 if (flags
& ~IORING_ACCEPT_MULTISHOT
)
1365 accept
->file_slot
= READ_ONCE(sqe
->file_index
);
1366 if (accept
->file_slot
) {
1367 if (accept
->flags
& SOCK_CLOEXEC
)
1369 if (flags
& IORING_ACCEPT_MULTISHOT
&&
1370 accept
->file_slot
!= IORING_FILE_INDEX_ALLOC
)
1373 if (accept
->flags
& ~(SOCK_CLOEXEC
| SOCK_NONBLOCK
))
1375 if (SOCK_NONBLOCK
!= O_NONBLOCK
&& (accept
->flags
& SOCK_NONBLOCK
))
1376 accept
->flags
= (accept
->flags
& ~SOCK_NONBLOCK
) | O_NONBLOCK
;
1377 if (flags
& IORING_ACCEPT_MULTISHOT
)
1378 req
->flags
|= REQ_F_APOLL_MULTISHOT
;
1382 int io_accept(struct io_kiocb
*req
, unsigned int issue_flags
)
1384 struct io_accept
*accept
= io_kiocb_to_cmd(req
, struct io_accept
);
1385 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
1386 unsigned int file_flags
= force_nonblock
? O_NONBLOCK
: 0;
1387 bool fixed
= !!accept
->file_slot
;
1393 fd
= __get_unused_fd_flags(accept
->flags
, accept
->nofile
);
1394 if (unlikely(fd
< 0))
1397 file
= do_accept(req
->file
, file_flags
, accept
->addr
, accept
->addr_len
,
1402 ret
= PTR_ERR(file
);
1403 if (ret
== -EAGAIN
&& force_nonblock
) {
1405 * if it's multishot and polled, we don't need to
1406 * return EAGAIN to arm the poll infra since it
1407 * has already been done
1409 if (issue_flags
& IO_URING_F_MULTISHOT
)
1410 return IOU_ISSUE_SKIP_COMPLETE
;
1413 if (ret
== -ERESTARTSYS
)
1416 } else if (!fixed
) {
1417 fd_install(fd
, file
);
1420 ret
= io_fixed_fd_install(req
, issue_flags
, file
,
1424 if (!(req
->flags
& REQ_F_APOLL_MULTISHOT
)) {
1425 io_req_set_res(req
, ret
, 0);
1431 if (io_fill_cqe_req_aux(req
, issue_flags
& IO_URING_F_COMPLETE_DEFER
,
1432 ret
, IORING_CQE_F_MORE
))
1435 io_req_set_res(req
, ret
, 0);
1436 return IOU_STOP_MULTISHOT
;
1439 int io_socket_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1441 struct io_socket
*sock
= io_kiocb_to_cmd(req
, struct io_socket
);
1443 if (sqe
->addr
|| sqe
->rw_flags
|| sqe
->buf_index
)
1446 sock
->domain
= READ_ONCE(sqe
->fd
);
1447 sock
->type
= READ_ONCE(sqe
->off
);
1448 sock
->protocol
= READ_ONCE(sqe
->len
);
1449 sock
->file_slot
= READ_ONCE(sqe
->file_index
);
1450 sock
->nofile
= rlimit(RLIMIT_NOFILE
);
1452 sock
->flags
= sock
->type
& ~SOCK_TYPE_MASK
;
1453 if (sock
->file_slot
&& (sock
->flags
& SOCK_CLOEXEC
))
1455 if (sock
->flags
& ~(SOCK_CLOEXEC
| SOCK_NONBLOCK
))
1460 int io_socket(struct io_kiocb
*req
, unsigned int issue_flags
)
1462 struct io_socket
*sock
= io_kiocb_to_cmd(req
, struct io_socket
);
1463 bool fixed
= !!sock
->file_slot
;
1468 fd
= __get_unused_fd_flags(sock
->flags
, sock
->nofile
);
1469 if (unlikely(fd
< 0))
1472 file
= __sys_socket_file(sock
->domain
, sock
->type
, sock
->protocol
);
1476 ret
= PTR_ERR(file
);
1477 if (ret
== -EAGAIN
&& (issue_flags
& IO_URING_F_NONBLOCK
))
1479 if (ret
== -ERESTARTSYS
)
1482 } else if (!fixed
) {
1483 fd_install(fd
, file
);
1486 ret
= io_fixed_fd_install(req
, issue_flags
, file
,
1489 io_req_set_res(req
, ret
, 0);
1493 int io_connect_prep_async(struct io_kiocb
*req
)
1495 struct io_async_connect
*io
= req
->async_data
;
1496 struct io_connect
*conn
= io_kiocb_to_cmd(req
, struct io_connect
);
1498 return move_addr_to_kernel(conn
->addr
, conn
->addr_len
, &io
->address
);
1501 int io_connect_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
1503 struct io_connect
*conn
= io_kiocb_to_cmd(req
, struct io_connect
);
1505 if (sqe
->len
|| sqe
->buf_index
|| sqe
->rw_flags
|| sqe
->splice_fd_in
)
1508 conn
->addr
= u64_to_user_ptr(READ_ONCE(sqe
->addr
));
1509 conn
->addr_len
= READ_ONCE(sqe
->addr2
);
1510 conn
->in_progress
= conn
->seen_econnaborted
= false;
1514 int io_connect(struct io_kiocb
*req
, unsigned int issue_flags
)
1516 struct io_connect
*connect
= io_kiocb_to_cmd(req
, struct io_connect
);
1517 struct io_async_connect __io
, *io
;
1518 unsigned file_flags
;
1520 bool force_nonblock
= issue_flags
& IO_URING_F_NONBLOCK
;
1522 if (req_has_async_data(req
)) {
1523 io
= req
->async_data
;
1525 ret
= move_addr_to_kernel(connect
->addr
,
1533 file_flags
= force_nonblock
? O_NONBLOCK
: 0;
1535 ret
= __sys_connect_file(req
->file
, &io
->address
,
1536 connect
->addr_len
, file_flags
);
1537 if ((ret
== -EAGAIN
|| ret
== -EINPROGRESS
|| ret
== -ECONNABORTED
)
1538 && force_nonblock
) {
1539 if (ret
== -EINPROGRESS
) {
1540 connect
->in_progress
= true;
1541 } else if (ret
== -ECONNABORTED
) {
1542 if (connect
->seen_econnaborted
)
1544 connect
->seen_econnaborted
= true;
1546 if (req_has_async_data(req
))
1548 if (io_alloc_async_data(req
)) {
1552 memcpy(req
->async_data
, &__io
, sizeof(__io
));
1555 if (connect
->in_progress
) {
1557 * At least bluetooth will return -EBADFD on a re-connect
1558 * attempt, and it's (supposedly) also valid to get -EISCONN
1559 * which means the previous result is good. For both of these,
1560 * grab the sock_error() and use that for the completion.
1562 if (ret
== -EBADFD
|| ret
== -EISCONN
)
1563 ret
= sock_error(sock_from_file(req
->file
)->sk
);
1565 if (ret
== -ERESTARTSYS
)
1570 io_req_set_res(req
, ret
, 0);
1574 void io_netmsg_cache_free(struct io_cache_entry
*entry
)
1576 kfree(container_of(entry
, struct io_async_msghdr
, cache
));