1 // SPDX-License-Identifier: GPL-2.0
3 * Support for async notification of waitid
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/compat.h>
10 #include <linux/io_uring.h>
12 #include <uapi/linux/io_uring.h>
17 #include "../kernel/exit.h"
19 static void io_waitid_cb(struct io_kiocb
*req
, struct io_tw_state
*ts
);
21 #define IO_WAITID_CANCEL_FLAG BIT(31)
22 #define IO_WAITID_REF_MASK GENMASK(30, 0)
30 struct wait_queue_head
*head
;
31 struct siginfo __user
*infop
;
32 struct waitid_info info
;
35 static void io_waitid_free(struct io_kiocb
*req
)
37 struct io_waitid_async
*iwa
= req
->async_data
;
39 put_pid(iwa
->wo
.wo_pid
);
40 kfree(req
->async_data
);
41 req
->async_data
= NULL
;
42 req
->flags
&= ~REQ_F_ASYNC_DATA
;
46 static bool io_waitid_compat_copy_si(struct io_waitid
*iw
, int signo
)
48 struct compat_siginfo __user
*infop
;
51 infop
= (struct compat_siginfo __user
*) iw
->infop
;
53 if (!user_write_access_begin(infop
, sizeof(*infop
)))
56 unsafe_put_user(signo
, &infop
->si_signo
, Efault
);
57 unsafe_put_user(0, &infop
->si_errno
, Efault
);
58 unsafe_put_user(iw
->info
.cause
, &infop
->si_code
, Efault
);
59 unsafe_put_user(iw
->info
.pid
, &infop
->si_pid
, Efault
);
60 unsafe_put_user(iw
->info
.uid
, &infop
->si_uid
, Efault
);
61 unsafe_put_user(iw
->info
.status
, &infop
->si_status
, Efault
);
64 user_write_access_end();
72 static bool io_waitid_copy_si(struct io_kiocb
*req
, int signo
)
74 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
82 return io_waitid_compat_copy_si(iw
, signo
);
85 if (!user_write_access_begin(iw
->infop
, sizeof(*iw
->infop
)))
88 unsafe_put_user(signo
, &iw
->infop
->si_signo
, Efault
);
89 unsafe_put_user(0, &iw
->infop
->si_errno
, Efault
);
90 unsafe_put_user(iw
->info
.cause
, &iw
->infop
->si_code
, Efault
);
91 unsafe_put_user(iw
->info
.pid
, &iw
->infop
->si_pid
, Efault
);
92 unsafe_put_user(iw
->info
.uid
, &iw
->infop
->si_uid
, Efault
);
93 unsafe_put_user(iw
->info
.status
, &iw
->infop
->si_status
, Efault
);
96 user_write_access_end();
103 static int io_waitid_finish(struct io_kiocb
*req
, int ret
)
112 if (!io_waitid_copy_si(req
, signo
))
118 static void io_waitid_complete(struct io_kiocb
*req
, int ret
)
120 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
121 struct io_tw_state ts
= { .locked
= true };
123 /* anyone completing better be holding a reference */
124 WARN_ON_ONCE(!(atomic_read(&iw
->refs
) & IO_WAITID_REF_MASK
));
126 lockdep_assert_held(&req
->ctx
->uring_lock
);
129 * Did cancel find it meanwhile?
131 if (hlist_unhashed(&req
->hash_node
))
134 hlist_del_init(&req
->hash_node
);
136 ret
= io_waitid_finish(req
, ret
);
139 io_req_set_res(req
, ret
, 0);
140 io_req_task_complete(req
, &ts
);
143 static bool __io_waitid_cancel(struct io_ring_ctx
*ctx
, struct io_kiocb
*req
)
145 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
146 struct io_waitid_async
*iwa
= req
->async_data
;
149 * Mark us canceled regardless of ownership. This will prevent a
150 * potential retry from a spurious wakeup.
152 atomic_or(IO_WAITID_CANCEL_FLAG
, &iw
->refs
);
154 /* claim ownership */
155 if (atomic_fetch_inc(&iw
->refs
) & IO_WAITID_REF_MASK
)
158 spin_lock_irq(&iw
->head
->lock
);
159 list_del_init(&iwa
->wo
.child_wait
.entry
);
160 spin_unlock_irq(&iw
->head
->lock
);
161 io_waitid_complete(req
, -ECANCELED
);
165 int io_waitid_cancel(struct io_ring_ctx
*ctx
, struct io_cancel_data
*cd
,
166 unsigned int issue_flags
)
168 struct hlist_node
*tmp
;
169 struct io_kiocb
*req
;
172 if (cd
->flags
& (IORING_ASYNC_CANCEL_FD
|IORING_ASYNC_CANCEL_FD_FIXED
))
175 io_ring_submit_lock(ctx
, issue_flags
);
176 hlist_for_each_entry_safe(req
, tmp
, &ctx
->waitid_list
, hash_node
) {
177 if (req
->cqe
.user_data
!= cd
->data
&&
178 !(cd
->flags
& IORING_ASYNC_CANCEL_ANY
))
180 if (__io_waitid_cancel(ctx
, req
))
182 if (!(cd
->flags
& IORING_ASYNC_CANCEL_ALL
))
185 io_ring_submit_unlock(ctx
, issue_flags
);
193 bool io_waitid_remove_all(struct io_ring_ctx
*ctx
, struct task_struct
*task
,
196 struct hlist_node
*tmp
;
197 struct io_kiocb
*req
;
200 lockdep_assert_held(&ctx
->uring_lock
);
202 hlist_for_each_entry_safe(req
, tmp
, &ctx
->waitid_list
, hash_node
) {
203 if (!io_match_task_safe(req
, task
, cancel_all
))
205 __io_waitid_cancel(ctx
, req
);
212 static inline bool io_waitid_drop_issue_ref(struct io_kiocb
*req
)
214 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
215 struct io_waitid_async
*iwa
= req
->async_data
;
217 if (!atomic_sub_return(1, &iw
->refs
))
221 * Wakeup triggered, racing with us. It was prevented from
222 * completing because of that, queue up the tw to do that.
224 req
->io_task_work
.func
= io_waitid_cb
;
225 io_req_task_work_add(req
);
226 remove_wait_queue(iw
->head
, &iwa
->wo
.child_wait
);
230 static void io_waitid_cb(struct io_kiocb
*req
, struct io_tw_state
*ts
)
232 struct io_waitid_async
*iwa
= req
->async_data
;
233 struct io_ring_ctx
*ctx
= req
->ctx
;
238 ret
= __do_wait(&iwa
->wo
);
241 * If we get -ERESTARTSYS here, we need to re-arm and check again
242 * to ensure we get another callback. If the retry works, then we can
243 * just remove ourselves from the waitqueue again and finish the
246 if (unlikely(ret
== -ERESTARTSYS
)) {
247 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
249 /* Don't retry if cancel found it meanwhile */
251 if (!(atomic_read(&iw
->refs
) & IO_WAITID_CANCEL_FLAG
)) {
252 iw
->head
= ¤t
->signal
->wait_chldexit
;
253 add_wait_queue(iw
->head
, &iwa
->wo
.child_wait
);
254 ret
= __do_wait(&iwa
->wo
);
255 if (ret
== -ERESTARTSYS
) {
256 /* retry armed, drop our ref */
257 io_waitid_drop_issue_ref(req
);
261 remove_wait_queue(iw
->head
, &iwa
->wo
.child_wait
);
265 io_waitid_complete(req
, ret
);
268 static int io_waitid_wait(struct wait_queue_entry
*wait
, unsigned mode
,
271 struct wait_opts
*wo
= container_of(wait
, struct wait_opts
, child_wait
);
272 struct io_waitid_async
*iwa
= container_of(wo
, struct io_waitid_async
, wo
);
273 struct io_kiocb
*req
= iwa
->req
;
274 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
275 struct task_struct
*p
= key
;
277 if (!pid_child_should_wake(wo
, p
))
280 /* cancel is in progress */
281 if (atomic_fetch_inc(&iw
->refs
) & IO_WAITID_REF_MASK
)
284 req
->io_task_work
.func
= io_waitid_cb
;
285 io_req_task_work_add(req
);
286 list_del_init(&wait
->entry
);
290 int io_waitid_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
292 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
294 if (sqe
->addr
|| sqe
->buf_index
|| sqe
->addr3
|| sqe
->waitid_flags
)
297 iw
->which
= READ_ONCE(sqe
->len
);
298 iw
->upid
= READ_ONCE(sqe
->fd
);
299 iw
->options
= READ_ONCE(sqe
->file_index
);
300 iw
->infop
= u64_to_user_ptr(READ_ONCE(sqe
->addr2
));
304 int io_waitid(struct io_kiocb
*req
, unsigned int issue_flags
)
306 struct io_waitid
*iw
= io_kiocb_to_cmd(req
, struct io_waitid
);
307 struct io_ring_ctx
*ctx
= req
->ctx
;
308 struct io_waitid_async
*iwa
;
311 if (io_alloc_async_data(req
))
314 iwa
= req
->async_data
;
317 ret
= kernel_waitid_prepare(&iwa
->wo
, iw
->which
, iw
->upid
, &iw
->info
,
323 * Mark the request as busy upfront, in case we're racing with the
324 * wakeup. If we are, then we'll notice when we drop this initial
325 * reference again after arming.
327 atomic_set(&iw
->refs
, 1);
330 * Cancel must hold the ctx lock, so there's no risk of cancelation
331 * finding us until a) we remain on the list, and b) the lock is
332 * dropped. We only need to worry about racing with the wakeup
335 io_ring_submit_lock(ctx
, issue_flags
);
336 hlist_add_head(&req
->hash_node
, &ctx
->waitid_list
);
338 init_waitqueue_func_entry(&iwa
->wo
.child_wait
, io_waitid_wait
);
339 iwa
->wo
.child_wait
.private = req
->task
;
340 iw
->head
= ¤t
->signal
->wait_chldexit
;
341 add_wait_queue(iw
->head
, &iwa
->wo
.child_wait
);
343 ret
= __do_wait(&iwa
->wo
);
344 if (ret
== -ERESTARTSYS
) {
346 * Nobody else grabbed a reference, it'll complete when we get
347 * a waitqueue callback, or if someone cancels it.
349 if (!io_waitid_drop_issue_ref(req
)) {
350 io_ring_submit_unlock(ctx
, issue_flags
);
351 return IOU_ISSUE_SKIP_COMPLETE
;
355 * Wakeup triggered, racing with us. It was prevented from
356 * completing because of that, queue up the tw to do that.
358 io_ring_submit_unlock(ctx
, issue_flags
);
359 return IOU_ISSUE_SKIP_COMPLETE
;
362 hlist_del_init(&req
->hash_node
);
363 remove_wait_queue(iw
->head
, &iwa
->wo
.child_wait
);
364 ret
= io_waitid_finish(req
, ret
);
366 io_ring_submit_unlock(ctx
, issue_flags
);
370 io_req_set_res(req
, ret
, 0);