1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
5 #include <linux/file.h>
7 #include <linux/slab.h>
8 #include <linux/namei.h>
9 #include <linux/nospec.h>
10 #include <linux/io_uring.h>
12 #include <uapi/linux/io_uring.h>
30 #define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
31 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
32 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)
35 * Returns true if the request matches the criteria outlined by 'cd'.
37 bool io_cancel_req_match(struct io_kiocb
*req
, struct io_cancel_data
*cd
)
39 bool match_user_data
= cd
->flags
& IORING_ASYNC_CANCEL_USERDATA
;
41 if (req
->ctx
!= cd
->ctx
)
44 if (!(cd
->flags
& (IORING_ASYNC_CANCEL_FD
| IORING_ASYNC_CANCEL_OP
)))
45 match_user_data
= true;
47 if (cd
->flags
& IORING_ASYNC_CANCEL_ANY
)
49 if (cd
->flags
& IORING_ASYNC_CANCEL_FD
) {
50 if (req
->file
!= cd
->file
)
53 if (cd
->flags
& IORING_ASYNC_CANCEL_OP
) {
54 if (req
->opcode
!= cd
->opcode
)
57 if (match_user_data
&& req
->cqe
.user_data
!= cd
->data
)
59 if (cd
->flags
& IORING_ASYNC_CANCEL_ALL
) {
61 if (cd
->seq
== req
->work
.cancel_seq
)
63 req
->work
.cancel_seq
= cd
->seq
;
69 static bool io_cancel_cb(struct io_wq_work
*work
, void *data
)
71 struct io_kiocb
*req
= container_of(work
, struct io_kiocb
, work
);
72 struct io_cancel_data
*cd
= data
;
74 return io_cancel_req_match(req
, cd
);
77 static int io_async_cancel_one(struct io_uring_task
*tctx
,
78 struct io_cancel_data
*cd
)
80 enum io_wq_cancel cancel_ret
;
84 if (!tctx
|| !tctx
->io_wq
)
87 all
= cd
->flags
& (IORING_ASYNC_CANCEL_ALL
|IORING_ASYNC_CANCEL_ANY
);
88 cancel_ret
= io_wq_cancel_cb(tctx
->io_wq
, io_cancel_cb
, cd
, all
);
93 case IO_WQ_CANCEL_RUNNING
:
96 case IO_WQ_CANCEL_NOTFOUND
:
104 int io_try_cancel(struct io_uring_task
*tctx
, struct io_cancel_data
*cd
,
105 unsigned issue_flags
)
107 struct io_ring_ctx
*ctx
= cd
->ctx
;
110 WARN_ON_ONCE(!io_wq_current_is_worker() && tctx
!= current
->io_uring
);
112 ret
= io_async_cancel_one(tctx
, cd
);
114 * Fall-through even for -EALREADY, as we may have poll armed
115 * that need unarming.
120 ret
= io_poll_cancel(ctx
, cd
, issue_flags
);
124 ret
= io_waitid_cancel(ctx
, cd
, issue_flags
);
128 ret
= io_futex_cancel(ctx
, cd
, issue_flags
);
132 spin_lock(&ctx
->completion_lock
);
133 if (!(cd
->flags
& IORING_ASYNC_CANCEL_FD
))
134 ret
= io_timeout_cancel(ctx
, cd
);
135 spin_unlock(&ctx
->completion_lock
);
139 int io_async_cancel_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
141 struct io_cancel
*cancel
= io_kiocb_to_cmd(req
, struct io_cancel
);
143 if (unlikely(req
->flags
& REQ_F_BUFFER_SELECT
))
145 if (sqe
->off
|| sqe
->splice_fd_in
)
148 cancel
->addr
= READ_ONCE(sqe
->addr
);
149 cancel
->flags
= READ_ONCE(sqe
->cancel_flags
);
150 if (cancel
->flags
& ~CANCEL_FLAGS
)
152 if (cancel
->flags
& IORING_ASYNC_CANCEL_FD
) {
153 if (cancel
->flags
& IORING_ASYNC_CANCEL_ANY
)
155 cancel
->fd
= READ_ONCE(sqe
->fd
);
157 if (cancel
->flags
& IORING_ASYNC_CANCEL_OP
) {
158 if (cancel
->flags
& IORING_ASYNC_CANCEL_ANY
)
160 cancel
->opcode
= READ_ONCE(sqe
->len
);
166 static int __io_async_cancel(struct io_cancel_data
*cd
,
167 struct io_uring_task
*tctx
,
168 unsigned int issue_flags
)
170 bool all
= cd
->flags
& (IORING_ASYNC_CANCEL_ALL
|IORING_ASYNC_CANCEL_ANY
);
171 struct io_ring_ctx
*ctx
= cd
->ctx
;
172 struct io_tctx_node
*node
;
176 ret
= io_try_cancel(tctx
, cd
, issue_flags
);
184 /* slow path, try all io-wq's */
185 io_ring_submit_lock(ctx
, issue_flags
);
187 list_for_each_entry(node
, &ctx
->tctx_list
, ctx_node
) {
188 struct io_uring_task
*tctx
= node
->task
->io_uring
;
190 ret
= io_async_cancel_one(tctx
, cd
);
191 if (ret
!= -ENOENT
) {
197 io_ring_submit_unlock(ctx
, issue_flags
);
198 return all
? nr
: ret
;
201 int io_async_cancel(struct io_kiocb
*req
, unsigned int issue_flags
)
203 struct io_cancel
*cancel
= io_kiocb_to_cmd(req
, struct io_cancel
);
204 struct io_cancel_data cd
= {
206 .data
= cancel
->addr
,
207 .flags
= cancel
->flags
,
208 .opcode
= cancel
->opcode
,
209 .seq
= atomic_inc_return(&req
->ctx
->cancel_seq
),
211 struct io_uring_task
*tctx
= req
->task
->io_uring
;
214 if (cd
.flags
& IORING_ASYNC_CANCEL_FD
) {
215 if (req
->flags
& REQ_F_FIXED_FILE
||
216 cd
.flags
& IORING_ASYNC_CANCEL_FD_FIXED
) {
217 req
->flags
|= REQ_F_FIXED_FILE
;
218 req
->file
= io_file_get_fixed(req
, cancel
->fd
,
221 req
->file
= io_file_get_normal(req
, cancel
->fd
);
230 ret
= __io_async_cancel(&cd
, tctx
, issue_flags
);
234 io_req_set_res(req
, ret
, 0);
238 void init_hash_table(struct io_hash_table
*table
, unsigned size
)
242 for (i
= 0; i
< size
; i
++) {
243 spin_lock_init(&table
->hbs
[i
].lock
);
244 INIT_HLIST_HEAD(&table
->hbs
[i
].list
);
248 static int __io_sync_cancel(struct io_uring_task
*tctx
,
249 struct io_cancel_data
*cd
, int fd
)
251 struct io_ring_ctx
*ctx
= cd
->ctx
;
253 /* fixed must be grabbed every time since we drop the uring_lock */
254 if ((cd
->flags
& IORING_ASYNC_CANCEL_FD
) &&
255 (cd
->flags
& IORING_ASYNC_CANCEL_FD_FIXED
)) {
256 if (unlikely(fd
>= ctx
->nr_user_files
))
258 fd
= array_index_nospec(fd
, ctx
->nr_user_files
);
259 cd
->file
= io_file_from_index(&ctx
->file_table
, fd
);
264 return __io_async_cancel(cd
, tctx
, 0);
267 int io_sync_cancel(struct io_ring_ctx
*ctx
, void __user
*arg
)
268 __must_hold(&ctx
->uring_lock
)
270 struct io_cancel_data cd
= {
272 .seq
= atomic_inc_return(&ctx
->cancel_seq
),
274 ktime_t timeout
= KTIME_MAX
;
275 struct io_uring_sync_cancel_reg sc
;
276 struct file
*file
= NULL
;
280 if (copy_from_user(&sc
, arg
, sizeof(sc
)))
282 if (sc
.flags
& ~CANCEL_FLAGS
)
284 for (i
= 0; i
< ARRAY_SIZE(sc
.pad
); i
++)
287 for (i
= 0; i
< ARRAY_SIZE(sc
.pad2
); i
++)
293 cd
.opcode
= sc
.opcode
;
295 /* we can grab a normal file descriptor upfront */
296 if ((cd
.flags
& IORING_ASYNC_CANCEL_FD
) &&
297 !(cd
.flags
& IORING_ASYNC_CANCEL_FD_FIXED
)) {
304 ret
= __io_sync_cancel(current
->io_uring
, &cd
, sc
.fd
);
306 /* found something, done! */
307 if (ret
!= -EALREADY
)
310 if (sc
.timeout
.tv_sec
!= -1UL || sc
.timeout
.tv_nsec
!= -1UL) {
311 struct timespec64 ts
= {
312 .tv_sec
= sc
.timeout
.tv_sec
,
313 .tv_nsec
= sc
.timeout
.tv_nsec
316 timeout
= ktime_add_ns(timespec64_to_ktime(ts
), ktime_get_ns());
320 * Keep looking until we get -ENOENT. we'll get woken everytime
321 * every time a request completes and will retry the cancelation.
324 cd
.seq
= atomic_inc_return(&ctx
->cancel_seq
);
326 prepare_to_wait(&ctx
->cq_wait
, &wait
, TASK_INTERRUPTIBLE
);
328 ret
= __io_sync_cancel(current
->io_uring
, &cd
, sc
.fd
);
330 mutex_unlock(&ctx
->uring_lock
);
331 if (ret
!= -EALREADY
)
334 ret
= io_run_task_work_sig(ctx
);
337 ret
= schedule_hrtimeout(&timeout
, HRTIMER_MODE_ABS
);
342 mutex_lock(&ctx
->uring_lock
);
345 finish_wait(&ctx
->cq_wait
, &wait
);
346 mutex_lock(&ctx
->uring_lock
);
348 if (ret
== -ENOENT
|| ret
> 0)