1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
5 #include <linux/file.h>
7 #include <linux/slab.h>
8 #include <linux/namei.h>
9 #include <linux/nospec.h>
10 #include <linux/io_uring.h>
12 #include <uapi/linux/io_uring.h>
28 #define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
29 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
30 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)
33 * Returns true if the request matches the criteria outlined by 'cd'.
35 bool io_cancel_req_match(struct io_kiocb
*req
, struct io_cancel_data
*cd
)
37 bool match_user_data
= cd
->flags
& IORING_ASYNC_CANCEL_USERDATA
;
39 if (req
->ctx
!= cd
->ctx
)
42 if (!(cd
->flags
& (IORING_ASYNC_CANCEL_FD
| IORING_ASYNC_CANCEL_OP
)))
43 match_user_data
= true;
45 if (cd
->flags
& IORING_ASYNC_CANCEL_ANY
)
47 if (cd
->flags
& IORING_ASYNC_CANCEL_FD
) {
48 if (req
->file
!= cd
->file
)
51 if (cd
->flags
& IORING_ASYNC_CANCEL_OP
) {
52 if (req
->opcode
!= cd
->opcode
)
55 if (match_user_data
&& req
->cqe
.user_data
!= cd
->data
)
57 if (cd
->flags
& IORING_ASYNC_CANCEL_ALL
) {
59 if (cd
->seq
== req
->work
.cancel_seq
)
61 req
->work
.cancel_seq
= cd
->seq
;
67 static bool io_cancel_cb(struct io_wq_work
*work
, void *data
)
69 struct io_kiocb
*req
= container_of(work
, struct io_kiocb
, work
);
70 struct io_cancel_data
*cd
= data
;
72 return io_cancel_req_match(req
, cd
);
75 static int io_async_cancel_one(struct io_uring_task
*tctx
,
76 struct io_cancel_data
*cd
)
78 enum io_wq_cancel cancel_ret
;
82 if (!tctx
|| !tctx
->io_wq
)
85 all
= cd
->flags
& (IORING_ASYNC_CANCEL_ALL
|IORING_ASYNC_CANCEL_ANY
);
86 cancel_ret
= io_wq_cancel_cb(tctx
->io_wq
, io_cancel_cb
, cd
, all
);
91 case IO_WQ_CANCEL_RUNNING
:
94 case IO_WQ_CANCEL_NOTFOUND
:
102 int io_try_cancel(struct io_uring_task
*tctx
, struct io_cancel_data
*cd
,
103 unsigned issue_flags
)
105 struct io_ring_ctx
*ctx
= cd
->ctx
;
108 WARN_ON_ONCE(!io_wq_current_is_worker() && tctx
!= current
->io_uring
);
110 ret
= io_async_cancel_one(tctx
, cd
);
112 * Fall-through even for -EALREADY, as we may have poll armed
113 * that need unarming.
118 ret
= io_poll_cancel(ctx
, cd
, issue_flags
);
122 spin_lock(&ctx
->completion_lock
);
123 if (!(cd
->flags
& IORING_ASYNC_CANCEL_FD
))
124 ret
= io_timeout_cancel(ctx
, cd
);
125 spin_unlock(&ctx
->completion_lock
);
129 int io_async_cancel_prep(struct io_kiocb
*req
, const struct io_uring_sqe
*sqe
)
131 struct io_cancel
*cancel
= io_kiocb_to_cmd(req
, struct io_cancel
);
133 if (unlikely(req
->flags
& REQ_F_BUFFER_SELECT
))
135 if (sqe
->off
|| sqe
->splice_fd_in
)
138 cancel
->addr
= READ_ONCE(sqe
->addr
);
139 cancel
->flags
= READ_ONCE(sqe
->cancel_flags
);
140 if (cancel
->flags
& ~CANCEL_FLAGS
)
142 if (cancel
->flags
& IORING_ASYNC_CANCEL_FD
) {
143 if (cancel
->flags
& IORING_ASYNC_CANCEL_ANY
)
145 cancel
->fd
= READ_ONCE(sqe
->fd
);
147 if (cancel
->flags
& IORING_ASYNC_CANCEL_OP
) {
148 if (cancel
->flags
& IORING_ASYNC_CANCEL_ANY
)
150 cancel
->opcode
= READ_ONCE(sqe
->len
);
156 static int __io_async_cancel(struct io_cancel_data
*cd
,
157 struct io_uring_task
*tctx
,
158 unsigned int issue_flags
)
160 bool all
= cd
->flags
& (IORING_ASYNC_CANCEL_ALL
|IORING_ASYNC_CANCEL_ANY
);
161 struct io_ring_ctx
*ctx
= cd
->ctx
;
162 struct io_tctx_node
*node
;
166 ret
= io_try_cancel(tctx
, cd
, issue_flags
);
174 /* slow path, try all io-wq's */
175 io_ring_submit_lock(ctx
, issue_flags
);
177 list_for_each_entry(node
, &ctx
->tctx_list
, ctx_node
) {
178 struct io_uring_task
*tctx
= node
->task
->io_uring
;
180 ret
= io_async_cancel_one(tctx
, cd
);
181 if (ret
!= -ENOENT
) {
187 io_ring_submit_unlock(ctx
, issue_flags
);
188 return all
? nr
: ret
;
191 int io_async_cancel(struct io_kiocb
*req
, unsigned int issue_flags
)
193 struct io_cancel
*cancel
= io_kiocb_to_cmd(req
, struct io_cancel
);
194 struct io_cancel_data cd
= {
196 .data
= cancel
->addr
,
197 .flags
= cancel
->flags
,
198 .opcode
= cancel
->opcode
,
199 .seq
= atomic_inc_return(&req
->ctx
->cancel_seq
),
201 struct io_uring_task
*tctx
= req
->task
->io_uring
;
204 if (cd
.flags
& IORING_ASYNC_CANCEL_FD
) {
205 if (req
->flags
& REQ_F_FIXED_FILE
||
206 cd
.flags
& IORING_ASYNC_CANCEL_FD_FIXED
) {
207 req
->flags
|= REQ_F_FIXED_FILE
;
208 req
->file
= io_file_get_fixed(req
, cancel
->fd
,
211 req
->file
= io_file_get_normal(req
, cancel
->fd
);
220 ret
= __io_async_cancel(&cd
, tctx
, issue_flags
);
224 io_req_set_res(req
, ret
, 0);
228 void init_hash_table(struct io_hash_table
*table
, unsigned size
)
232 for (i
= 0; i
< size
; i
++) {
233 spin_lock_init(&table
->hbs
[i
].lock
);
234 INIT_HLIST_HEAD(&table
->hbs
[i
].list
);
238 static int __io_sync_cancel(struct io_uring_task
*tctx
,
239 struct io_cancel_data
*cd
, int fd
)
241 struct io_ring_ctx
*ctx
= cd
->ctx
;
243 /* fixed must be grabbed every time since we drop the uring_lock */
244 if ((cd
->flags
& IORING_ASYNC_CANCEL_FD
) &&
245 (cd
->flags
& IORING_ASYNC_CANCEL_FD_FIXED
)) {
246 if (unlikely(fd
>= ctx
->nr_user_files
))
248 fd
= array_index_nospec(fd
, ctx
->nr_user_files
);
249 cd
->file
= io_file_from_index(&ctx
->file_table
, fd
);
254 return __io_async_cancel(cd
, tctx
, 0);
257 int io_sync_cancel(struct io_ring_ctx
*ctx
, void __user
*arg
)
258 __must_hold(&ctx
->uring_lock
)
260 struct io_cancel_data cd
= {
262 .seq
= atomic_inc_return(&ctx
->cancel_seq
),
264 ktime_t timeout
= KTIME_MAX
;
265 struct io_uring_sync_cancel_reg sc
;
270 if (copy_from_user(&sc
, arg
, sizeof(sc
)))
272 if (sc
.flags
& ~CANCEL_FLAGS
)
274 for (i
= 0; i
< ARRAY_SIZE(sc
.pad
); i
++)
277 for (i
= 0; i
< ARRAY_SIZE(sc
.pad2
); i
++)
283 cd
.opcode
= sc
.opcode
;
285 /* we can grab a normal file descriptor upfront */
286 if ((cd
.flags
& IORING_ASYNC_CANCEL_FD
) &&
287 !(cd
.flags
& IORING_ASYNC_CANCEL_FD_FIXED
)) {
294 ret
= __io_sync_cancel(current
->io_uring
, &cd
, sc
.fd
);
296 /* found something, done! */
297 if (ret
!= -EALREADY
)
300 if (sc
.timeout
.tv_sec
!= -1UL || sc
.timeout
.tv_nsec
!= -1UL) {
301 struct timespec64 ts
= {
302 .tv_sec
= sc
.timeout
.tv_sec
,
303 .tv_nsec
= sc
.timeout
.tv_nsec
306 timeout
= ktime_add_ns(timespec64_to_ktime(ts
), ktime_get_ns());
310 * Keep looking until we get -ENOENT. we'll get woken everytime
311 * every time a request completes and will retry the cancelation.
314 cd
.seq
= atomic_inc_return(&ctx
->cancel_seq
);
316 prepare_to_wait(&ctx
->cq_wait
, &wait
, TASK_INTERRUPTIBLE
);
318 ret
= __io_sync_cancel(current
->io_uring
, &cd
, sc
.fd
);
320 mutex_unlock(&ctx
->uring_lock
);
321 if (ret
!= -EALREADY
)
324 ret
= io_run_task_work_sig(ctx
);
327 ret
= schedule_hrtimeout(&timeout
, HRTIMER_MODE_ABS
);
332 mutex_lock(&ctx
->uring_lock
);
335 finish_wait(&ctx
->cq_wait
, &wait
);
336 mutex_lock(&ctx
->uring_lock
);
338 if (ret
== -ENOENT
|| ret
> 0)