1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
6 #include <linux/slab.h>
7 #include <linux/nospec.h>
8 #include <linux/io_uring.h>
10 #include <uapi/linux/io_uring.h>
14 #include "filetable.h"
16 static int io_file_bitmap_get(struct io_ring_ctx
*ctx
)
18 struct io_file_table
*table
= &ctx
->file_table
;
19 unsigned long nr
= ctx
->file_alloc_end
;
26 ret
= find_next_zero_bit(table
->bitmap
, nr
, table
->alloc_hint
);
30 if (table
->alloc_hint
== ctx
->file_alloc_start
)
32 nr
= table
->alloc_hint
;
33 table
->alloc_hint
= ctx
->file_alloc_start
;
39 bool io_alloc_file_tables(struct io_file_table
*table
, unsigned nr_files
)
41 table
->files
= kvcalloc(nr_files
, sizeof(table
->files
[0]),
43 if (unlikely(!table
->files
))
46 table
->bitmap
= bitmap_zalloc(nr_files
, GFP_KERNEL_ACCOUNT
);
47 if (unlikely(!table
->bitmap
)) {
55 void io_free_file_tables(struct io_file_table
*table
)
58 bitmap_free(table
->bitmap
);
63 static int io_install_fixed_file(struct io_ring_ctx
*ctx
, struct file
*file
,
65 __must_hold(&req
->ctx
->uring_lock
)
67 struct io_fixed_file
*file_slot
;
70 if (io_is_uring_fops(file
))
74 if (slot_index
>= ctx
->nr_user_files
)
77 slot_index
= array_index_nospec(slot_index
, ctx
->nr_user_files
);
78 file_slot
= io_fixed_file_slot(&ctx
->file_table
, slot_index
);
80 if (file_slot
->file_ptr
) {
81 ret
= io_queue_rsrc_removal(ctx
->file_data
, slot_index
,
82 io_slot_file(file_slot
));
86 file_slot
->file_ptr
= 0;
87 io_file_bitmap_clear(&ctx
->file_table
, slot_index
);
90 ret
= io_scm_file_account(ctx
, file
);
92 *io_get_tag_slot(ctx
->file_data
, slot_index
) = 0;
93 io_fixed_file_set(file_slot
, file
);
94 io_file_bitmap_set(&ctx
->file_table
, slot_index
);
99 int __io_fixed_fd_install(struct io_ring_ctx
*ctx
, struct file
*file
,
100 unsigned int file_slot
)
102 bool alloc_slot
= file_slot
== IORING_FILE_INDEX_ALLOC
;
106 ret
= io_file_bitmap_get(ctx
);
107 if (unlikely(ret
< 0))
114 ret
= io_install_fixed_file(ctx
, file
, file_slot
);
115 if (!ret
&& alloc_slot
)
120 * Note when io_fixed_fd_install() returns error value, it will ensure
121 * fput() is called correspondingly.
123 int io_fixed_fd_install(struct io_kiocb
*req
, unsigned int issue_flags
,
124 struct file
*file
, unsigned int file_slot
)
126 struct io_ring_ctx
*ctx
= req
->ctx
;
129 io_ring_submit_lock(ctx
, issue_flags
);
130 ret
= __io_fixed_fd_install(ctx
, file
, file_slot
);
131 io_ring_submit_unlock(ctx
, issue_flags
);
133 if (unlikely(ret
< 0))
138 int io_fixed_fd_remove(struct io_ring_ctx
*ctx
, unsigned int offset
)
140 struct io_fixed_file
*file_slot
;
143 if (unlikely(!ctx
->file_data
))
145 if (offset
>= ctx
->nr_user_files
)
148 offset
= array_index_nospec(offset
, ctx
->nr_user_files
);
149 file_slot
= io_fixed_file_slot(&ctx
->file_table
, offset
);
150 if (!file_slot
->file_ptr
)
153 ret
= io_queue_rsrc_removal(ctx
->file_data
, offset
,
154 io_slot_file(file_slot
));
158 file_slot
->file_ptr
= 0;
159 io_file_bitmap_clear(&ctx
->file_table
, offset
);
163 int io_register_file_alloc_range(struct io_ring_ctx
*ctx
,
164 struct io_uring_file_index_range __user
*arg
)
166 struct io_uring_file_index_range range
;
169 if (copy_from_user(&range
, arg
, sizeof(range
)))
171 if (check_add_overflow(range
.off
, range
.len
, &end
))
173 if (range
.resv
|| end
> ctx
->nr_user_files
)
176 io_file_table_set_alloc_range(ctx
, range
.off
, range
.len
);