[thirdparty/kernel/stable.git] / io_uring / filetable.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/io_uring.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "rsrc.h"
#include "filetable.h"

static int io_file_bitmap_get(struct io_ring_ctx *ctx)
{
	struct io_file_table *table = &ctx->file_table;
	unsigned long nr = ctx->file_alloc_end;
	int ret;

	if (!table->bitmap)
		return -ENFILE;

	do {
		ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
		if (ret != nr)
			return ret;

		if (table->alloc_hint == ctx->file_alloc_start)
			break;
		nr = table->alloc_hint;
		table->alloc_hint = ctx->file_alloc_start;
	} while (1);

	return -ENFILE;
}

bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
{
	table->files = kvcalloc(nr_files, sizeof(table->files[0]),
				GFP_KERNEL_ACCOUNT);
	if (unlikely(!table->files))
		return false;

	table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
	if (unlikely(!table->bitmap)) {
		kvfree(table->files);
		return false;
	}

	return true;
}

void io_free_file_tables(struct io_file_table *table)
{
	kvfree(table->files);
	bitmap_free(table->bitmap);
	table->files = NULL;
	table->bitmap = NULL;
}

static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
				 u32 slot_index)
	__must_hold(&req->ctx->uring_lock)
{
	struct io_fixed_file *file_slot;
	int ret;

	if (io_is_uring_fops(file))
		return -EBADF;
	if (!ctx->file_data)
		return -ENXIO;
	if (slot_index >= ctx->nr_user_files)
		return -EINVAL;

	slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
	file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);

	if (file_slot->file_ptr) {
		ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
					    io_slot_file(file_slot));
		if (ret)
			return ret;

		file_slot->file_ptr = 0;
		io_file_bitmap_clear(&ctx->file_table, slot_index);
	}

	*io_get_tag_slot(ctx->file_data, slot_index) = 0;
	io_fixed_file_set(file_slot, file);
	io_file_bitmap_set(&ctx->file_table, slot_index);
	return 0;
}

int __io_fixed_fd_install(struct io_ring_ctx *ctx, struct file *file,
			  unsigned int file_slot)
{
	bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
	int ret;

	if (alloc_slot) {
		ret = io_file_bitmap_get(ctx);
		if (unlikely(ret < 0))
			return ret;
		file_slot = ret;
	} else {
		file_slot--;
	}

	ret = io_install_fixed_file(ctx, file, file_slot);
	if (!ret && alloc_slot)
		ret = file_slot;
	return ret;
}
/*
 * Note when io_fixed_fd_install() returns error value, it will ensure
 * fput() is called correspondingly.
 */
int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
			struct file *file, unsigned int file_slot)
{
	struct io_ring_ctx *ctx = req->ctx;
	int ret;

	io_ring_submit_lock(ctx, issue_flags);
	ret = __io_fixed_fd_install(ctx, file, file_slot);
	io_ring_submit_unlock(ctx, issue_flags);

	if (unlikely(ret < 0))
		fput(file);
	return ret;
}

int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
{
	struct io_fixed_file *file_slot;
	int ret;

	if (unlikely(!ctx->file_data))
		return -ENXIO;
	if (offset >= ctx->nr_user_files)
		return -EINVAL;

	offset = array_index_nospec(offset, ctx->nr_user_files);
	file_slot = io_fixed_file_slot(&ctx->file_table, offset);
	if (!file_slot->file_ptr)
		return -EBADF;

	ret = io_queue_rsrc_removal(ctx->file_data, offset,
				    io_slot_file(file_slot));
	if (ret)
		return ret;

	file_slot->file_ptr = 0;
	io_file_bitmap_clear(&ctx->file_table, offset);
	return 0;
}

int io_register_file_alloc_range(struct io_ring_ctx *ctx,
				 struct io_uring_file_index_range __user *arg)
{
	struct io_uring_file_index_range range;
	u32 end;

	if (copy_from_user(&range, arg, sizeof(range)))
		return -EFAULT;
	if (check_add_overflow(range.off, range.len, &end))
		return -EOVERFLOW;
	if (range.resv || end > ctx->nr_user_files)
		return -EINVAL;

	io_file_table_set_alloc_range(ctx, range.off, range.len);
	return 0;
}
Commit	Line	Data
453b329b JA	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/errno.h>
	4	#include <linux/file.h>
	5	#include <linux/mm.h>
	6	#include <linux/slab.h>
c98817e6	7	#include <linux/nospec.h>
453b329b JA	8	#include <linux/io_uring.h>
	9
	10	#include <uapi/linux/io_uring.h>
	11
453b329b	12	#include "io_uring.h"
c98817e6 JA	13	#include "rsrc.h"
c98817e6 JA	14	#include "filetable.h"
453b329b	15
c98817e6	16	static int io_file_bitmap_get(struct io_ring_ctx *ctx)
453b329b JA	17	{
453b329b JA	18	struct io_file_table *table = &ctx->file_table;
6e73dffb	19	unsigned long nr = ctx->file_alloc_end;
453b329b JA	20	int ret;
453b329b JA	21
02a4d923 SD	22	if (!table->bitmap)
	23	return -ENFILE;
	24
453b329b JA	25	do {
	26	ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
	27	if (ret != nr)
	28	return ret;
	29
6e73dffb	30	if (table->alloc_hint == ctx->file_alloc_start)
453b329b	31	break;
453b329b	32	nr = table->alloc_hint;
6e73dffb	33	table->alloc_hint = ctx->file_alloc_start;
453b329b JA	34	} while (1);
	35
	36	return -ENFILE;
	37	}
	38
	39	bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
	40	{
	41	table->files = kvcalloc(nr_files, sizeof(table->files[0]),
	42	GFP_KERNEL_ACCOUNT);
	43	if (unlikely(!table->files))
	44	return false;
	45
	46	table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
	47	if (unlikely(!table->bitmap)) {
	48	kvfree(table->files);
	49	return false;
	50	}
	51
	52	return true;
	53	}
	54
	55	void io_free_file_tables(struct io_file_table *table)
	56	{
	57	kvfree(table->files);
	58	bitmap_free(table->bitmap);
	59	table->files = NULL;
	60	table->bitmap = NULL;
	61	}
c98817e6	62
f110ed84 JA	63	static int io_install_fixed_file(struct io_ring_ctx ctx, struct file file,
f110ed84 JA	64	u32 slot_index)
c98817e6 JA	65	__must_hold(&req->ctx->uring_lock)
c98817e6 JA	66	{
c98817e6 JA	67	struct io_fixed_file *file_slot;
	68	int ret;
	69
	70	if (io_is_uring_fops(file))
	71	return -EBADF;
	72	if (!ctx->file_data)
	73	return -ENXIO;
	74	if (slot_index >= ctx->nr_user_files)
	75	return -EINVAL;
	76
	77	slot_index = array_index_nospec(slot_index, ctx->nr_user_files);
	78	file_slot = io_fixed_file_slot(&ctx->file_table, slot_index);
	79
	80	if (file_slot->file_ptr) {
4bfb0c9a CH	81	ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
4bfb0c9a CH	82	io_slot_file(file_slot));
c98817e6	83	if (ret)
c87fd583 PB	84	return ret;
c87fd583 PB	85
c98817e6 JA	86	file_slot->file_ptr = 0;
c98817e6 JA	87	io_file_bitmap_clear(&ctx->file_table, slot_index);
c98817e6 JA	88	}
c98817e6 JA	89
6e5e6d27 JA	90	*io_get_tag_slot(ctx->file_data, slot_index) = 0;
	91	io_fixed_file_set(file_slot, file);
	92	io_file_bitmap_set(&ctx->file_table, slot_index);
	93	return 0;
c98817e6 JA	94	}
c98817e6 JA	95
f110ed84 JA	96	int __io_fixed_fd_install(struct io_ring_ctx ctx, struct file file,
f110ed84 JA	97	unsigned int file_slot)
c98817e6 JA	98	{
c98817e6 JA	99	bool alloc_slot = file_slot == IORING_FILE_INDEX_ALLOC;
c98817e6 JA	100	int ret;
c98817e6 JA	101
c98817e6 JA	102	if (alloc_slot) {
	103	ret = io_file_bitmap_get(ctx);
	104	if (unlikely(ret < 0))
f110ed84	105	return ret;
c98817e6 JA	106	file_slot = ret;
	107	} else {
	108	file_slot--;
	109	}
	110
f110ed84	111	ret = io_install_fixed_file(ctx, file, file_slot);
c98817e6 JA	112	if (!ret && alloc_slot)
c98817e6 JA	113	ret = file_slot;
f110ed84 JA	114	return ret;
	115	}
	116	/*
	117	* Note when io_fixed_fd_install() returns error value, it will ensure
	118	* fput() is called correspondingly.
	119	*/
	120	int io_fixed_fd_install(struct io_kiocb *req, unsigned int issue_flags,
	121	struct file *file, unsigned int file_slot)
	122	{
	123	struct io_ring_ctx *ctx = req->ctx;
	124	int ret;
	125
	126	io_ring_submit_lock(ctx, issue_flags);
	127	ret = __io_fixed_fd_install(ctx, file, file_slot);
c98817e6	128	io_ring_submit_unlock(ctx, issue_flags);
f110ed84	129
c98817e6 JA	130	if (unlikely(ret < 0))
	131	fput(file);
	132	return ret;
	133	}
f110ed84 JA	134
	135	int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
	136	{
	137	struct io_fixed_file *file_slot;
f110ed84 JA	138	int ret;
	139
	140	if (unlikely(!ctx->file_data))
	141	return -ENXIO;
	142	if (offset >= ctx->nr_user_files)
	143	return -EINVAL;
f110ed84 JA	144
	145	offset = array_index_nospec(offset, ctx->nr_user_files);
	146	file_slot = io_fixed_file_slot(&ctx->file_table, offset);
	147	if (!file_slot->file_ptr)
	148	return -EBADF;
	149
4bfb0c9a CH	150	ret = io_queue_rsrc_removal(ctx->file_data, offset,
4bfb0c9a CH	151	io_slot_file(file_slot));
f110ed84 JA	152	if (ret)
	153	return ret;
	154
	155	file_slot->file_ptr = 0;
	156	io_file_bitmap_clear(&ctx->file_table, offset);
f110ed84 JA	157	return 0;
f110ed84 JA	158	}
6e73dffb PB	159
	160	int io_register_file_alloc_range(struct io_ring_ctx *ctx,
	161	struct io_uring_file_index_range __user *arg)
	162	{
	163	struct io_uring_file_index_range range;
	164	u32 end;
	165
	166	if (copy_from_user(&range, arg, sizeof(range)))
	167	return -EFAULT;
	168	if (check_add_overflow(range.off, range.len, &end))
	169	return -EOVERFLOW;
	170	if (range.resv \|\| end > ctx->nr_user_files)
	171	return -EINVAL;
	172
	173	io_file_table_set_alloc_range(ctx, range.off, range.len);
	174	return 0;
	175	}