[people/ms/linux.git] / kernel / bpf / syscall.c

/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 */
#include <linux/bpf.h>
#include <linux/syscalls.h>
#include <linux/slab.h>
#include <linux/anon_inodes.h>
#include <linux/file.h>
#include <linux/license.h>
#include <linux/filter.h>
#include <linux/version.h>

int sysctl_unprivileged_bpf_disabled __read_mostly;

static LIST_HEAD(bpf_map_types);

static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
{
	struct bpf_map_type_list *tl;
	struct bpf_map *map;

	list_for_each_entry(tl, &bpf_map_types, list_node) {
		if (tl->type == attr->map_type) {
			map = tl->ops->map_alloc(attr);
			if (IS_ERR(map))
				return map;
			map->ops = tl->ops;
			map->map_type = attr->map_type;
			return map;
		}
	}
	return ERR_PTR(-EINVAL);
}

/* boot time registration of different map implementations */
void bpf_register_map_type(struct bpf_map_type_list *tl)
{
	list_add(&tl->list_node, &bpf_map_types);
}

static int bpf_map_charge_memlock(struct bpf_map *map)
{
	struct user_struct *user = get_current_user();
	unsigned long memlock_limit;

	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	atomic_long_add(map->pages, &user->locked_vm);

	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
		atomic_long_sub(map->pages, &user->locked_vm);
		free_uid(user);
		return -EPERM;
	}
	map->user = user;
	return 0;
}

static void bpf_map_uncharge_memlock(struct bpf_map *map)
{
	struct user_struct *user = map->user;

	atomic_long_sub(map->pages, &user->locked_vm);
	free_uid(user);
}

/* called from workqueue */
static void bpf_map_free_deferred(struct work_struct *work)
{
	struct bpf_map *map = container_of(work, struct bpf_map, work);

	bpf_map_uncharge_memlock(map);
	/* implementation dependent freeing */
	map->ops->map_free(map);
}

static void bpf_map_put_uref(struct bpf_map *map)
{
	if (atomic_dec_and_test(&map->usercnt)) {
		if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
			bpf_fd_array_map_clear(map);
	}
}

/* decrement map refcnt and schedule it for freeing via workqueue
 * (unrelying map implementation ops->map_free() might sleep)
 */
void bpf_map_put(struct bpf_map *map)
{
	if (atomic_dec_and_test(&map->refcnt)) {
		INIT_WORK(&map->work, bpf_map_free_deferred);
		schedule_work(&map->work);
	}
}

void bpf_map_put_with_uref(struct bpf_map *map)
{
	bpf_map_put_uref(map);
	bpf_map_put(map);
}

static int bpf_map_release(struct inode *inode, struct file *filp)
{
	bpf_map_put_with_uref(filp->private_data);
	return 0;
}

static const struct file_operations bpf_map_fops = {
	.release = bpf_map_release,
};

int bpf_map_new_fd(struct bpf_map *map)
{
	return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
				O_RDWR | O_CLOEXEC);
}

/* helper macro to check that unused fields 'union bpf_attr' are zero */
#define CHECK_ATTR(CMD) \
	memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
		   sizeof(attr->CMD##_LAST_FIELD), 0, \
		   sizeof(*attr) - \
		   offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
		   sizeof(attr->CMD##_LAST_FIELD)) != NULL

#define BPF_MAP_CREATE_LAST_FIELD max_entries
/* called via syscall */
static int map_create(union bpf_attr *attr)
{
	struct bpf_map *map;
	int err;

	err = CHECK_ATTR(BPF_MAP_CREATE);
	if (err)
		return -EINVAL;

	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
	map = find_and_alloc_map(attr);
	if (IS_ERR(map))
		return PTR_ERR(map);

	atomic_set(&map->refcnt, 1);
	atomic_set(&map->usercnt, 1);

	err = bpf_map_charge_memlock(map);
	if (err)
		goto free_map;

	err = bpf_map_new_fd(map);
	if (err < 0)
		/* failed to allocate fd */
		goto free_map;

	return err;

free_map:
	map->ops->map_free(map);
	return err;
}

/* if error is returned, fd is released.
 * On success caller should complete fd access with matching fdput()
 */
struct bpf_map *__bpf_map_get(struct fd f)
{
	if (!f.file)
		return ERR_PTR(-EBADF);
	if (f.file->f_op != &bpf_map_fops) {
		fdput(f);
		return ERR_PTR(-EINVAL);
	}

	return f.file->private_data;
}

void bpf_map_inc(struct bpf_map *map, bool uref)
{
	atomic_inc(&map->refcnt);
	if (uref)
		atomic_inc(&map->usercnt);
}

struct bpf_map *bpf_map_get_with_uref(u32 ufd)
{
	struct fd f = fdget(ufd);
	struct bpf_map *map;

	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return map;

	bpf_map_inc(map, true);
	fdput(f);

	return map;
}

/* helper to convert user pointers passed inside __aligned_u64 fields */
static void __user *u64_to_ptr(__u64 val)
{
	return (void __user *) (unsigned long) val;
}

/* last field in 'union bpf_attr' used by this command */
#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value

static int map_lookup_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *uvalue = u64_to_ptr(attr->value);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *value, *ptr;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
	if (!value)
		goto free_key;

	rcu_read_lock();
	ptr = map->ops->map_lookup_elem(map, key);
	if (ptr)
		memcpy(value, ptr, map->value_size);
	rcu_read_unlock();

	err = -ENOENT;
	if (!ptr)
		goto free_value;

	err = -EFAULT;
	if (copy_to_user(uvalue, value, map->value_size) != 0)
		goto free_value;

	err = 0;

free_value:
	kfree(value);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags

static int map_update_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *uvalue = u64_to_ptr(attr->value);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *value;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
	if (!value)
		goto free_key;

	err = -EFAULT;
	if (copy_from_user(value, uvalue, map->value_size) != 0)
		goto free_value;

	/* eBPF program that use maps are running under rcu_read_lock(),
	 * therefore all map accessors rely on this fact, so do the same here
	 */
	rcu_read_lock();
	err = map->ops->map_update_elem(map, key, value, attr->flags);
	rcu_read_unlock();

free_value:
	kfree(value);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

#define BPF_MAP_DELETE_ELEM_LAST_FIELD key

static int map_delete_elem(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	struct fd f;
	void *key;
	int err;

	if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	rcu_read_lock();
	err = map->ops->map_delete_elem(map, key);
	rcu_read_unlock();

free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

/* last field in 'union bpf_attr' used by this command */
#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key

static int map_get_next_key(union bpf_attr *attr)
{
	void __user *ukey = u64_to_ptr(attr->key);
	void __user *unext_key = u64_to_ptr(attr->next_key);
	int ufd = attr->map_fd;
	struct bpf_map *map;
	void *key, *next_key;
	struct fd f;
	int err;

	if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
		return -EINVAL;

	f = fdget(ufd);
	map = __bpf_map_get(f);
	if (IS_ERR(map))
		return PTR_ERR(map);

	err = -ENOMEM;
	key = kmalloc(map->key_size, GFP_USER);
	if (!key)
		goto err_put;

	err = -EFAULT;
	if (copy_from_user(key, ukey, map->key_size) != 0)
		goto free_key;

	err = -ENOMEM;
	next_key = kmalloc(map->key_size, GFP_USER);
	if (!next_key)
		goto free_key;

	rcu_read_lock();
	err = map->ops->map_get_next_key(map, key, next_key);
	rcu_read_unlock();
	if (err)
		goto free_next_key;

	err = -EFAULT;
	if (copy_to_user(unext_key, next_key, map->key_size) != 0)
		goto free_next_key;

	err = 0;

free_next_key:
	kfree(next_key);
free_key:
	kfree(key);
err_put:
	fdput(f);
	return err;
}

static LIST_HEAD(bpf_prog_types);

static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
{
	struct bpf_prog_type_list *tl;

	list_for_each_entry(tl, &bpf_prog_types, list_node) {
		if (tl->type == type) {
			prog->aux->ops = tl->ops;
			prog->type = type;
			return 0;
		}
	}

	return -EINVAL;
}

void bpf_register_prog_type(struct bpf_prog_type_list *tl)
{
	list_add(&tl->list_node, &bpf_prog_types);
}

/* fixup insn->imm field of bpf_call instructions:
 * if (insn->imm == BPF_FUNC_map_lookup_elem)
 *      insn->imm = bpf_map_lookup_elem - __bpf_call_base;
 * else if (insn->imm == BPF_FUNC_map_update_elem)
 *      insn->imm = bpf_map_update_elem - __bpf_call_base;
 * else ...
 *
 * this function is called after eBPF program passed verification
 */
static void fixup_bpf_calls(struct bpf_prog *prog)
{
	const struct bpf_func_proto *fn;
	int i;

	for (i = 0; i < prog->len; i++) {
		struct bpf_insn *insn = &prog->insnsi[i];

		if (insn->code == (BPF_JMP | BPF_CALL)) {
			/* we reach here when program has bpf_call instructions
			 * and it passed bpf_check(), means that
			 * ops->get_func_proto must have been supplied, check it
			 */
			BUG_ON(!prog->aux->ops->get_func_proto);

			if (insn->imm == BPF_FUNC_get_route_realm)
				prog->dst_needed = 1;
			if (insn->imm == BPF_FUNC_get_prandom_u32)
				bpf_user_rnd_init_once();
			if (insn->imm == BPF_FUNC_tail_call) {
				/* mark bpf_tail_call as different opcode
				 * to avoid conditional branch in
				 * interpeter for every normal call
				 * and to prevent accidental JITing by
				 * JIT compiler that doesn't support
				 * bpf_tail_call yet
				 */
				insn->imm = 0;
				insn->code |= BPF_X;
				continue;
			}

			fn = prog->aux->ops->get_func_proto(insn->imm);
			/* all functions that have prototype and verifier allowed
			 * programs to call them, must be real in-kernel functions
			 */
			BUG_ON(!fn->func);
			insn->imm = fn->func - __bpf_call_base;
		}
	}
}

/* drop refcnt on maps used by eBPF program and free auxilary data */
static void free_used_maps(struct bpf_prog_aux *aux)
{
	int i;

	for (i = 0; i < aux->used_map_cnt; i++)
		bpf_map_put(aux->used_maps[i]);

	kfree(aux->used_maps);
}

static int bpf_prog_charge_memlock(struct bpf_prog *prog)
{
	struct user_struct *user = get_current_user();
	unsigned long memlock_limit;

	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;

	atomic_long_add(prog->pages, &user->locked_vm);
	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
		atomic_long_sub(prog->pages, &user->locked_vm);
		free_uid(user);
		return -EPERM;
	}
	prog->aux->user = user;
	return 0;
}

static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
{
	struct user_struct *user = prog->aux->user;

	atomic_long_sub(prog->pages, &user->locked_vm);
	free_uid(user);
}

static void __prog_put_common(struct rcu_head *rcu)
{
	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);

	free_used_maps(aux);
	bpf_prog_uncharge_memlock(aux->prog);
	bpf_prog_free(aux->prog);
}

/* version of bpf_prog_put() that is called after a grace period */
void bpf_prog_put_rcu(struct bpf_prog *prog)
{
	if (atomic_dec_and_test(&prog->aux->refcnt))
		call_rcu(&prog->aux->rcu, __prog_put_common);
}

void bpf_prog_put(struct bpf_prog *prog)
{
	if (atomic_dec_and_test(&prog->aux->refcnt))
		__prog_put_common(&prog->aux->rcu);
}
EXPORT_SYMBOL_GPL(bpf_prog_put);

static int bpf_prog_release(struct inode *inode, struct file *filp)
{
	struct bpf_prog *prog = filp->private_data;

	bpf_prog_put_rcu(prog);
	return 0;
}

static const struct file_operations bpf_prog_fops = {
        .release = bpf_prog_release,
};

int bpf_prog_new_fd(struct bpf_prog *prog)
{
	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
				O_RDWR | O_CLOEXEC);
}

static struct bpf_prog *__bpf_prog_get(struct fd f)
{
	if (!f.file)
		return ERR_PTR(-EBADF);
	if (f.file->f_op != &bpf_prog_fops) {
		fdput(f);
		return ERR_PTR(-EINVAL);
	}

	return f.file->private_data;
}

/* called by sockets/tracing/seccomp before attaching program to an event
 * pairs with bpf_prog_put()
 */
struct bpf_prog *bpf_prog_get(u32 ufd)
{
	struct fd f = fdget(ufd);
	struct bpf_prog *prog;

	prog = __bpf_prog_get(f);
	if (IS_ERR(prog))
		return prog;

	atomic_inc(&prog->aux->refcnt);
	fdput(f);

	return prog;
}
EXPORT_SYMBOL_GPL(bpf_prog_get);

/* last field in 'union bpf_attr' used by this command */
#define	BPF_PROG_LOAD_LAST_FIELD kern_version

static int bpf_prog_load(union bpf_attr *attr)
{
	enum bpf_prog_type type = attr->prog_type;
	struct bpf_prog *prog;
	int err;
	char license[128];
	bool is_gpl;

	if (CHECK_ATTR(BPF_PROG_LOAD))
		return -EINVAL;

	/* copy eBPF program license from user space */
	if (strncpy_from_user(license, u64_to_ptr(attr->license),
			      sizeof(license) - 1) < 0)
		return -EFAULT;
	license[sizeof(license) - 1] = 0;

	/* eBPF programs must be GPL compatible to use GPL-ed functions */
	is_gpl = license_is_gpl_compatible(license);

	if (attr->insn_cnt >= BPF_MAXINSNS)
		return -EINVAL;

	if (type == BPF_PROG_TYPE_KPROBE &&
	    attr->kern_version != LINUX_VERSION_CODE)
		return -EINVAL;

	if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	/* plain bpf_prog allocation */
	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
	if (!prog)
		return -ENOMEM;

	err = bpf_prog_charge_memlock(prog);
	if (err)
		goto free_prog_nouncharge;

	prog->len = attr->insn_cnt;

	err = -EFAULT;
	if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
			   prog->len * sizeof(struct bpf_insn)) != 0)
		goto free_prog;

	prog->orig_prog = NULL;
	prog->jited = 0;

	atomic_set(&prog->aux->refcnt, 1);
	prog->gpl_compatible = is_gpl ? 1 : 0;

	/* find program type: socket_filter vs tracing_filter */
	err = find_prog_type(type, prog);
	if (err < 0)
		goto free_prog;

	/* run eBPF verifier */
	err = bpf_check(&prog, attr);
	if (err < 0)
		goto free_used_maps;

	/* fixup BPF_CALL->imm field */
	fixup_bpf_calls(prog);

	/* eBPF program is ready to be JITed */
	err = bpf_prog_select_runtime(prog);
	if (err < 0)
		goto free_used_maps;

	err = bpf_prog_new_fd(prog);
	if (err < 0)
		/* failed to allocate fd */
		goto free_used_maps;

	return err;

free_used_maps:
	free_used_maps(prog->aux);
free_prog:
	bpf_prog_uncharge_memlock(prog);
free_prog_nouncharge:
	bpf_prog_free(prog);
	return err;
}

#define BPF_OBJ_LAST_FIELD bpf_fd

static int bpf_obj_pin(const union bpf_attr *attr)
{
	if (CHECK_ATTR(BPF_OBJ))
		return -EINVAL;

	return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname));
}

static int bpf_obj_get(const union bpf_attr *attr)
{
	if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0)
		return -EINVAL;

	return bpf_obj_get_user(u64_to_ptr(attr->pathname));
}

SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
{
	union bpf_attr attr = {};
	int err;

	if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
		return -EPERM;

	if (!access_ok(VERIFY_READ, uattr, 1))
		return -EFAULT;

	if (size > PAGE_SIZE)	/* silly large */
		return -E2BIG;

	/* If we're handed a bigger struct than we know of,
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
	 */
	if (size > sizeof(attr)) {
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;

		addr = (void __user *)uattr + sizeof(attr);
		end  = (void __user *)uattr + size;

		for (; addr < end; addr++) {
			err = get_user(val, addr);
			if (err)
				return err;
			if (val)
				return -E2BIG;
		}
		size = sizeof(attr);
	}

	/* copy attributes from user space, may be less than sizeof(bpf_attr) */
	if (copy_from_user(&attr, uattr, size) != 0)
		return -EFAULT;

	switch (cmd) {
	case BPF_MAP_CREATE:
		err = map_create(&attr);
		break;
	case BPF_MAP_LOOKUP_ELEM:
		err = map_lookup_elem(&attr);
		break;
	case BPF_MAP_UPDATE_ELEM:
		err = map_update_elem(&attr);
		break;
	case BPF_MAP_DELETE_ELEM:
		err = map_delete_elem(&attr);
		break;
	case BPF_MAP_GET_NEXT_KEY:
		err = map_get_next_key(&attr);
		break;
	case BPF_PROG_LOAD:
		err = bpf_prog_load(&attr);
		break;
	case BPF_OBJ_PIN:
		err = bpf_obj_pin(&attr);
		break;
	case BPF_OBJ_GET:
		err = bpf_obj_get(&attr);
		break;
	default:
		err = -EINVAL;
		break;
	}

	return err;
}
Commit	Line	Data
99c55f7d AS	1	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
	2	*
	3	* This program is free software; you can redistribute it and/or
	4	* modify it under the terms of version 2 of the GNU General Public
	5	* License as published by the Free Software Foundation.
	6	*
	7	* This program is distributed in the hope that it will be useful, but
	8	* WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	10	* General Public License for more details.
	11	*/
	12	#include <linux/bpf.h>
	13	#include <linux/syscalls.h>
	14	#include <linux/slab.h>
	15	#include <linux/anon_inodes.h>
db20fd2b	16	#include <linux/file.h>
09756af4 AS	17	#include <linux/license.h>
09756af4 AS	18	#include <linux/filter.h>
2541517c	19	#include <linux/version.h>
99c55f7d	20
1be7f75d AS	21	int sysctl_unprivileged_bpf_disabled __read_mostly;
1be7f75d AS	22
99c55f7d AS	23	static LIST_HEAD(bpf_map_types);
	24
	25	static struct bpf_map find_and_alloc_map(union bpf_attr attr)
	26	{
	27	struct bpf_map_type_list *tl;
	28	struct bpf_map *map;
	29
	30	list_for_each_entry(tl, &bpf_map_types, list_node) {
	31	if (tl->type == attr->map_type) {
	32	map = tl->ops->map_alloc(attr);
	33	if (IS_ERR(map))
	34	return map;
	35	map->ops = tl->ops;
	36	map->map_type = attr->map_type;
	37	return map;
	38	}
	39	}
	40	return ERR_PTR(-EINVAL);
	41	}
	42
	43	/* boot time registration of different map implementations */
	44	void bpf_register_map_type(struct bpf_map_type_list *tl)
	45	{
	46	list_add(&tl->list_node, &bpf_map_types);
	47	}
	48
aaac3ba9 AS	49	static int bpf_map_charge_memlock(struct bpf_map *map)
	50	{
	51	struct user_struct *user = get_current_user();
	52	unsigned long memlock_limit;
	53
	54	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	55
	56	atomic_long_add(map->pages, &user->locked_vm);
	57
	58	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
	59	atomic_long_sub(map->pages, &user->locked_vm);
	60	free_uid(user);
	61	return -EPERM;
	62	}
	63	map->user = user;
	64	return 0;
	65	}
	66
	67	static void bpf_map_uncharge_memlock(struct bpf_map *map)
	68	{
	69	struct user_struct *user = map->user;
	70
	71	atomic_long_sub(map->pages, &user->locked_vm);
	72	free_uid(user);
	73	}
	74
99c55f7d AS	75	/* called from workqueue */
	76	static void bpf_map_free_deferred(struct work_struct *work)
	77	{
	78	struct bpf_map *map = container_of(work, struct bpf_map, work);
	79
aaac3ba9	80	bpf_map_uncharge_memlock(map);
99c55f7d AS	81	/* implementation dependent freeing */
	82	map->ops->map_free(map);
	83	}
	84
c9da161c DB	85	static void bpf_map_put_uref(struct bpf_map *map)
	86	{
	87	if (atomic_dec_and_test(&map->usercnt)) {
	88	if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY)
	89	bpf_fd_array_map_clear(map);
	90	}
	91	}
	92
99c55f7d AS	93	/* decrement map refcnt and schedule it for freeing via workqueue
	94	* (unrelying map implementation ops->map_free() might sleep)
	95	*/
	96	void bpf_map_put(struct bpf_map *map)
	97	{
	98	if (atomic_dec_and_test(&map->refcnt)) {
	99	INIT_WORK(&map->work, bpf_map_free_deferred);
	100	schedule_work(&map->work);
	101	}
	102	}
	103
c9da161c	104	void bpf_map_put_with_uref(struct bpf_map *map)
99c55f7d	105	{
c9da161c	106	bpf_map_put_uref(map);
99c55f7d	107	bpf_map_put(map);
c9da161c DB	108	}
	109
	110	static int bpf_map_release(struct inode inode, struct file filp)
	111	{
	112	bpf_map_put_with_uref(filp->private_data);
99c55f7d AS	113	return 0;
	114	}
	115
	116	static const struct file_operations bpf_map_fops = {
	117	.release = bpf_map_release,
	118	};
	119
b2197755	120	int bpf_map_new_fd(struct bpf_map *map)
aa79781b DB	121	{
	122	return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
	123	O_RDWR \| O_CLOEXEC);
	124	}
	125
99c55f7d AS	126	/* helper macro to check that unused fields 'union bpf_attr' are zero */
	127	#define CHECK_ATTR(CMD) \
	128	memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
	129	sizeof(attr->CMD##_LAST_FIELD), 0, \
	130	sizeof(*attr) - \
	131	offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
	132	sizeof(attr->CMD##_LAST_FIELD)) != NULL
	133
	134	#define BPF_MAP_CREATE_LAST_FIELD max_entries
	135	/* called via syscall */
	136	static int map_create(union bpf_attr *attr)
	137	{
	138	struct bpf_map *map;
	139	int err;
	140
	141	err = CHECK_ATTR(BPF_MAP_CREATE);
	142	if (err)
	143	return -EINVAL;
	144
	145	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
	146	map = find_and_alloc_map(attr);
	147	if (IS_ERR(map))
	148	return PTR_ERR(map);
	149
	150	atomic_set(&map->refcnt, 1);
c9da161c	151	atomic_set(&map->usercnt, 1);
99c55f7d	152
aaac3ba9 AS	153	err = bpf_map_charge_memlock(map);
	154	if (err)
	155	goto free_map;
	156
aa79781b	157	err = bpf_map_new_fd(map);
99c55f7d AS	158	if (err < 0)
	159	/* failed to allocate fd */
	160	goto free_map;
	161
	162	return err;
	163
	164	free_map:
	165	map->ops->map_free(map);
	166	return err;
	167	}
	168
db20fd2b AS	169	/* if error is returned, fd is released.
	170	* On success caller should complete fd access with matching fdput()
	171	*/
c2101297	172	struct bpf_map *__bpf_map_get(struct fd f)
db20fd2b	173	{
db20fd2b AS	174	if (!f.file)
db20fd2b AS	175	return ERR_PTR(-EBADF);
db20fd2b AS	176	if (f.file->f_op != &bpf_map_fops) {
	177	fdput(f);
	178	return ERR_PTR(-EINVAL);
	179	}
	180
c2101297 DB	181	return f.file->private_data;
	182	}
	183
c9da161c DB	184	void bpf_map_inc(struct bpf_map *map, bool uref)
	185	{
	186	atomic_inc(&map->refcnt);
	187	if (uref)
	188	atomic_inc(&map->usercnt);
	189	}
	190
	191	struct bpf_map *bpf_map_get_with_uref(u32 ufd)
c2101297 DB	192	{
	193	struct fd f = fdget(ufd);
	194	struct bpf_map *map;
	195
	196	map = __bpf_map_get(f);
	197	if (IS_ERR(map))
	198	return map;
	199
c9da161c	200	bpf_map_inc(map, true);
c2101297	201	fdput(f);
db20fd2b AS	202
	203	return map;
	204	}
	205
	206	/* helper to convert user pointers passed inside __aligned_u64 fields */
	207	static void __user *u64_to_ptr(__u64 val)
	208	{
	209	return (void __user *) (unsigned long) val;
	210	}
	211
	212	/* last field in 'union bpf_attr' used by this command */
	213	#define BPF_MAP_LOOKUP_ELEM_LAST_FIELD value
	214
	215	static int map_lookup_elem(union bpf_attr *attr)
	216	{
	217	void __user *ukey = u64_to_ptr(attr->key);
	218	void __user *uvalue = u64_to_ptr(attr->value);
	219	int ufd = attr->map_fd;
db20fd2b	220	struct bpf_map *map;
8ebe667c	221	void key, value, *ptr;
592867bf	222	struct fd f;
db20fd2b AS	223	int err;
	224
	225	if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
	226	return -EINVAL;
	227
592867bf	228	f = fdget(ufd);
c2101297	229	map = __bpf_map_get(f);
db20fd2b AS	230	if (IS_ERR(map))
	231	return PTR_ERR(map);
	232
	233	err = -ENOMEM;
	234	key = kmalloc(map->key_size, GFP_USER);
	235	if (!key)
	236	goto err_put;
	237
	238	err = -EFAULT;
	239	if (copy_from_user(key, ukey, map->key_size) != 0)
	240	goto free_key;
	241
8ebe667c	242	err = -ENOMEM;
01b3f521	243	value = kmalloc(map->value_size, GFP_USER \| __GFP_NOWARN);
db20fd2b	244	if (!value)
8ebe667c AS	245	goto free_key;
	246
	247	rcu_read_lock();
	248	ptr = map->ops->map_lookup_elem(map, key);
	249	if (ptr)
	250	memcpy(value, ptr, map->value_size);
	251	rcu_read_unlock();
	252
	253	err = -ENOENT;
	254	if (!ptr)
	255	goto free_value;
db20fd2b AS	256
	257	err = -EFAULT;
	258	if (copy_to_user(uvalue, value, map->value_size) != 0)
8ebe667c	259	goto free_value;
db20fd2b AS	260
	261	err = 0;
	262
8ebe667c AS	263	free_value:
8ebe667c AS	264	kfree(value);
db20fd2b AS	265	free_key:
	266	kfree(key);
	267	err_put:
	268	fdput(f);
	269	return err;
	270	}
	271
3274f520	272	#define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
db20fd2b AS	273
	274	static int map_update_elem(union bpf_attr *attr)
	275	{
	276	void __user *ukey = u64_to_ptr(attr->key);
	277	void __user *uvalue = u64_to_ptr(attr->value);
	278	int ufd = attr->map_fd;
db20fd2b AS	279	struct bpf_map *map;
db20fd2b AS	280	void key, value;
592867bf	281	struct fd f;
db20fd2b AS	282	int err;
	283
	284	if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
	285	return -EINVAL;
	286
592867bf	287	f = fdget(ufd);
c2101297	288	map = __bpf_map_get(f);
db20fd2b AS	289	if (IS_ERR(map))
	290	return PTR_ERR(map);
	291
	292	err = -ENOMEM;
	293	key = kmalloc(map->key_size, GFP_USER);
	294	if (!key)
	295	goto err_put;
	296
	297	err = -EFAULT;
	298	if (copy_from_user(key, ukey, map->key_size) != 0)
	299	goto free_key;
	300
	301	err = -ENOMEM;
01b3f521	302	value = kmalloc(map->value_size, GFP_USER \| __GFP_NOWARN);
db20fd2b AS	303	if (!value)
	304	goto free_key;
	305
	306	err = -EFAULT;
	307	if (copy_from_user(value, uvalue, map->value_size) != 0)
	308	goto free_value;
	309
	310	/* eBPF program that use maps are running under rcu_read_lock(),
	311	* therefore all map accessors rely on this fact, so do the same here
	312	*/
	313	rcu_read_lock();
3274f520	314	err = map->ops->map_update_elem(map, key, value, attr->flags);
db20fd2b AS	315	rcu_read_unlock();
	316
	317	free_value:
	318	kfree(value);
	319	free_key:
	320	kfree(key);
	321	err_put:
	322	fdput(f);
	323	return err;
	324	}
	325
	326	#define BPF_MAP_DELETE_ELEM_LAST_FIELD key
	327
	328	static int map_delete_elem(union bpf_attr *attr)
	329	{
	330	void __user *ukey = u64_to_ptr(attr->key);
	331	int ufd = attr->map_fd;
db20fd2b	332	struct bpf_map *map;
592867bf	333	struct fd f;
db20fd2b AS	334	void *key;
	335	int err;
	336
	337	if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
	338	return -EINVAL;
	339
592867bf	340	f = fdget(ufd);
c2101297	341	map = __bpf_map_get(f);
db20fd2b AS	342	if (IS_ERR(map))
	343	return PTR_ERR(map);
	344
	345	err = -ENOMEM;
	346	key = kmalloc(map->key_size, GFP_USER);
	347	if (!key)
	348	goto err_put;
	349
	350	err = -EFAULT;
	351	if (copy_from_user(key, ukey, map->key_size) != 0)
	352	goto free_key;
	353
	354	rcu_read_lock();
	355	err = map->ops->map_delete_elem(map, key);
	356	rcu_read_unlock();
	357
	358	free_key:
	359	kfree(key);
	360	err_put:
	361	fdput(f);
	362	return err;
	363	}
	364
	365	/* last field in 'union bpf_attr' used by this command */
	366	#define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
	367
	368	static int map_get_next_key(union bpf_attr *attr)
	369	{
	370	void __user *ukey = u64_to_ptr(attr->key);
	371	void __user *unext_key = u64_to_ptr(attr->next_key);
	372	int ufd = attr->map_fd;
db20fd2b AS	373	struct bpf_map *map;
db20fd2b AS	374	void key, next_key;
592867bf	375	struct fd f;
db20fd2b AS	376	int err;
	377
	378	if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
	379	return -EINVAL;
	380
592867bf	381	f = fdget(ufd);
c2101297	382	map = __bpf_map_get(f);
db20fd2b AS	383	if (IS_ERR(map))
	384	return PTR_ERR(map);
	385
	386	err = -ENOMEM;
	387	key = kmalloc(map->key_size, GFP_USER);
	388	if (!key)
	389	goto err_put;
	390
	391	err = -EFAULT;
	392	if (copy_from_user(key, ukey, map->key_size) != 0)
	393	goto free_key;
	394
	395	err = -ENOMEM;
	396	next_key = kmalloc(map->key_size, GFP_USER);
	397	if (!next_key)
	398	goto free_key;
	399
	400	rcu_read_lock();
	401	err = map->ops->map_get_next_key(map, key, next_key);
	402	rcu_read_unlock();
	403	if (err)
	404	goto free_next_key;
	405
	406	err = -EFAULT;
	407	if (copy_to_user(unext_key, next_key, map->key_size) != 0)
	408	goto free_next_key;
	409
	410	err = 0;
	411
	412	free_next_key:
	413	kfree(next_key);
	414	free_key:
	415	kfree(key);
	416	err_put:
	417	fdput(f);
	418	return err;
	419	}
	420
09756af4 AS	421	static LIST_HEAD(bpf_prog_types);
	422
	423	static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
	424	{
	425	struct bpf_prog_type_list *tl;
	426
	427	list_for_each_entry(tl, &bpf_prog_types, list_node) {
	428	if (tl->type == type) {
	429	prog->aux->ops = tl->ops;
24701ece	430	prog->type = type;
09756af4 AS	431	return 0;
	432	}
	433	}
24701ece	434
09756af4 AS	435	return -EINVAL;
	436	}
	437
	438	void bpf_register_prog_type(struct bpf_prog_type_list *tl)
	439	{
	440	list_add(&tl->list_node, &bpf_prog_types);
	441	}
	442
0a542a86 AS	443	/* fixup insn->imm field of bpf_call instructions:
	444	* if (insn->imm == BPF_FUNC_map_lookup_elem)
	445	* insn->imm = bpf_map_lookup_elem - __bpf_call_base;
	446	* else if (insn->imm == BPF_FUNC_map_update_elem)
	447	* insn->imm = bpf_map_update_elem - __bpf_call_base;
	448	* else ...
	449	*
	450	* this function is called after eBPF program passed verification
	451	*/
	452	static void fixup_bpf_calls(struct bpf_prog *prog)
	453	{
	454	const struct bpf_func_proto *fn;
	455	int i;
	456
	457	for (i = 0; i < prog->len; i++) {
	458	struct bpf_insn *insn = &prog->insnsi[i];
	459
	460	if (insn->code == (BPF_JMP \| BPF_CALL)) {
	461	/* we reach here when program has bpf_call instructions
	462	* and it passed bpf_check(), means that
	463	* ops->get_func_proto must have been supplied, check it
	464	*/
	465	BUG_ON(!prog->aux->ops->get_func_proto);
	466
c46646d0 DB	467	if (insn->imm == BPF_FUNC_get_route_realm)
c46646d0 DB	468	prog->dst_needed = 1;
3ad00405 DB	469	if (insn->imm == BPF_FUNC_get_prandom_u32)
3ad00405 DB	470	bpf_user_rnd_init_once();
04fd61ab AS	471	if (insn->imm == BPF_FUNC_tail_call) {
	472	/* mark bpf_tail_call as different opcode
	473	* to avoid conditional branch in
	474	* interpeter for every normal call
	475	* and to prevent accidental JITing by
	476	* JIT compiler that doesn't support
	477	* bpf_tail_call yet
	478	*/
	479	insn->imm = 0;
	480	insn->code \|= BPF_X;
	481	continue;
	482	}
	483
0a542a86 AS	484	fn = prog->aux->ops->get_func_proto(insn->imm);
	485	/* all functions that have prototype and verifier allowed
	486	* programs to call them, must be real in-kernel functions
	487	*/
	488	BUG_ON(!fn->func);
	489	insn->imm = fn->func - __bpf_call_base;
	490	}
	491	}
	492	}
	493
09756af4 AS	494	/* drop refcnt on maps used by eBPF program and free auxilary data */
	495	static void free_used_maps(struct bpf_prog_aux *aux)
	496	{
	497	int i;
	498
	499	for (i = 0; i < aux->used_map_cnt; i++)
	500	bpf_map_put(aux->used_maps[i]);
	501
	502	kfree(aux->used_maps);
	503	}
	504
aaac3ba9 AS	505	static int bpf_prog_charge_memlock(struct bpf_prog *prog)
	506	{
	507	struct user_struct *user = get_current_user();
	508	unsigned long memlock_limit;
	509
	510	memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
	511
	512	atomic_long_add(prog->pages, &user->locked_vm);
	513	if (atomic_long_read(&user->locked_vm) > memlock_limit) {
	514	atomic_long_sub(prog->pages, &user->locked_vm);
	515	free_uid(user);
	516	return -EPERM;
	517	}
	518	prog->aux->user = user;
	519	return 0;
	520	}
	521
	522	static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
	523	{
	524	struct user_struct *user = prog->aux->user;
	525
	526	atomic_long_sub(prog->pages, &user->locked_vm);
	527	free_uid(user);
	528	}
	529
e9d8afa9	530	static void __prog_put_common(struct rcu_head *rcu)
abf2e7d6 AS	531	{
	532	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
	533
	534	free_used_maps(aux);
aaac3ba9	535	bpf_prog_uncharge_memlock(aux->prog);
abf2e7d6 AS	536	bpf_prog_free(aux->prog);
	537	}
	538
	539	/* version of bpf_prog_put() that is called after a grace period */
	540	void bpf_prog_put_rcu(struct bpf_prog *prog)
	541	{
e9d8afa9 DB	542	if (atomic_dec_and_test(&prog->aux->refcnt))
e9d8afa9 DB	543	call_rcu(&prog->aux->rcu, __prog_put_common);
abf2e7d6 AS	544	}
abf2e7d6 AS	545
09756af4 AS	546	void bpf_prog_put(struct bpf_prog *prog)
09756af4 AS	547	{
e9d8afa9 DB	548	if (atomic_dec_and_test(&prog->aux->refcnt))
e9d8afa9 DB	549	__prog_put_common(&prog->aux->rcu);
09756af4	550	}
e2e9b654	551	EXPORT_SYMBOL_GPL(bpf_prog_put);
09756af4 AS	552
	553	static int bpf_prog_release(struct inode inode, struct file filp)
	554	{
	555	struct bpf_prog *prog = filp->private_data;
	556
abf2e7d6	557	bpf_prog_put_rcu(prog);
09756af4 AS	558	return 0;
	559	}
	560
	561	static const struct file_operations bpf_prog_fops = {
	562	.release = bpf_prog_release,
	563	};
	564
b2197755	565	int bpf_prog_new_fd(struct bpf_prog *prog)
aa79781b DB	566	{
	567	return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
	568	O_RDWR \| O_CLOEXEC);
	569	}
	570
c2101297	571	static struct bpf_prog *__bpf_prog_get(struct fd f)
09756af4	572	{
09756af4 AS	573	if (!f.file)
09756af4 AS	574	return ERR_PTR(-EBADF);
09756af4 AS	575	if (f.file->f_op != &bpf_prog_fops) {
	576	fdput(f);
	577	return ERR_PTR(-EINVAL);
	578	}
	579
c2101297	580	return f.file->private_data;
09756af4 AS	581	}
	582
	583	/* called by sockets/tracing/seccomp before attaching program to an event
	584	* pairs with bpf_prog_put()
	585	*/
	586	struct bpf_prog *bpf_prog_get(u32 ufd)
	587	{
	588	struct fd f = fdget(ufd);
	589	struct bpf_prog *prog;
	590
c2101297	591	prog = __bpf_prog_get(f);
09756af4 AS	592	if (IS_ERR(prog))
	593	return prog;
	594
	595	atomic_inc(&prog->aux->refcnt);
	596	fdput(f);
c2101297	597
09756af4 AS	598	return prog;
09756af4 AS	599	}
e2e9b654	600	EXPORT_SYMBOL_GPL(bpf_prog_get);
09756af4 AS	601
09756af4 AS	602	/* last field in 'union bpf_attr' used by this command */
2541517c	603	#define BPF_PROG_LOAD_LAST_FIELD kern_version
09756af4 AS	604
	605	static int bpf_prog_load(union bpf_attr *attr)
	606	{
	607	enum bpf_prog_type type = attr->prog_type;
	608	struct bpf_prog *prog;
	609	int err;
	610	char license[128];
	611	bool is_gpl;
	612
	613	if (CHECK_ATTR(BPF_PROG_LOAD))
	614	return -EINVAL;
	615
	616	/* copy eBPF program license from user space */
	617	if (strncpy_from_user(license, u64_to_ptr(attr->license),
	618	sizeof(license) - 1) < 0)
	619	return -EFAULT;
	620	license[sizeof(license) - 1] = 0;
	621
	622	/* eBPF programs must be GPL compatible to use GPL-ed functions */
	623	is_gpl = license_is_gpl_compatible(license);
	624
	625	if (attr->insn_cnt >= BPF_MAXINSNS)
	626	return -EINVAL;
	627
2541517c AS	628	if (type == BPF_PROG_TYPE_KPROBE &&
	629	attr->kern_version != LINUX_VERSION_CODE)
	630	return -EINVAL;
	631
1be7f75d AS	632	if (type != BPF_PROG_TYPE_SOCKET_FILTER && !capable(CAP_SYS_ADMIN))
	633	return -EPERM;
	634
09756af4 AS	635	/* plain bpf_prog allocation */
	636	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
	637	if (!prog)
	638	return -ENOMEM;
	639
aaac3ba9 AS	640	err = bpf_prog_charge_memlock(prog);
	641	if (err)
	642	goto free_prog_nouncharge;
	643
09756af4 AS	644	prog->len = attr->insn_cnt;
	645
	646	err = -EFAULT;
	647	if (copy_from_user(prog->insns, u64_to_ptr(attr->insns),
	648	prog->len * sizeof(struct bpf_insn)) != 0)
	649	goto free_prog;
	650
	651	prog->orig_prog = NULL;
a91263d5	652	prog->jited = 0;
09756af4 AS	653
09756af4 AS	654	atomic_set(&prog->aux->refcnt, 1);
a91263d5	655	prog->gpl_compatible = is_gpl ? 1 : 0;
09756af4 AS	656
	657	/* find program type: socket_filter vs tracing_filter */
	658	err = find_prog_type(type, prog);
	659	if (err < 0)
	660	goto free_prog;
	661
	662	/* run eBPF verifier */
9bac3d6d	663	err = bpf_check(&prog, attr);
09756af4 AS	664	if (err < 0)
	665	goto free_used_maps;
	666
0a542a86 AS	667	/* fixup BPF_CALL->imm field */
	668	fixup_bpf_calls(prog);
	669
09756af4	670	/* eBPF program is ready to be JITed */
04fd61ab AS	671	err = bpf_prog_select_runtime(prog);
	672	if (err < 0)
	673	goto free_used_maps;
09756af4	674
aa79781b	675	err = bpf_prog_new_fd(prog);
09756af4 AS	676	if (err < 0)
	677	/* failed to allocate fd */
	678	goto free_used_maps;
	679
	680	return err;
	681
	682	free_used_maps:
	683	free_used_maps(prog->aux);
	684	free_prog:
aaac3ba9 AS	685	bpf_prog_uncharge_memlock(prog);
aaac3ba9 AS	686	free_prog_nouncharge:
09756af4 AS	687	bpf_prog_free(prog);
	688	return err;
	689	}
	690
b2197755 DB	691	#define BPF_OBJ_LAST_FIELD bpf_fd
	692
	693	static int bpf_obj_pin(const union bpf_attr *attr)
	694	{
	695	if (CHECK_ATTR(BPF_OBJ))
	696	return -EINVAL;
	697
	698	return bpf_obj_pin_user(attr->bpf_fd, u64_to_ptr(attr->pathname));
	699	}
	700
	701	static int bpf_obj_get(const union bpf_attr *attr)
	702	{
	703	if (CHECK_ATTR(BPF_OBJ) \|\| attr->bpf_fd != 0)
	704	return -EINVAL;
	705
	706	return bpf_obj_get_user(u64_to_ptr(attr->pathname));
	707	}
	708
99c55f7d AS	709	SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
	710	{
	711	union bpf_attr attr = {};
	712	int err;
	713
1be7f75d	714	if (!capable(CAP_SYS_ADMIN) && sysctl_unprivileged_bpf_disabled)
99c55f7d AS	715	return -EPERM;
	716
	717	if (!access_ok(VERIFY_READ, uattr, 1))
	718	return -EFAULT;
	719
	720	if (size > PAGE_SIZE) /* silly large */
	721	return -E2BIG;
	722
	723	/* If we're handed a bigger struct than we know of,
	724	* ensure all the unknown bits are 0 - i.e. new
	725	* user-space does not rely on any kernel feature
	726	* extensions we dont know about yet.
	727	*/
	728	if (size > sizeof(attr)) {
	729	unsigned char __user *addr;
	730	unsigned char __user *end;
	731	unsigned char val;
	732
	733	addr = (void __user *)uattr + sizeof(attr);
	734	end = (void __user *)uattr + size;
	735
	736	for (; addr < end; addr++) {
	737	err = get_user(val, addr);
	738	if (err)
	739	return err;
	740	if (val)
	741	return -E2BIG;
	742	}
	743	size = sizeof(attr);
	744	}
	745
	746	/* copy attributes from user space, may be less than sizeof(bpf_attr) */
	747	if (copy_from_user(&attr, uattr, size) != 0)
	748	return -EFAULT;
	749
	750	switch (cmd) {
	751	case BPF_MAP_CREATE:
	752	err = map_create(&attr);
	753	break;
db20fd2b AS	754	case BPF_MAP_LOOKUP_ELEM:
	755	err = map_lookup_elem(&attr);
	756	break;
	757	case BPF_MAP_UPDATE_ELEM:
	758	err = map_update_elem(&attr);
	759	break;
	760	case BPF_MAP_DELETE_ELEM:
	761	err = map_delete_elem(&attr);
	762	break;
	763	case BPF_MAP_GET_NEXT_KEY:
	764	err = map_get_next_key(&attr);
	765	break;
09756af4 AS	766	case BPF_PROG_LOAD:
	767	err = bpf_prog_load(&attr);
	768	break;
b2197755 DB	769	case BPF_OBJ_PIN:
	770	err = bpf_obj_pin(&attr);
	771	break;
	772	case BPF_OBJ_GET:
	773	err = bpf_obj_get(&attr);
	774	break;
99c55f7d AS	775	default:
	776	err = -EINVAL;
	777	break;
	778	}
	779
	780	return err;
	781	}