Signed-off-by: Jiri Slaby <jslaby@suse.cz>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-diff --git a/include/linux/filter.h b/include/linux/filter.h
-index 4ff0e647598f..c4aae496f376 100644
+
+---
+ include/linux/filter.h | 3 ++
+ kernel/bpf/core.c | 71 +++++++++++++++++++++++++++++++++++++++++++++++++
+ kernel/bpf/verifier.c | 53 ++++++------------------------------
+ 3 files changed, 83 insertions(+), 44 deletions(-)
+
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
-@@ -495,6 +495,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
+@@ -466,6 +466,9 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64
void bpf_int_jit_compile(struct bpf_prog *fp);
bool bpf_helper_changes_skb_data(void *func);
+ const struct bpf_insn *patch, u32 len);
+
#ifdef CONFIG_BPF_JIT
- extern int bpf_jit_enable;
+ typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
-diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
-index 5313d09d4b62..49b5538a5301 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
-@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp)
- vfree(fp);
+@@ -137,6 +137,77 @@ void __bpf_prog_free(struct bpf_prog *fp
}
+ EXPORT_SYMBOL_GPL(__bpf_prog_free);
+static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
+{
#ifdef CONFIG_BPF_JIT
struct bpf_binary_header *
bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
-diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
-index 84bff68cf80e..a08d66215245 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
-@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env)
+@@ -2098,26 +2098,6 @@ static void convert_pseudo_ld_imm64(stru
insn->src_reg = 0;
}
/* convert load instructions that access fields of 'struct __sk_buff'
* into sequence of instructions that access fields of 'struct sk_buff'
*/
-@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env)
+@@ -2127,14 +2107,15 @@ static int convert_ctx_accesses(struct v
int insn_cnt = env->prog->len;
struct bpf_insn insn_buf[16];
struct bpf_prog *new_prog;
if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))
type = BPF_READ;
else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))
-@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env)
+@@ -2156,34 +2137,18 @@ static int convert_ctx_accesses(struct v
return -EINVAL;
}
--- /dev/null
+From 8041902dae5299c1f194ba42d14383f734631009 Mon Sep 17 00:00:00 2001
+From: Alexei Starovoitov <ast@fb.com>
+Date: Wed, 15 Mar 2017 18:26:41 -0700
+Subject: bpf: adjust insn_aux_data when patching insns
+
+From: Alexei Starovoitov <ast@fb.com>
+
+commit 8041902dae5299c1f194ba42d14383f734631009 upstream.
+
+convert_ctx_accesses() replaces single bpf instruction with a set of
+instructions. Adjust corresponding insn_aux_data while patching.
+It's needed to make sure subsequent 'for(all insn)' loops
+have matching insn and insn_aux_data.
+
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Acked-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ kernel/bpf/verifier.c | 40 +++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 37 insertions(+), 3 deletions(-)
+
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -2107,6 +2107,41 @@ static void convert_pseudo_ld_imm64(stru
+ insn->src_reg = 0;
+ }
+
++/* single env->prog->insni[off] instruction was replaced with the range
++ * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying
++ * [0, off) and [off, end) to new locations, so the patched range stays zero
++ */
++static int adjust_insn_aux_data(struct verifier_env *env, u32 prog_len,
++ u32 off, u32 cnt)
++{
++ struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
++
++ if (cnt == 1)
++ return 0;
++ new_data = vzalloc(sizeof(struct bpf_insn_aux_data) * prog_len);
++ if (!new_data)
++ return -ENOMEM;
++ memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
++ memcpy(new_data + off + cnt - 1, old_data + off,
++ sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
++ env->insn_aux_data = new_data;
++ vfree(old_data);
++ return 0;
++}
++
++static struct bpf_prog *bpf_patch_insn_data(struct verifier_env *env, u32 off,
++ const struct bpf_insn *patch, u32 len)
++{
++ struct bpf_prog *new_prog;
++
++ new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
++ if (!new_prog)
++ return NULL;
++ if (adjust_insn_aux_data(env, new_prog->len, off, len))
++ return NULL;
++ return new_prog;
++}
++
+ /* convert load instructions that access fields of 'struct __sk_buff'
+ * into sequence of instructions that access fields of 'struct sk_buff'
+ */
+@@ -2132,7 +2167,7 @@ static int convert_ctx_accesses(struct v
+ else
+ continue;
+
+- if (env->insn_aux_data[i].ptr_type != PTR_TO_CTX)
++ if (env->insn_aux_data[i + delta].ptr_type != PTR_TO_CTX)
+ continue;
+
+ cnt = env->prog->aux->ops->
+@@ -2143,8 +2178,7 @@ static int convert_ctx_accesses(struct v
+ return -EINVAL;
+ }
+
+- new_prog = bpf_patch_insn_single(env->prog, i + delta, insn_buf,
+- cnt);
++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
+ if (!new_prog)
+ return -ENOMEM;
+
--- /dev/null
+From jslaby@suse.cz Sat Jan 13 16:20:05 2018
+From: Jiri Slaby <jslaby@suse.cz>
+Date: Fri, 12 Jan 2018 17:58:05 +0100
+Subject: bpf, array: fix overflow in max_entries and undefined behavior in index_mask
+To: gregkh@linuxfoundation.org
+Cc: stable@vger.kernel.org, ast@kernel.org, netdev@vger.kernel.org, Daniel Borkmann <daniel@iogearbox.net>, Jiri Slaby <jslaby@suse.cz>
+Message-ID: <20180112165805.10791-1-jslaby@suse.cz>
+
+From: Daniel Borkmann <daniel@iogearbox.net>
+
+commit bbeb6e4323dad9b5e0ee9f60c223dd532e2403b1 upstream.
+
+syzkaller tried to alloc a map with 0xfffffffd entries out of a userns,
+and thus unprivileged. With the recently added logic in b2157399cc98
+("bpf: prevent out-of-bounds speculation") we round this up to the next
+power of two value for max_entries for unprivileged such that we can
+apply proper masking into potentially zeroed out map slots.
+
+However, this will generate an index_mask of 0xffffffff, and therefore
+a + 1 will let this overflow into new max_entries of 0. This will pass
+allocation, etc, and later on map access we still enforce on the original
+attr->max_entries value which was 0xfffffffd, therefore triggering GPF
+all over the place. Thus bail out on overflow in such case.
+
+Moreover, on 32 bit archs roundup_pow_of_two() can also not be used,
+since fls_long(max_entries - 1) can result in 32 and 1UL << 32 in 32 bit
+space is undefined. Therefore, do this by hand in a 64 bit variable.
+
+This fixes all the issues triggered by syzkaller's reproducers.
+
+Fixes: b2157399cc98 ("bpf: prevent out-of-bounds speculation")
+Reported-by: syzbot+b0efb8e572d01bce1ae0@syzkaller.appspotmail.com
+Reported-by: syzbot+6c15e9744f75f2364773@syzkaller.appspotmail.com
+Reported-by: syzbot+d2f5524fb46fd3b312ee@syzkaller.appspotmail.com
+Reported-by: syzbot+61d23c95395cc90dbc2b@syzkaller.appspotmail.com
+Reported-by: syzbot+0d363c942452cca68c01@syzkaller.appspotmail.com
+Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/bpf/arraymap.c | 17 +++++++++++++++--
+ 1 file changed, 15 insertions(+), 2 deletions(-)
+
+--- a/kernel/bpf/arraymap.c
++++ b/kernel/bpf/arraymap.c
+@@ -23,6 +23,7 @@ static struct bpf_map *array_map_alloc(u
+ u32 elem_size, array_size, index_mask, max_entries;
+ bool unpriv = !capable(CAP_SYS_ADMIN);
+ struct bpf_array *array;
++ u64 mask64;
+
+ /* check sanity of attributes */
+ if (attr->max_entries == 0 || attr->key_size != 4 ||
+@@ -38,13 +39,25 @@ static struct bpf_map *array_map_alloc(u
+ elem_size = round_up(attr->value_size, 8);
+
+ max_entries = attr->max_entries;
+- index_mask = roundup_pow_of_two(max_entries) - 1;
+
+- if (unpriv)
++ /* On 32 bit archs roundup_pow_of_two() with max_entries that has
++ * upper most bit set in u32 space is undefined behavior due to
++ * resulting 1U << 32, so do it manually here in u64 space.
++ */
++ mask64 = fls_long(max_entries - 1);
++ mask64 = 1ULL << mask64;
++ mask64 -= 1;
++
++ index_mask = mask64;
++ if (unpriv) {
+ /* round up array size to nearest power of 2,
+ * since cpu will speculate within index_mask limits
+ */
+ max_entries = index_mask + 1;
++ /* Check for overflows. */
++ if (max_entries < attr->max_entries)
++ return ERR_PTR(-E2BIG);
++ }
+
+ /* check round_up into zero and u32 overflow */
+ if (elem_size == 0 ||
--- /dev/null
+From 3df126f35f88dc76eea33769f85a3c3bb8ce6c6b Mon Sep 17 00:00:00 2001
+From: Jakub Kicinski <jakub.kicinski@netronome.com>
+Date: Wed, 21 Sep 2016 11:43:56 +0100
+Subject: bpf: don't (ab)use instructions to store state
+
+From: Jakub Kicinski <jakub.kicinski@netronome.com>
+
+commit 3df126f35f88dc76eea33769f85a3c3bb8ce6c6b upstream.
+
+Storing state in reserved fields of instructions makes
+it impossible to run verifier on programs already
+marked as read-only. Allocate and use an array of
+per-instruction state instead.
+
+While touching the error path rename and move existing
+jump target.
+
+Suggested-by: Alexei Starovoitov <ast@kernel.org>
+Signed-off-by: Jakub Kicinski <jakub.kicinski@netronome.com>
+Acked-by: Alexei Starovoitov <ast@kernel.org>
+Acked-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ kernel/bpf/verifier.c | 67 +++++++++++++++++++++++++++++---------------------
+ 1 file changed, 39 insertions(+), 28 deletions(-)
+
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -186,6 +186,10 @@ struct verifier_stack_elem {
+ struct verifier_stack_elem *next;
+ };
+
++struct bpf_insn_aux_data {
++ enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
++};
++
+ #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
+
+ /* single container for all structs
+@@ -200,6 +204,7 @@ struct verifier_env {
+ struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */
+ u32 used_map_cnt; /* number of used maps */
+ bool allow_ptr_leaks;
++ struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
+ };
+
+ /* verbose verifier prints what it's seeing
+@@ -1784,7 +1789,7 @@ static int do_check(struct verifier_env
+ return err;
+
+ } else if (class == BPF_LDX) {
+- enum bpf_reg_type src_reg_type;
++ enum bpf_reg_type *prev_src_type, src_reg_type;
+
+ /* check for reserved fields is already done */
+
+@@ -1813,16 +1818,18 @@ static int do_check(struct verifier_env
+ continue;
+ }
+
+- if (insn->imm == 0) {
++ prev_src_type = &env->insn_aux_data[insn_idx].ptr_type;
++
++ if (*prev_src_type == NOT_INIT) {
+ /* saw a valid insn
+ * dst_reg = *(u32 *)(src_reg + off)
+- * use reserved 'imm' field to mark this insn
++ * save type to validate intersecting paths
+ */
+- insn->imm = src_reg_type;
++ *prev_src_type = src_reg_type;
+
+- } else if (src_reg_type != insn->imm &&
++ } else if (src_reg_type != *prev_src_type &&
+ (src_reg_type == PTR_TO_CTX ||
+- insn->imm == PTR_TO_CTX)) {
++ *prev_src_type == PTR_TO_CTX)) {
+ /* ABuser program is trying to use the same insn
+ * dst_reg = *(u32*) (src_reg + off)
+ * with different pointer types:
+@@ -1835,7 +1842,7 @@ static int do_check(struct verifier_env
+ }
+
+ } else if (class == BPF_STX) {
+- enum bpf_reg_type dst_reg_type;
++ enum bpf_reg_type *prev_dst_type, dst_reg_type;
+
+ if (BPF_MODE(insn->code) == BPF_XADD) {
+ err = check_xadd(env, insn);
+@@ -1863,11 +1870,13 @@ static int do_check(struct verifier_env
+ if (err)
+ return err;
+
+- if (insn->imm == 0) {
+- insn->imm = dst_reg_type;
+- } else if (dst_reg_type != insn->imm &&
++ prev_dst_type = &env->insn_aux_data[insn_idx].ptr_type;
++
++ if (*prev_dst_type == NOT_INIT) {
++ *prev_dst_type = dst_reg_type;
++ } else if (dst_reg_type != *prev_dst_type &&
+ (dst_reg_type == PTR_TO_CTX ||
+- insn->imm == PTR_TO_CTX)) {
++ *prev_dst_type == PTR_TO_CTX)) {
+ verbose("same insn cannot be used with different pointers\n");
+ return -EINVAL;
+ }
+@@ -2104,17 +2113,17 @@ static void convert_pseudo_ld_imm64(stru
+ static int convert_ctx_accesses(struct verifier_env *env)
+ {
+ struct bpf_insn *insn = env->prog->insnsi;
+- int insn_cnt = env->prog->len;
++ const int insn_cnt = env->prog->len;
+ struct bpf_insn insn_buf[16];
+ struct bpf_prog *new_prog;
+ enum bpf_access_type type;
+- int i;
++ int i, delta = 0;
+
+ if (!env->prog->aux->ops->convert_ctx_access)
+ return 0;
+
+ for (i = 0; i < insn_cnt; i++, insn++) {
+- u32 insn_delta, cnt;
++ u32 cnt;
+
+ if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))
+ type = BPF_READ;
+@@ -2123,11 +2132,8 @@ static int convert_ctx_accesses(struct v
+ else
+ continue;
+
+- if (insn->imm != PTR_TO_CTX) {
+- /* clear internal mark */
+- insn->imm = 0;
++ if (env->insn_aux_data[i].ptr_type != PTR_TO_CTX)
+ continue;
+- }
+
+ cnt = env->prog->aux->ops->
+ convert_ctx_access(type, insn->dst_reg, insn->src_reg,
+@@ -2137,18 +2143,16 @@ static int convert_ctx_accesses(struct v
+ return -EINVAL;
+ }
+
+- new_prog = bpf_patch_insn_single(env->prog, i, insn_buf, cnt);
++ new_prog = bpf_patch_insn_single(env->prog, i + delta, insn_buf,
++ cnt);
+ if (!new_prog)
+ return -ENOMEM;
+
+- insn_delta = cnt - 1;
++ delta += cnt - 1;
+
+ /* keep walking new program and skip insns we just inserted */
+ env->prog = new_prog;
+- insn = new_prog->insnsi + i + insn_delta;
+-
+- insn_cnt += insn_delta;
+- i += insn_delta;
++ insn = new_prog->insnsi + i + delta;
+ }
+
+ return 0;
+@@ -2192,6 +2196,11 @@ int bpf_check(struct bpf_prog **prog, un
+ if (!env)
+ return -ENOMEM;
+
++ env->insn_aux_data = vzalloc(sizeof(struct bpf_insn_aux_data) *
++ (*prog)->len);
++ ret = -ENOMEM;
++ if (!env->insn_aux_data)
++ goto err_free_env;
+ env->prog = *prog;
+
+ /* grab the mutex to protect few globals used by verifier */
+@@ -2210,12 +2219,12 @@ int bpf_check(struct bpf_prog **prog, un
+ /* log_* values have to be sane */
+ if (log_size < 128 || log_size > UINT_MAX >> 8 ||
+ log_level == 0 || log_ubuf == NULL)
+- goto free_env;
++ goto err_unlock;
+
+ ret = -ENOMEM;
+ log_buf = vmalloc(log_size);
+ if (!log_buf)
+- goto free_env;
++ goto err_unlock;
+ } else {
+ log_level = 0;
+ }
+@@ -2284,14 +2293,16 @@ skip_full_check:
+ free_log_buf:
+ if (log_level)
+ vfree(log_buf);
+-free_env:
+ if (!env->prog->aux->used_maps)
+ /* if we didn't copy map pointers into bpf_prog_info, release
+ * them now. Otherwise free_bpf_prog_info() will release them.
+ */
+ release_maps(env);
+ *prog = env->prog;
+- kfree(env);
++err_unlock:
+ mutex_unlock(&bpf_verifier_lock);
++ vfree(env->insn_aux_data);
++err_free_env:
++ kfree(env);
+ return ret;
+ }
--- /dev/null
+From e245c5c6a5656e4d61aa7bb08e9694fd6e5b2b9d Mon Sep 17 00:00:00 2001
+From: Alexei Starovoitov <ast@fb.com>
+Date: Wed, 15 Mar 2017 18:26:39 -0700
+Subject: bpf: move fixup_bpf_calls() function
+
+From: Alexei Starovoitov <ast@fb.com>
+
+commit e245c5c6a5656e4d61aa7bb08e9694fd6e5b2b9d upstream.
+
+no functional change.
+move fixup_bpf_calls() to verifier.c
+it's being refactored in the next patch
+
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Acked-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ kernel/bpf/syscall.c | 54 -------------------------------------------------
+ kernel/bpf/verifier.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 55 insertions(+), 54 deletions(-)
+
+--- a/kernel/bpf/syscall.c
++++ b/kernel/bpf/syscall.c
+@@ -447,57 +447,6 @@ void bpf_register_prog_type(struct bpf_p
+ list_add(&tl->list_node, &bpf_prog_types);
+ }
+
+-/* fixup insn->imm field of bpf_call instructions:
+- * if (insn->imm == BPF_FUNC_map_lookup_elem)
+- * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
+- * else if (insn->imm == BPF_FUNC_map_update_elem)
+- * insn->imm = bpf_map_update_elem - __bpf_call_base;
+- * else ...
+- *
+- * this function is called after eBPF program passed verification
+- */
+-static void fixup_bpf_calls(struct bpf_prog *prog)
+-{
+- const struct bpf_func_proto *fn;
+- int i;
+-
+- for (i = 0; i < prog->len; i++) {
+- struct bpf_insn *insn = &prog->insnsi[i];
+-
+- if (insn->code == (BPF_JMP | BPF_CALL)) {
+- /* we reach here when program has bpf_call instructions
+- * and it passed bpf_check(), means that
+- * ops->get_func_proto must have been supplied, check it
+- */
+- BUG_ON(!prog->aux->ops->get_func_proto);
+-
+- if (insn->imm == BPF_FUNC_get_route_realm)
+- prog->dst_needed = 1;
+- if (insn->imm == BPF_FUNC_get_prandom_u32)
+- bpf_user_rnd_init_once();
+- if (insn->imm == BPF_FUNC_tail_call) {
+- /* mark bpf_tail_call as different opcode
+- * to avoid conditional branch in
+- * interpeter for every normal call
+- * and to prevent accidental JITing by
+- * JIT compiler that doesn't support
+- * bpf_tail_call yet
+- */
+- insn->imm = 0;
+- insn->code |= BPF_X;
+- continue;
+- }
+-
+- fn = prog->aux->ops->get_func_proto(insn->imm);
+- /* all functions that have prototype and verifier allowed
+- * programs to call them, must be real in-kernel functions
+- */
+- BUG_ON(!fn->func);
+- insn->imm = fn->func - __bpf_call_base;
+- }
+- }
+-}
+-
+ /* drop refcnt on maps used by eBPF program and free auxilary data */
+ static void free_used_maps(struct bpf_prog_aux *aux)
+ {
+@@ -680,9 +629,6 @@ static int bpf_prog_load(union bpf_attr
+ if (err < 0)
+ goto free_used_maps;
+
+- /* fixup BPF_CALL->imm field */
+- fixup_bpf_calls(prog);
+-
+ /* eBPF program is ready to be JITed */
+ err = bpf_prog_select_runtime(prog);
+ if (err < 0)
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -2158,6 +2158,58 @@ static int convert_ctx_accesses(struct v
+ return 0;
+ }
+
++/* fixup insn->imm field of bpf_call instructions:
++ * if (insn->imm == BPF_FUNC_map_lookup_elem)
++ * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
++ * else if (insn->imm == BPF_FUNC_map_update_elem)
++ * insn->imm = bpf_map_update_elem - __bpf_call_base;
++ * else ...
++ *
++ * this function is called after eBPF program passed verification
++ */
++static void fixup_bpf_calls(struct bpf_prog *prog)
++{
++ const struct bpf_func_proto *fn;
++ int i;
++
++ for (i = 0; i < prog->len; i++) {
++ struct bpf_insn *insn = &prog->insnsi[i];
++
++ if (insn->code == (BPF_JMP | BPF_CALL)) {
++ /* we reach here when program has bpf_call instructions
++ * and it passed bpf_check(), means that
++ * ops->get_func_proto must have been supplied, check it
++ */
++ BUG_ON(!prog->aux->ops->get_func_proto);
++
++ if (insn->imm == BPF_FUNC_get_route_realm)
++ prog->dst_needed = 1;
++ if (insn->imm == BPF_FUNC_get_prandom_u32)
++ bpf_user_rnd_init_once();
++ if (insn->imm == BPF_FUNC_tail_call) {
++ /* mark bpf_tail_call as different opcode
++ * to avoid conditional branch in
++ * interpeter for every normal call
++ * and to prevent accidental JITing by
++ * JIT compiler that doesn't support
++ * bpf_tail_call yet
++ */
++ insn->imm = 0;
++ insn->code |= BPF_X;
++ continue;
++ }
++
++ fn = prog->aux->ops->get_func_proto(insn->imm);
++ /* all functions that have prototype and verifier allowed
++ * programs to call them, must be real in-kernel functions
++ */
++ BUG_ON(!fn->func);
++ insn->imm = fn->func - __bpf_call_base;
++ }
++ }
++}
++
++
+ static void free_states(struct verifier_env *env)
+ {
+ struct verifier_state_list *sl, *sln;
+@@ -2256,6 +2308,9 @@ skip_full_check:
+ /* program is valid, convert *(u32*)(ctx + off) accesses */
+ ret = convert_ctx_accesses(env);
+
++ if (ret == 0)
++ fixup_bpf_calls(env->prog);
++
+ if (log_level && log_len >= log_size - 1) {
+ BUG_ON(log_len >= log_size);
+ /* verifier log exceeded user supplied buffer */
--- /dev/null
+From b2157399cc9898260d6031c5bfe45fe137c1fbe7 Mon Sep 17 00:00:00 2001
+From: Alexei Starovoitov <ast@kernel.org>
+Date: Sun, 7 Jan 2018 17:33:02 -0800
+Subject: bpf: prevent out-of-bounds speculation
+
+From: Alexei Starovoitov <ast@kernel.org>
+
+commit b2157399cc9898260d6031c5bfe45fe137c1fbe7 upstream.
+
+Under speculation, CPUs may mis-predict branches in bounds checks. Thus,
+memory accesses under a bounds check may be speculated even if the
+bounds check fails, providing a primitive for building a side channel.
+
+To avoid leaking kernel data round up array-based maps and mask the index
+after bounds check, so speculated load with out of bounds index will load
+either valid value from the array or zero from the padded area.
+
+Unconditionally mask index for all array types even when max_entries
+are not rounded to power of 2 for root user.
+When map is created by unpriv user generate a sequence of bpf insns
+that includes AND operation to make sure that JITed code includes
+the same 'index & index_mask' operation.
+
+If prog_array map is created by unpriv user replace
+ bpf_tail_call(ctx, map, index);
+with
+ if (index >= max_entries) {
+ index &= map->index_mask;
+ bpf_tail_call(ctx, map, index);
+ }
+(along with roundup to power 2) to prevent out-of-bounds speculation.
+There is secondary redundant 'if (index >= max_entries)' in the interpreter
+and in all JITs, but they can be optimized later if necessary.
+
+Other array-like maps (cpumap, devmap, sockmap, perf_event_array, cgroup_array)
+cannot be used by unpriv, so no changes there.
+
+That fixes bpf side of "Variant 1: bounds check bypass (CVE-2017-5753)" on
+all architectures with and without JIT.
+
+v2->v3:
+Daniel noticed that attack potentially can be crafted via syscall commands
+without loading the program, so add masking to those paths as well.
+
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Acked-by: John Fastabend <john.fastabend@gmail.com>
+Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ include/linux/bpf.h | 2 ++
+ kernel/bpf/arraymap.c | 24 +++++++++++++++++++-----
+ kernel/bpf/verifier.c | 46 ++++++++++++++++++++++++++++++++++++++++++----
+ 3 files changed, 63 insertions(+), 9 deletions(-)
+
+--- a/include/linux/bpf.h
++++ b/include/linux/bpf.h
+@@ -37,6 +37,7 @@ struct bpf_map {
+ u32 value_size;
+ u32 max_entries;
+ u32 pages;
++ bool unpriv_array;
+ struct user_struct *user;
+ const struct bpf_map_ops *ops;
+ struct work_struct work;
+@@ -141,6 +142,7 @@ struct bpf_prog_aux {
+ struct bpf_array {
+ struct bpf_map map;
+ u32 elem_size;
++ u32 index_mask;
+ /* 'ownership' of prog_array is claimed by the first program that
+ * is going to use this map or by the first program which FD is stored
+ * in the map to make sure that all callers and callees have the same
+--- a/kernel/bpf/arraymap.c
++++ b/kernel/bpf/arraymap.c
+@@ -20,8 +20,9 @@
+ /* Called from syscall */
+ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
+ {
++ u32 elem_size, array_size, index_mask, max_entries;
++ bool unpriv = !capable(CAP_SYS_ADMIN);
+ struct bpf_array *array;
+- u32 elem_size, array_size;
+
+ /* check sanity of attributes */
+ if (attr->max_entries == 0 || attr->key_size != 4 ||
+@@ -36,12 +37,21 @@ static struct bpf_map *array_map_alloc(u
+
+ elem_size = round_up(attr->value_size, 8);
+
++ max_entries = attr->max_entries;
++ index_mask = roundup_pow_of_two(max_entries) - 1;
++
++ if (unpriv)
++ /* round up array size to nearest power of 2,
++ * since cpu will speculate within index_mask limits
++ */
++ max_entries = index_mask + 1;
++
+ /* check round_up into zero and u32 overflow */
+ if (elem_size == 0 ||
+- attr->max_entries > (U32_MAX - PAGE_SIZE - sizeof(*array)) / elem_size)
++ max_entries > (U32_MAX - PAGE_SIZE - sizeof(*array)) / elem_size)
+ return ERR_PTR(-ENOMEM);
+
+- array_size = sizeof(*array) + attr->max_entries * elem_size;
++ array_size = sizeof(*array) + max_entries * elem_size;
+
+ /* allocate all map elements and zero-initialize them */
+ array = kzalloc(array_size, GFP_USER | __GFP_NOWARN);
+@@ -50,6 +60,8 @@ static struct bpf_map *array_map_alloc(u
+ if (!array)
+ return ERR_PTR(-ENOMEM);
+ }
++ array->index_mask = index_mask;
++ array->map.unpriv_array = unpriv;
+
+ /* copy mandatory map attributes */
+ array->map.key_size = attr->key_size;
+@@ -70,7 +82,7 @@ static void *array_map_lookup_elem(struc
+ if (index >= array->map.max_entries)
+ return NULL;
+
+- return array->value + array->elem_size * index;
++ return array->value + array->elem_size * (index & array->index_mask);
+ }
+
+ /* Called from syscall */
+@@ -111,7 +123,9 @@ static int array_map_update_elem(struct
+ /* all elements already exist */
+ return -EEXIST;
+
+- memcpy(array->value + array->elem_size * index, value, map->value_size);
++ memcpy(array->value +
++ array->elem_size * (index & array->index_mask),
++ value, map->value_size);
+ return 0;
+ }
+
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -187,7 +187,10 @@ struct verifier_stack_elem {
+ };
+
+ struct bpf_insn_aux_data {
+- enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
++ union {
++ enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
++ struct bpf_map *map_ptr; /* pointer for call insn into lookup_elem */
++ };
+ };
+
+ #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
+@@ -950,7 +953,7 @@ error:
+ return -EINVAL;
+ }
+
+-static int check_call(struct verifier_env *env, int func_id)
++static int check_call(struct verifier_env *env, int func_id, int insn_idx)
+ {
+ struct verifier_state *state = &env->cur_state;
+ const struct bpf_func_proto *fn = NULL;
+@@ -986,6 +989,13 @@ static int check_call(struct verifier_en
+ err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &map);
+ if (err)
+ return err;
++ if (func_id == BPF_FUNC_tail_call) {
++ if (map == NULL) {
++ verbose("verifier bug\n");
++ return -EINVAL;
++ }
++ env->insn_aux_data[insn_idx].map_ptr = map;
++ }
+ err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &map);
+ if (err)
+ return err;
+@@ -1911,7 +1921,7 @@ static int do_check(struct verifier_env
+ return -EINVAL;
+ }
+
+- err = check_call(env, insn->imm);
++ err = check_call(env, insn->imm, insn_idx);
+ if (err)
+ return err;
+
+@@ -2202,7 +2212,10 @@ static int fixup_bpf_calls(struct verifi
+ struct bpf_insn *insn = prog->insnsi;
+ const struct bpf_func_proto *fn;
+ const int insn_cnt = prog->len;
+- int i;
++ struct bpf_insn insn_buf[16];
++ struct bpf_prog *new_prog;
++ struct bpf_map *map_ptr;
++ int i, cnt, delta = 0;
+
+ for (i = 0; i < insn_cnt; i++, insn++) {
+ if (insn->code != (BPF_JMP | BPF_CALL))
+@@ -2220,6 +2233,31 @@ static int fixup_bpf_calls(struct verifi
+ */
+ insn->imm = 0;
+ insn->code |= BPF_X;
++
++ /* instead of changing every JIT dealing with tail_call
++ * emit two extra insns:
++ * if (index >= max_entries) goto out;
++ * index &= array->index_mask;
++ * to avoid out-of-bounds cpu speculation
++ */
++ map_ptr = env->insn_aux_data[i + delta].map_ptr;
++ if (!map_ptr->unpriv_array)
++ continue;
++ insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
++ map_ptr->max_entries, 2);
++ insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
++ container_of(map_ptr,
++ struct bpf_array,
++ map)->index_mask);
++ insn_buf[2] = *insn;
++ cnt = 3;
++ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
++ if (!new_prog)
++ return -ENOMEM;
++
++ delta += cnt - 1;
++ env->prog = prog = new_prog;
++ insn = new_prog->insnsi + i + delta;
+ continue;
+ }
+
--- /dev/null
+From 79741b3bdec01a8628368fbcfccc7d189ed606cb Mon Sep 17 00:00:00 2001
+From: Alexei Starovoitov <ast@fb.com>
+Date: Wed, 15 Mar 2017 18:26:40 -0700
+Subject: bpf: refactor fixup_bpf_calls()
+
+From: Alexei Starovoitov <ast@fb.com>
+
+commit 79741b3bdec01a8628368fbcfccc7d189ed606cb upstream.
+
+reduce indent and make it iterate over instructions similar to
+convert_ctx_accesses(). Also convert hard BUG_ON into soft verifier error.
+
+Signed-off-by: Alexei Starovoitov <ast@kernel.org>
+Acked-by: Daniel Borkmann <daniel@iogearbox.net>
+Signed-off-by: David S. Miller <davem@davemloft.net>
+Signed-off-by: Jiri Slaby <jslaby@suse.cz>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ kernel/bpf/verifier.c | 74 ++++++++++++++++++++++----------------------------
+ 1 file changed, 34 insertions(+), 40 deletions(-)
+
+--- a/kernel/bpf/verifier.c
++++ b/kernel/bpf/verifier.c
+@@ -2158,57 +2158,51 @@ static int convert_ctx_accesses(struct v
+ return 0;
+ }
+
+-/* fixup insn->imm field of bpf_call instructions:
+- * if (insn->imm == BPF_FUNC_map_lookup_elem)
+- * insn->imm = bpf_map_lookup_elem - __bpf_call_base;
+- * else if (insn->imm == BPF_FUNC_map_update_elem)
+- * insn->imm = bpf_map_update_elem - __bpf_call_base;
+- * else ...
++/* fixup insn->imm field of bpf_call instructions
+ *
+ * this function is called after eBPF program passed verification
+ */
+-static void fixup_bpf_calls(struct bpf_prog *prog)
++static int fixup_bpf_calls(struct verifier_env *env)
+ {
++ struct bpf_prog *prog = env->prog;
++ struct bpf_insn *insn = prog->insnsi;
+ const struct bpf_func_proto *fn;
++ const int insn_cnt = prog->len;
+ int i;
+
+- for (i = 0; i < prog->len; i++) {
+- struct bpf_insn *insn = &prog->insnsi[i];
+-
+- if (insn->code == (BPF_JMP | BPF_CALL)) {
+- /* we reach here when program has bpf_call instructions
+- * and it passed bpf_check(), means that
+- * ops->get_func_proto must have been supplied, check it
++ for (i = 0; i < insn_cnt; i++, insn++) {
++ if (insn->code != (BPF_JMP | BPF_CALL))
++ continue;
++
++ if (insn->imm == BPF_FUNC_get_route_realm)
++ prog->dst_needed = 1;
++ if (insn->imm == BPF_FUNC_get_prandom_u32)
++ bpf_user_rnd_init_once();
++ if (insn->imm == BPF_FUNC_tail_call) {
++ /* mark bpf_tail_call as different opcode to avoid
++ * conditional branch in the interpeter for every normal
++ * call and to prevent accidental JITing by JIT compiler
++ * that doesn't support bpf_tail_call yet
+ */
+- BUG_ON(!prog->aux->ops->get_func_proto);
+-
+- if (insn->imm == BPF_FUNC_get_route_realm)
+- prog->dst_needed = 1;
+- if (insn->imm == BPF_FUNC_get_prandom_u32)
+- bpf_user_rnd_init_once();
+- if (insn->imm == BPF_FUNC_tail_call) {
+- /* mark bpf_tail_call as different opcode
+- * to avoid conditional branch in
+- * interpeter for every normal call
+- * and to prevent accidental JITing by
+- * JIT compiler that doesn't support
+- * bpf_tail_call yet
+- */
+- insn->imm = 0;
+- insn->code |= BPF_X;
+- continue;
+- }
++ insn->imm = 0;
++ insn->code |= BPF_X;
++ continue;
++ }
+
+- fn = prog->aux->ops->get_func_proto(insn->imm);
+- /* all functions that have prototype and verifier allowed
+- * programs to call them, must be real in-kernel functions
+- */
+- BUG_ON(!fn->func);
+- insn->imm = fn->func - __bpf_call_base;
++ fn = prog->aux->ops->get_func_proto(insn->imm);
++ /* all functions that have prototype and verifier allowed
++ * programs to call them, must be real in-kernel functions
++ */
++ if (!fn->func) {
++ verbose("kernel subsystem misconfigured func %d\n",
++ insn->imm);
++ return -EFAULT;
+ }
++ insn->imm = fn->func - __bpf_call_base;
+ }
+-}
+
++ return 0;
++}
+
+ static void free_states(struct verifier_env *env)
+ {
+@@ -2309,7 +2303,7 @@ skip_full_check:
+ ret = convert_ctx_accesses(env);
+
+ if (ret == 0)
+- fixup_bpf_calls(env->prog);
++ ret = fixup_bpf_calls(env);
+
+ if (log_level && log_len >= log_size - 1) {
+ BUG_ON(log_len >= log_size);
drm-vmwgfx-potential-off-by-one-in-vmw_view_add.patch
kaiser-set-_page_nx-only-if-supported.patch
bpf-add-bpf_patch_insn_single-helper.patch
+bpf-don-t-ab-use-instructions-to-store-state.patch
+bpf-move-fixup_bpf_calls-function.patch
+bpf-refactor-fixup_bpf_calls.patch
+bpf-adjust-insn_aux_data-when-patching-insns.patch
+bpf-prevent-out-of-bounds-speculation.patch
+bpf-array-fix-overflow-in-max_entries-and-undefined-behavior-in-index_mask.patch
projects / thirdparty / kernel / stable-queue.git / commitdiff
