#define BPF_PRIV_STACK_MIN_SIZE 64
-static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
-static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id);
-static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
+static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id);
+static int release_reference_nomark(struct bpf_verifier_state *state, int id);
+static int release_reference(struct bpf_verifier_env *env, int id);
static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
static int ref_set_non_owning(struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
-static bool is_trusted_reg(const struct bpf_reg_state *reg);
+static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg);
static inline bool in_sleepable_context(struct bpf_verifier_env *env);
static const char *non_sleepable_context_description(struct bpf_verifier_env *env);
static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, struct bpf_reg_state *src_reg);
int access_size;
int mem_size;
u64 msize_max_value;
- int ref_obj_id;
+ u32 id;
int func_id;
struct btf *btf;
u32 btf_id;
verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval);
}
-static bool reg_not_null(const struct bpf_reg_state *reg)
+static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
{
enum bpf_reg_type type;
type == PTR_TO_MAP_VALUE ||
type == PTR_TO_MAP_KEY ||
type == PTR_TO_SOCK_COMMON ||
- (type == PTR_TO_BTF_ID && is_trusted_reg(reg)) ||
+ (type == PTR_TO_BTF_ID && is_trusted_reg(env, reg)) ||
(type == PTR_TO_MEM && !(reg->type & PTR_UNTRUSTED)) ||
type == CONST_PTR_TO_MAP;
}
}
}
-static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
+static bool dynptr_type_referenced(enum bpf_dynptr_type type)
{
return type == BPF_DYNPTR_TYPE_RINGBUF || type == BPF_DYNPTR_TYPE_FILE;
}
static void __mark_dynptr_reg(struct bpf_reg_state *reg,
enum bpf_dynptr_type type,
- bool first_slot, int dynptr_id);
+ bool first_slot, int id, int parent_id);
static void mark_dynptr_stack_regs(struct bpf_verifier_env *env,
struct bpf_reg_state *sreg1,
struct bpf_reg_state *sreg2,
- enum bpf_dynptr_type type)
+ enum bpf_dynptr_type type, int parent_id)
{
int id = ++env->id_gen;
- __mark_dynptr_reg(sreg1, type, true, id);
- __mark_dynptr_reg(sreg2, type, false, id);
+ __mark_dynptr_reg(sreg1, type, true, id, parent_id);
+ __mark_dynptr_reg(sreg2, type, false, id, parent_id);
}
static void mark_dynptr_cb_reg(struct bpf_verifier_env *env,
struct bpf_reg_state *reg,
enum bpf_dynptr_type type)
{
- __mark_dynptr_reg(reg, type, true, ++env->id_gen);
+ __mark_dynptr_reg(reg, type, true, ++env->id_gen, 0);
}
static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
struct bpf_func_state *state, int spi);
static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
- enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)
+ enum bpf_arg_type arg_type, int insn_idx, int parent_id,
+ struct bpf_dynptr_desc *dynptr)
{
struct bpf_func_state *state = bpf_func(env, reg);
- enum bpf_dynptr_type type;
int spi, i, err;
+ enum bpf_dynptr_type type;
spi = dynptr_get_spi(env, reg);
if (spi < 0)
if (type == BPF_DYNPTR_TYPE_INVALID)
return -EINVAL;
- mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
- &state->stack[spi - 1].spilled_ptr, type);
-
- if (dynptr_type_refcounted(type)) {
- /* The id is used to track proper releasing */
- int id;
+ if (dynptr->type == BPF_DYNPTR_TYPE_INVALID) { /* dynptr constructors */
+ if (dynptr_type_referenced(type)) {
+ int id;
- if (clone_ref_obj_id)
- id = clone_ref_obj_id;
- else
- id = acquire_reference(env, insn_idx);
-
- if (id < 0)
- return id;
+ /*
+ * Create an intermediate reference that tracks the referenced
+ * object for the referenced dynptr. Freeing a referenced dynptr
+ * through helpers/kfuncs will invalidate all clones.
+ */
+ id = acquire_reference(env, insn_idx, parent_id);
+ if (id < 0)
+ return id;
- state->stack[spi].spilled_ptr.ref_obj_id = id;
- state->stack[spi - 1].spilled_ptr.ref_obj_id = id;
+ parent_id = id;
+ }
+ } else { /* bpf_dynptr_clone() */
+ parent_id = dynptr->parent_id;
}
+ mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr,
+ &state->stack[spi - 1].spilled_ptr, type, parent_id);
+
return 0;
}
-static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi)
+static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_stack_state *stack)
{
int i;
for (i = 0; i < BPF_REG_SIZE; i++) {
- state->stack[spi].slot_type[i] = STACK_INVALID;
- state->stack[spi - 1].slot_type[i] = STACK_INVALID;
+ stack[0].slot_type[i] = STACK_INVALID;
+ stack[1].slot_type[i] = STACK_INVALID;
}
- bpf_mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
- bpf_mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
+ bpf_mark_reg_not_init(env, &stack[0].spilled_ptr);
+ bpf_mark_reg_not_init(env, &stack[1].spilled_ptr);
}
static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
{
struct bpf_func_state *state = bpf_func(env, reg);
- int spi, ref_obj_id, i;
+ int spi;
spi = dynptr_get_spi(env, reg);
if (spi < 0)
return spi;
- if (!dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
- invalidate_dynptr(env, state, spi);
- return 0;
- }
-
- ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;
-
- /* If the dynptr has a ref_obj_id, then we need to invalidate
- * two things:
- *
- * 1) Any dynptrs with a matching ref_obj_id (clones)
- * 2) Any slices derived from this dynptr.
+ /*
+ * For referenced dynptr, release the parent ref which cascades to
+ * all clones and derived slices. For non-referenced dynptr, only
+ * the dynptr and slices derived from it will be invalidated.
*/
-
- /* Invalidate any slices associated with this dynptr */
- WARN_ON_ONCE(release_reference(env, ref_obj_id));
-
- /* Invalidate any dynptr clones */
- for (i = 1; i < state->allocated_stack / BPF_REG_SIZE; i++) {
- if (state->stack[i].spilled_ptr.ref_obj_id != ref_obj_id)
- continue;
-
- /* it should always be the case that if the ref obj id
- * matches then the stack slot also belongs to a
- * dynptr
- */
- if (state->stack[i].slot_type[0] != STACK_DYNPTR) {
- verifier_bug(env, "misconfigured ref_obj_id");
- return -EFAULT;
- }
- if (state->stack[i].spilled_ptr.dynptr.first_slot)
- invalidate_dynptr(env, state, i);
- }
-
- return 0;
+ reg = &state->stack[spi].spilled_ptr;
+ return release_reference(env, dynptr_type_referenced(reg->dynptr.type)
+ ? reg->parent_id
+ : reg->id);
}
static void __mark_reg_unknown(const struct bpf_verifier_env *env,
static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
struct bpf_func_state *state, int spi)
{
- struct bpf_func_state *fstate;
- struct bpf_reg_state *dreg;
- int i, dynptr_id;
+ int i, err = 0;
/* We always ensure that STACK_DYNPTR is never set partially,
* hence just checking for slot_type[0] is enough. This is
if (!state->stack[spi].spilled_ptr.dynptr.first_slot)
spi = spi + 1;
- if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
- int ref_obj_id = state->stack[spi].spilled_ptr.ref_obj_id;
+ if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type)) {
+ int v_parent_id = state->stack[spi].spilled_ptr.parent_id;
int ref_cnt = 0;
/*
* A referenced dynptr can be overwritten only if there is at
- * least one other dynptr sharing the same ref_obj_id,
+ * least one other dynptr sharing the same virtual ref parent,
* ensuring the reference can still be properly released.
*/
for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
continue;
if (!state->stack[i].spilled_ptr.dynptr.first_slot)
continue;
- if (state->stack[i].spilled_ptr.ref_obj_id == ref_obj_id)
+ if (state->stack[i].spilled_ptr.parent_id == v_parent_id)
ref_cnt++;
}
}
}
- mark_stack_slot_scratched(env, spi);
- mark_stack_slot_scratched(env, spi - 1);
-
- /* Writing partially to one dynptr stack slot destroys both. */
- for (i = 0; i < BPF_REG_SIZE; i++) {
- state->stack[spi].slot_type[i] = STACK_INVALID;
- state->stack[spi - 1].slot_type[i] = STACK_INVALID;
+ /* Invalidate the dynptr and any derived slices */
+ err = release_reference(env, state->stack[spi].spilled_ptr.id);
+ if (!err) {
+ mark_stack_slot_scratched(env, spi);
+ mark_stack_slot_scratched(env, spi - 1);
}
- dynptr_id = state->stack[spi].spilled_ptr.id;
- /* Invalidate any slices associated with this dynptr */
- bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({
- /* Dynptr slices are only PTR_TO_MEM_OR_NULL and PTR_TO_MEM */
- if (dreg->type != (PTR_TO_MEM | PTR_MAYBE_NULL) && dreg->type != PTR_TO_MEM)
- continue;
- if (dreg->dynptr_id == dynptr_id)
- mark_reg_invalid(env, dreg);
- }));
-
- /* Do not release reference state, we are destroying dynptr on stack,
- * not using some helper to release it. Just reset register.
- */
- bpf_mark_reg_not_init(env, &state->stack[spi].spilled_ptr);
- bpf_mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr);
-
- return 0;
+ return err;
}
static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
if (spi < 0)
return spi;
- id = acquire_reference(env, insn_idx);
+ id = acquire_reference(env, insn_idx, 0);
if (id < 0)
return id;
else
st->type |= PTR_UNTRUSTED;
}
- st->ref_obj_id = i == 0 ? id : 0;
+ st->id = i == 0 ? id : 0;
st->iter.btf = btf;
st->iter.btf_id = btf_id;
st->iter.state = BPF_ITER_STATE_ACTIVE;
struct bpf_reg_state *st = &slot->spilled_ptr;
if (i == 0)
- WARN_ON_ONCE(release_reference(env, st->ref_obj_id));
+ WARN_ON_ONCE(release_reference(env, st->id));
bpf_mark_reg_not_init(env, st);
if (st->type & PTR_UNTRUSTED)
return -EPROTO;
- /* only main (first) slot has ref_obj_id set */
- if (i == 0 && !st->ref_obj_id)
+ /* only main (first) slot has id set */
+ if (i == 0 && !st->id)
return -EINVAL;
- if (i != 0 && st->ref_obj_id)
+ if (i != 0 && st->id)
return -EINVAL;
if (st->iter.btf != btf || st->iter.btf_id != btf_id)
return -EINVAL;
__mark_reg_known_zero(st);
st->type = PTR_TO_STACK; /* we don't have dedicated reg type */
- st->ref_obj_id = id;
+ st->id = id;
st->irq.kfunc_class = kfunc_class;
for (i = 0; i < BPF_REG_SIZE; i++)
return -EINVAL;
}
- err = release_irq_state(env->cur_state, st->ref_obj_id);
+ err = release_irq_state(env->cur_state, st->id);
WARN_ON_ONCE(err && err != -EACCES);
if (err) {
int insn_idx = 0;
slot = &state->stack[spi];
st = &slot->spilled_ptr;
- if (!st->ref_obj_id)
+ if (!st->id)
return -EINVAL;
for (i = 0; i < BPF_REG_SIZE; i++)
return &state->refs[new_ofs];
}
-static int acquire_reference(struct bpf_verifier_env *env, int insn_idx)
+static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id)
{
struct bpf_reference_state *s;
return -ENOMEM;
s->type = REF_TYPE_PTR;
s->id = ++env->id_gen;
+ s->parent_id = parent_id;
return s->id;
}
return;
}
-static bool find_reference_state(struct bpf_verifier_state *state, int ptr_id)
+static bool find_reference_state(struct bpf_verifier_state *state, int id)
{
int i;
- for (i = 0; i < state->acquired_refs; i++)
- if (state->refs[i].id == ptr_id)
+ for (i = 0; i < state->acquired_refs; i++) {
+ if (state->refs[i].type != REF_TYPE_PTR)
+ continue;
+ if (state->refs[i].id == id)
return true;
+ }
return false;
}
+static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
+{
+ return find_reference_state(env->cur_state, reg->id);
+}
+
static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr)
{
void *prev_ptr = NULL;
memset(((u8 *)reg) + sizeof(reg->type), 0,
offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
reg->id = 0;
- reg->ref_obj_id = 0;
+ reg->parent_id = 0;
___mark_reg_known(reg, imm);
}
}
static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type,
- bool first_slot, int dynptr_id)
+ bool first_slot, int id, int parent_id)
{
/* reg->type has no meaning for STACK_DYNPTR, but when we set reg for
* callback arguments, it does need to be CONST_PTR_TO_DYNPTR, so simply
__mark_reg_known_zero(reg);
reg->type = CONST_PTR_TO_DYNPTR;
/* Give each dynptr a unique id to uniquely associate slices to it. */
- reg->id = dynptr_id;
+ reg->id = id;
+ reg->parent_id = parent_id;
reg->dynptr.type = type;
reg->dynptr.first_slot = first_slot;
}
/* Mark a register as having a completely unknown (scalar) value. */
void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
{
- /*
- * Clear type, off, and union(map_ptr, range) and
- * padding between 'type' and union
- */
- memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
+ s32 subreg_def = reg->subreg_def;
+
+ memset(reg, 0, sizeof(*reg));
reg->type = SCALAR_VALUE;
- reg->id = 0;
- reg->ref_obj_id = 0;
reg->var_off = tnum_unknown;
- reg->frameno = 0;
- reg->precise = false;
+ reg->subreg_def = subreg_def;
__mark_reg_unbounded(reg);
}
* referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
* normal store of unreferenced kptr, we must ensure var_off is zero.
* Since ref_ptr cannot be accessed directly by BPF insns, check for
- * reg->ref_obj_id is not needed here.
+ * reg->id is not needed here.
*/
if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true))
return -EACCES;
* type of narrower access.
*/
if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
- if (info->ref_obj_id &&
- !find_reference_state(env->cur_state, info->ref_obj_id)) {
+ if (info->ref_id &&
+ !find_reference_state(env->cur_state, info->ref_id)) {
verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
off);
return -EACCES;
[CONST_PTR_TO_MAP] = btf_bpf_map_id,
};
-static bool is_trusted_reg(const struct bpf_reg_state *reg)
+static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
{
/* A referenced register is always trusted. */
- if (reg->ref_obj_id)
+ if (reg_is_referenced(env, reg))
return true;
/* Types listed in the reg2btf_ids are always trusted */
ret = env->ops->btf_struct_access(&env->log, reg, off, size);
} else {
/* Writes are permitted with default btf_struct_access for
- * program allocated objects (which always have ref_obj_id > 0),
+ * program allocated objects (which always have id > 0),
* but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
*/
if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
}
if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
- !(reg->type & MEM_RCU) && !reg->ref_obj_id) {
- verifier_bug(env, "ref_obj_id for allocated object must be non-zero");
+ !(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) {
+ verifier_bug(env, "allocated object must have a referenced id");
return -EFAULT;
}
*/
flag = PTR_UNTRUSTED;
- } else if (is_trusted_reg(reg) || is_rcu_reg(reg)) {
+ } else if (is_trusted_reg(env, reg) || is_rcu_reg(reg)) {
/* By default any pointer obtained from walking a trusted pointer is no
* longer trusted, unless the field being accessed has explicitly been
* marked as inheriting its parent's state of trust (either full or RCU).
if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
regs[value_regno].btf = info.btf;
regs[value_regno].btf_id = info.btf_id;
- regs[value_regno].id = info.ref_obj_id;
- regs[value_regno].ref_obj_id = info.ref_obj_id;
+ regs[value_regno].id = info.ref_id;
}
if (type_may_be_null(info.reg_type) && !regs[value_regno].id)
regs[value_regno].id = ++env->id_gen;
return 0;
}
-/* There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
+/*
+ * Validate dynptr arguments for helper, kfunc and subprog.
+ *
+ * @dynptr is both input and output. It is populated when the argument is
+ * tagged with MEM_UNINIT (i.e., the dynptr argument that will be constructed)
+ * and consumed when the argument is expecting to be an initialized dynptr.
+ * @parent_id is used to track the referenced parent object (e.g., file or skb in
+ * qdisc program) when constructing a dynptr.
+ *
+ * There are two register types representing a bpf_dynptr, one is PTR_TO_STACK
* which points to a stack slot, and the other is CONST_PTR_TO_DYNPTR.
*
* In both cases we deal with the first 8 bytes, but need to mark the next 8
*/
static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
argno_t argno, int insn_idx, enum bpf_arg_type arg_type,
- int clone_ref_obj_id, struct bpf_dynptr_desc *dynptr)
+ int parent_id, struct bpf_dynptr_desc *dynptr)
{
int spi, err = 0;
return err;
}
- err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
+ err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, parent_id, dynptr);
} else /* OBJ_RELEASE and None case from above */ {
/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) {
if (dynptr) {
dynptr->type = reg->dynptr.type;
dynptr->id = reg->id;
- dynptr->ref_obj_id = reg->ref_obj_id;
+ dynptr->parent_id = reg->parent_id;
}
}
return err;
}
-static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
+static u32 iter_ref_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi)
{
struct bpf_func_state *state = bpf_func(env, reg);
- return state->stack[spi].spilled_ptr.ref_obj_id;
+ return state->stack[spi].spilled_ptr.id;
}
static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
/* remember meta->iter info for process_iter_next_call() */
meta->iter.spi = spi;
meta->iter.frameno = reg->frameno;
- meta->ref_obj_id = iter_ref_obj_id(env, reg, spi);
+ meta->id = iter_ref_id(env, reg, spi);
if (is_iter_destroy_kfunc(meta)) {
err = unmark_stack_slots_iter(env, reg, nr_slots);
/* When referenced register is passed to release function, its fixed
* offset must be 0.
*
- * We will check arg_type_is_release reg has ref_obj_id when storing
+ * We will check arg_type_is_release reg has id when storing
* meta->release_regno.
*/
if (arg_type_is_release(arg_type)) {
*/
if (reg->type == PTR_TO_STACK) {
spi = dynptr_get_spi(env, reg);
- if (spi < 0 || !state->stack[spi].spilled_ptr.ref_obj_id) {
+ if (spi < 0 || !state->stack[spi].spilled_ptr.id) {
verbose(env, "arg %d is an unacquired reference\n", regno);
return -EINVAL;
}
verbose(env, "cannot release unowned const bpf_dynptr\n");
return -EINVAL;
}
- } else if (!reg->ref_obj_id && !bpf_register_is_null(reg)) {
+ } else if (!reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) {
verbose(env, "R%d must be referenced when passed to release function\n",
regno);
return -EINVAL;
meta->release_regno = regno;
}
- if (reg->ref_obj_id && base_type(arg_type) != ARG_KPTR_XCHG_DEST) {
- if (meta->ref_obj_id) {
- verbose(env, "more than one arg with ref_obj_id R%d %u %u",
- regno, reg->ref_obj_id,
- meta->ref_obj_id);
+ if (reg_is_referenced(env, reg) && base_type(arg_type) != ARG_KPTR_XCHG_DEST) {
+ if (meta->id) {
+ verbose(env, "more than one arg with referenced id R%d %u %u",
+ regno, reg->id,
+ meta->id);
return -EACCES;
}
- meta->ref_obj_id = reg->ref_obj_id;
+ meta->id = reg->id;
}
switch (base_type(arg_type)) {
reg->range = AT_PKT_END;
}
-static int release_reference_nomark(struct bpf_verifier_state *state, int ref_obj_id)
+static int release_reference_nomark(struct bpf_verifier_state *state, int id)
{
int i;
for (i = 0; i < state->acquired_refs; i++) {
if (state->refs[i].type != REF_TYPE_PTR)
continue;
- if (state->refs[i].id == ref_obj_id) {
+ if (state->refs[i].id == id) {
release_reference_state(state, i);
return 0;
}
return -EINVAL;
}
-/* The pointer with the specified id has released its reference to kernel
- * resources. Identify all copies of the same pointer and clear the reference.
- *
- * This is the release function corresponding to acquire_reference(). Idempotent.
- */
-static int release_reference(struct bpf_verifier_env *env, int ref_obj_id)
+static int idstack_push(struct bpf_idmap *idmap, u32 id)
+{
+ int i;
+
+ if (!id)
+ return 0;
+
+ for (i = 0; i < idmap->cnt; i++)
+ if (idmap->map[i].old == id)
+ return 0;
+
+ if (WARN_ON_ONCE(idmap->cnt >= BPF_ID_MAP_SIZE))
+ return -EFAULT;
+
+ idmap->map[idmap->cnt++].old = id;
+ return 0;
+}
+
+static int idstack_pop(struct bpf_idmap *idmap)
{
+ if (!idmap->cnt)
+ return 0;
+
+ return idmap->map[--idmap->cnt].old;
+}
+
+/* Release id and objects derived from it iteratively in a DFS manner */
+static int release_reference(struct bpf_verifier_env *env, int id)
+{
+ u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR);
struct bpf_verifier_state *vstate = env->cur_state;
+ struct bpf_idmap *idstack = &env->idmap_scratch;
+ struct bpf_stack_state *stack;
struct bpf_func_state *state;
struct bpf_reg_state *reg;
- int err;
+ int i, err;
- err = release_reference_nomark(vstate, ref_obj_id);
+ idstack->cnt = 0;
+ err = idstack_push(idstack, id);
if (err)
return err;
- bpf_for_each_reg_in_vstate(vstate, state, reg, ({
- if (reg->ref_obj_id == ref_obj_id)
- mark_reg_invalid(env, reg);
- }));
+ if (find_reference_state(vstate, id))
+ WARN_ON_ONCE(release_reference_nomark(vstate, id));
+
+ while ((id = idstack_pop(idstack))) {
+ /*
+ * Child references are inaccessible after parent is released,
+ * any child references that exist at this point are a leak.
+ */
+ for (i = 0; i < vstate->acquired_refs; i++) {
+ if (vstate->refs[i].type != REF_TYPE_PTR)
+ continue;
+ if (vstate->refs[i].parent_id != id)
+ continue;
+ verbose(env, "Leaking reference id=%d alloc_insn=%d. Release it first.\n",
+ vstate->refs[i].id, vstate->refs[i].insn_idx);
+ return -EINVAL;
+ }
+
+ bpf_for_each_reg_in_vstate_mask(vstate, state, reg, stack, mask, ({
+ if (reg->id != id && reg->parent_id != id)
+ continue;
+
+ /* Free objects derived from the current object */
+ if (reg->parent_id == id) {
+ err = idstack_push(idstack, reg->id);
+ if (err)
+ return err;
+ }
+
+ if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL)
+ mark_reg_invalid(env, reg);
+ else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR)
+ invalidate_dynptr(env, stack);
+ }));
+ }
return 0;
}
* kernel. Type checks are performed later in check_return_code.
*/
if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit &&
- reg->ref_obj_id == state->refs[i].id)
+ reg->id == state->refs[i].id)
continue;
verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
state->refs[i].id, state->refs[i].insn_idx);
err = -EINVAL;
if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]);
- } else if (func_id == BPF_FUNC_kptr_xchg && meta.ref_obj_id) {
- u32 ref_obj_id = meta.ref_obj_id;
+ } else if (func_id == BPF_FUNC_kptr_xchg && meta.id) {
+ u32 id = meta.id;
bool in_rcu = in_rcu_cs(env);
struct bpf_func_state *state;
struct bpf_reg_state *reg;
- err = release_reference_nomark(env->cur_state, ref_obj_id);
+ err = release_reference_nomark(env->cur_state, id);
if (!err) {
bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
- if (reg->ref_obj_id == ref_obj_id) {
+ if (reg->id == id) {
if (in_rcu && (reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) {
- reg->ref_obj_id = 0;
+ reg->id = 0;
reg->type &= ~MEM_ALLOC;
reg->type |= MEM_RCU;
} else {
}
}));
}
- } else if (meta.ref_obj_id) {
- err = release_reference(env, meta.ref_obj_id);
+ } else if (meta.id) {
+ err = release_reference(env, meta.id);
} else if (bpf_register_is_null(®s[meta.release_regno])) {
- /* meta.ref_obj_id can only be 0 if register that is meant to be
+ /* meta.id can only be 0 if register that is meant to be
* released is NULL, which must be > R0.
*/
err = 0;
}
- if (err) {
- verbose(env, "func %s#%d reference has not been acquired before\n",
- func_id_name(func_id), func_id);
+ if (err)
return err;
- }
}
switch (func_id) {
return -EFAULT;
}
- if (is_ptr_cast_function(func_id)) {
- /* For release_reference() */
- regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
+ if (is_ptr_cast_function(func_id) &&
+ find_reference_state(env->cur_state, meta.id)) {
+ struct bpf_verifier_state *branch;
+ struct bpf_reg_state *r0;
+
+ /*
+ * In order for a release of any of the original or cast pointers
+ * to invalidate all other pointers, reuse the same reference id for
+ * the cast result.
+ * This reference id can't be used for nullness propagation,
+ * as cast might return NULL for a non-NULL input.
+ * Hence, explore the NULL case as a separate branch.
+ */
+ branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
+ if (IS_ERR(branch))
+ return PTR_ERR(branch);
+
+ r0 = &branch->frame[branch->curframe]->regs[BPF_REG_0];
+ __mark_reg_known_zero(r0);
+ r0->type = SCALAR_VALUE;
+
+ regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL;
+ regs[BPF_REG_0].id = meta.id;
} else if (is_acquire_function(func_id, meta.map.ptr)) {
- int id = acquire_reference(env, insn_idx);
+ int id = acquire_reference(env, insn_idx, 0);
if (id < 0)
return id;
- /* For mark_ptr_or_null_reg() */
+
regs[BPF_REG_0].id = id;
- /* For release_reference() */
- regs[BPF_REG_0].ref_obj_id = id;
}
- if (func_id == BPF_FUNC_dynptr_data) {
- regs[BPF_REG_0].dynptr_id = meta.dynptr.id;
- regs[BPF_REG_0].ref_obj_id = meta.dynptr.ref_obj_id;
- }
+ if (func_id == BPF_FUNC_dynptr_data)
+ regs[BPF_REG_0].parent_id = meta.dynptr.id;
err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
if (err)
* btf_struct_ids_match() to walk the struct at the 0th offset, and
* resolve types.
*/
- if ((is_kfunc_release(meta) && reg->ref_obj_id) ||
+ if ((is_kfunc_release(meta) && reg_is_referenced(env, reg)) ||
btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id))
strict_type_match = true;
return 0;
}
-static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id)
+static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id)
{
- struct bpf_verifier_state *state = env->cur_state;
struct bpf_func_state *unused;
struct bpf_reg_state *reg;
- int i;
- if (!ref_obj_id) {
- verifier_bug(env, "ref_obj_id is zero for owning -> non-owning conversion");
- return -EFAULT;
- }
+ WARN_ON_ONCE(release_reference_nomark(env->cur_state, id));
- for (i = 0; i < state->acquired_refs; i++) {
- if (state->refs[i].id != ref_obj_id)
- continue;
-
- /* Clear ref_obj_id here so release_reference doesn't clobber
- * the whole reg
- */
- bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
- if (reg->ref_obj_id == ref_obj_id) {
- reg->ref_obj_id = 0;
- ref_set_non_owning(env, reg);
- }
- }));
- return 0;
- }
+ bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({
+ if (reg->id == id) {
+ reg->id = 0;
+ ref_set_non_owning(env, reg);
+ }
+ }));
- verifier_bug(env, "ref state missing for ref_obj_id");
- return -EFAULT;
+ return;
}
/* Implementation details:
return -EACCES;
}
- if (reg->ref_obj_id) {
- if (is_kfunc_release(meta) && meta->ref_obj_id) {
- verifier_bug(env, "more than one arg with ref_obj_id %s %u %u",
- reg_arg_name(env, argno), reg->ref_obj_id,
- meta->ref_obj_id);
+ if (reg_is_referenced(env, reg)) {
+ if (is_kfunc_release(meta) && meta->id) {
+ verifier_bug(env, "more than one arg with referenced id %s %u %u",
+ reg_arg_name(env, argno), reg->id,
+ meta->id);
return -EFAULT;
}
- meta->ref_obj_id = reg->ref_obj_id;
+ meta->id = reg->id;
if (is_kfunc_release(meta)) {
if (regno < 0) {
verbose(env, "%s release arg cannot be a stack argument\n",
fallthrough;
case KF_ARG_PTR_TO_ALLOC_BTF_ID:
case KF_ARG_PTR_TO_BTF_ID:
- if (!is_trusted_reg(reg)) {
+ if (!is_trusted_reg(env, reg)) {
if (!is_kfunc_rcu(meta)) {
verbose(env, "%s must be referenced or trusted\n",
reg_arg_name(env, argno));
return -EFAULT;
}
- if (is_kfunc_release(meta) && reg->ref_obj_id)
+ if (is_kfunc_release(meta) && reg_is_referenced(env, reg))
arg_type |= OBJ_RELEASE;
ret = check_func_arg_reg_off(env, reg, argno, arg_type);
if (ret < 0)
reg_arg_name(env, argno));
return -EINVAL;
}
- if (!reg->ref_obj_id) {
+ if (!reg_is_referenced(env, reg)) {
verbose(env, "allocated object must be referenced\n");
return -EINVAL;
}
case KF_ARG_PTR_TO_DYNPTR:
{
enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
- int clone_ref_obj_id = 0;
if (is_kfunc_arg_uninit(btf, &args[i]))
dynptr_arg_type |= MEM_UNINIT;
}
dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type);
- clone_ref_obj_id = meta->dynptr.ref_obj_id;
- if (dynptr_type_refcounted(parent_type) && !clone_ref_obj_id) {
- verifier_bug(env, "missing ref obj id for parent of clone");
- return -EFAULT;
- }
}
ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type,
- clone_ref_obj_id, &meta->dynptr);
+ meta->id, &meta->dynptr);
if (ret < 0)
return ret;
break;
reg_arg_name(env, argno));
return -EINVAL;
}
- if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
+ if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
+ !reg_is_referenced(env, reg)) {
verbose(env, "allocated object must be referenced\n");
return -EINVAL;
}
reg_arg_name(env, argno));
return -EINVAL;
}
- if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
+ if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) &&
+ !reg_is_referenced(env, reg)) {
verbose(env, "allocated object must be referenced\n");
return -EINVAL;
}
break;
case KF_ARG_PTR_TO_LIST_NODE:
if (is_kfunc_arg_nonown_allowed(btf, &args[i]) &&
- type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) {
+ type_is_non_owning_ref(reg->type) && !reg_is_referenced(env, reg)) {
/* Allow bpf_list_front/back return value for
* __nonown_allowed list-node arguments.
*/
reg_arg_name(env, argno));
return -EINVAL;
}
- if (!reg->ref_obj_id) {
+ if (!reg_is_referenced(env, reg)) {
verbose(env, "allocated object must be referenced\n");
return -EINVAL;
}
reg_arg_name(env, argno));
return -EINVAL;
}
- if (!reg->ref_obj_id) {
+ if (!reg_is_referenced(env, reg)) {
verbose(env, "allocated object must be referenced\n");
return -EINVAL;
}
} else {
- if (!type_is_non_owning_ref(reg->type) && !reg->ref_obj_id) {
+ if (!type_is_non_owning_ref(reg->type) &&
+ !reg_is_referenced(env, reg)) {
verbose(env, "%s can only take non-owning or refcounted bpf_rb_node pointer\n", func_name);
return -EINVAL;
}
verifier_bug(env, "no dynptr id");
return -EFAULT;
}
- regs[BPF_REG_0].dynptr_id = meta->dynptr.id;
-
- /* we don't need to set BPF_REG_0's ref obj id
- * because packet slices are not refcounted (see
- * dynptr_type_refcounted)
- */
+ regs[BPF_REG_0].parent_id = meta->dynptr.id;
} else {
return 0;
}
int *insn_idx_p)
{
bool sleepable, rcu_lock, rcu_unlock, preempt_disable, preempt_enable;
- u32 i, nargs, ptr_type_id, release_ref_obj_id;
struct bpf_reg_state *regs = cur_regs(env);
const char *func_name, *ptr_type_name;
const struct btf_type *t, *ptr_type;
struct bpf_kfunc_call_arg_meta meta;
struct bpf_insn_aux_data *insn_aux;
int err, insn_idx = *insn_idx_p;
+ u32 i, nargs, ptr_type_id, id;
const struct btf_param *args;
struct btf *desc_btf;
if (rcu_lock) {
env->cur_state->active_rcu_locks++;
} else if (rcu_unlock) {
+ struct bpf_stack_state *stack;
struct bpf_func_state *state;
struct bpf_reg_state *reg;
u32 clear_mask = (1 << STACK_SPILL) | (1 << STACK_ITER);
return -EINVAL;
}
if (--env->cur_state->active_rcu_locks == 0) {
- bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({
+ bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({
if (reg->type & MEM_RCU) {
reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
reg->type |= PTR_UNTRUSTED;
if (meta.release_regno) {
struct bpf_reg_state *reg = ®s[meta.release_regno];
- if (meta.dynptr.ref_obj_id) {
+ if (meta.dynptr.id) {
err = unmark_stack_slots_dynptr(env, reg);
} else {
- err = release_reference(env, reg->ref_obj_id);
- if (err)
- verbose(env, "kfunc %s#%d reference has not been acquired before\n",
- func_name, meta.func_id);
+ err = release_reference(env, reg->id);
}
if (err)
return err;
}
if (is_bpf_list_push_kfunc(meta.func_id) || is_bpf_rbtree_add_kfunc(meta.func_id)) {
- release_ref_obj_id = regs[BPF_REG_2].ref_obj_id;
+ id = regs[BPF_REG_2].id;
insn_aux->insert_off = regs[BPF_REG_2].var_off.value;
insn_aux->kptr_struct_meta = btf_find_struct_meta(meta.arg_btf, meta.arg_btf_id);
- err = ref_convert_owning_non_owning(env, release_ref_obj_id);
- if (err) {
- verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n",
- func_name, meta.func_id);
- return err;
- }
-
- err = release_reference(env, release_ref_obj_id);
- if (err) {
- verbose(env, "kfunc %s#%d reference has not been acquired before\n",
- func_name, meta.func_id);
- return err;
- }
+ ref_convert_owning_non_owning(env, id);
}
if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
regs[BPF_REG_0].type |= MEM_RDONLY;
/* Ensures we don't access the memory after a release_reference() */
- if (meta.ref_obj_id)
- regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
+ if (meta.id)
+ regs[BPF_REG_0].parent_id = meta.id;
if (is_kfunc_rcu_protected(&meta))
regs[BPF_REG_0].type |= MEM_RCU;
}
mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
if (is_kfunc_acquire(&meta)) {
- int id = acquire_reference(env, insn_idx);
-
+ id = acquire_reference(env, insn_idx, 0);
if (id < 0)
return id;
- if (is_kfunc_ret_null(&meta))
- regs[BPF_REG_0].id = id;
- regs[BPF_REG_0].ref_obj_id = id;
+ regs[BPF_REG_0].id = id;
} else if (is_rbtree_node_type(ptr_type) || is_list_node_type(ptr_type)) {
ref_set_non_owning(env, ®s[BPF_REG_0]);
}
if (!is_reg_const(reg2, is_jmp32))
return -1;
- if (!reg_not_null(reg1))
+ if (!reg_not_null(env, reg1))
return -1;
/* If pointer is valid tests against zero will fail so we can
WARN_ON_ONCE(!tnum_equals_const(reg->var_off, 0)))
return;
if (is_null) {
- /* We don't need id and ref_obj_id from this point
+ /* We don't need id from this point
* onwards anymore, thus we should better reset it,
* so that state pruning has chances to take effect.
*/
{
struct bpf_func_state *state = vstate->frame[vstate->curframe];
struct bpf_reg_state *regs = state->regs, *reg;
- u32 ref_obj_id = regs[regno].ref_obj_id;
u32 id = regs[regno].id;
- if (ref_obj_id && ref_obj_id == id && is_null)
+ if (is_null && find_reference_state(vstate, id))
/* regs[regno] is in the " == NULL" branch.
* No one could have freed the reference state before
* doing the NULL check.
ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
prog->aux->attach_func_proto->type,
NULL);
- if (ret_type && ret_type == reg_type && reg->ref_obj_id)
+ if (ret_type && ret_type == reg_type && reg_is_referenced(env, reg))
return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);
}
mark_reg_unknown(env, regs, i);
} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
/* assume unspecial LOCAL dynptr type */
- __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
+ __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen, 0);
} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
reg->type = PTR_TO_MEM;
reg->type |= arg->arg_type &
/* Acquire references for struct_ops program arguments tagged with "__ref" */
if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
for (i = 0; i < aux->ctx_arg_info_size; i++)
- aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ?
- acquire_reference(env, 0) : 0;
+ aux->ctx_arg_info[i].ref_id = aux->ctx_arg_info[i].refcounted ?
+ acquire_reference(env, 0, 0) : 0;
}
ret = do_check(env);
projects / thirdparty / linux.git / commitdiff
