bpf: Include verifier memory allocations in memcg statistics

This commit adds __GFP_ACCOUNT flag to verifier induced memory
allocations. The intent is to account for all allocations reachable
from BPF_PROG_LOAD command, which is needed to track verifier memory
consumption in veristat. This includes allocations done in verifier.c,
and some allocations in btf.c, functions in log.c do not allocate.

There is also a utility function bpf_memcg_flags() which selectively
adds GFP_ACCOUNT flag depending on the `cgroup.memory=nobpf` option.
As far as I understand [1], the idea is to remove bpf_prog instances
and maps from memcg accounting as these objects do not strictly belong
to cgroup, hence it should not apply here.

(btf_parse_fields() is reachable from both program load and map
 creation, but allocated record is not persistent as is freed as soon
 as map_check_btf() exits).

[1] https://lore.kernel.org/all/20230210154734.4416-1-laoar.shao@gmail.com/

Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20250613072147.3938139-2-eddyz87@gmail.com

diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 1d2cf898e21e32c58490753fa6bc39b8d78e356a..682acb1ed23421f9336272621ad6989ca01e378f 100644 (file)
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3443,7 +3443,8 @@ btf_find_graph_root(const struct btf *btf, const struct btf_type *pt,
        node_field_name = strstr(value_type, ":");
        if (!node_field_name)
                return -EINVAL;
-       value_type = kstrndup(value_type, node_field_name - value_type, GFP_KERNEL | __GFP_NOWARN);
+       value_type = kstrndup(value_type, node_field_name - value_type,
+                             GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!value_type)
                return -ENOMEM;
        id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT);
@@ -3958,7 +3959,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
        /* This needs to be kzalloc to zero out padding and unused fields, see
         * comment in btf_record_equal.
         */
-       rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL | __GFP_NOWARN);
+       rec = kzalloc(struct_size(rec, fields, cnt), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!rec)
                return ERR_PTR(-ENOMEM);
 
@@ -9019,7 +9020,7 @@ static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands,
                bpf_free_cands_from_cache(*cc);
                *cc = NULL;
        }
-       new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL);
+       new_cands = kmemdup(cands, sizeof_cands(cands->cnt), GFP_KERNEL_ACCOUNT);
        if (!new_cands) {
                bpf_free_cands(cands);
                return ERR_PTR(-ENOMEM);
@@ -9027,7 +9028,7 @@ static struct bpf_cand_cache *populate_cand_cache(struct bpf_cand_cache *cands,
        /* strdup the name, since it will stay in cache.
         * the cands->name points to strings in prog's BTF and the prog can be unloaded.
         */
-       new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL);
+       new_cands->name = kmemdup_nul(cands->name, cands->name_len, GFP_KERNEL_ACCOUNT);
        bpf_free_cands(cands);
        if (!new_cands->name) {
                kfree(new_cands);
@@ -9111,7 +9112,7 @@ bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf,
                        continue;
 
                /* most of the time there is only one candidate for a given kind+name pair */
-               new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL);
+               new_cands = kmalloc(sizeof_cands(cands->cnt + 1), GFP_KERNEL_ACCOUNT);
                if (!new_cands) {
                        bpf_free_cands(cands);
                        return ERR_PTR(-ENOMEM);
@@ -9228,7 +9229,7 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
        /* ~4k of temp memory necessary to convert LLVM spec like "0:1:0:5"
         * into arrays of btf_ids of struct fields and array indices.
         */
-       specs = kcalloc(3, sizeof(*specs), GFP_KERNEL);
+       specs = kcalloc(3, sizeof(*specs), GFP_KERNEL_ACCOUNT);
        if (!specs)
                return -ENOMEM;
 
@@ -9253,7 +9254,7 @@ int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
                        goto out;
                }
                if (cc->cnt) {
-                       cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL);
+                       cands.cands = kcalloc(cc->cnt, sizeof(*cands.cands), GFP_KERNEL_ACCOUNT);
                        if (!cands.cands) {
                                err = -ENOMEM;
                                goto out;

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index c378074516cfde39a2bb6f61aff834b230888d11..fd81cff116170dd65e116457dffac18afcf9450f 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -1404,7 +1404,7 @@ static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
                goto out;
 
        alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
-       new_arr = krealloc(arr, alloc_size, GFP_KERNEL);
+       new_arr = krealloc(arr, alloc_size, GFP_KERNEL_ACCOUNT);
        if (!new_arr) {
                kfree(arr);
                return NULL;
@@ -1421,7 +1421,7 @@ out:
 static int copy_reference_state(struct bpf_verifier_state *dst, const struct bpf_verifier_state *src)
 {
        dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
-                              sizeof(struct bpf_reference_state), GFP_KERNEL);
+                              sizeof(struct bpf_reference_state), GFP_KERNEL_ACCOUNT);
        if (!dst->refs)
                return -ENOMEM;
 
@@ -1440,7 +1440,7 @@ static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_st
        size_t n = src->allocated_stack / BPF_REG_SIZE;
 
        dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
-                               GFP_KERNEL);
+                               GFP_KERNEL_ACCOUNT);
        if (!dst->stack)
                return -ENOMEM;
 
@@ -1731,7 +1731,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
 
        dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
                                          src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
-                                         GFP_USER);
+                                         GFP_KERNEL_ACCOUNT);
        if (!dst_state->jmp_history)
                return -ENOMEM;
        dst_state->jmp_history_cnt = src->jmp_history_cnt;
@@ -1760,7 +1760,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
        for (i = 0; i <= src->curframe; i++) {
                dst = dst_state->frame[i];
                if (!dst) {
-                       dst = kzalloc(sizeof(*dst), GFP_KERNEL);
+                       dst = kzalloc(sizeof(*dst), GFP_KERNEL_ACCOUNT);
                        if (!dst)
                                return -ENOMEM;
                        dst_state->frame[i] = dst;
@@ -1874,7 +1874,7 @@ static struct bpf_scc_visit *scc_visit_alloc(struct bpf_verifier_env *env,
        info = env->scc_info[scc];
        num_visits = info ? info->num_visits : 0;
        new_sz = sizeof(*info) + sizeof(struct bpf_scc_visit) * (num_visits + 1);
-       info = kvrealloc(env->scc_info[scc], new_sz, GFP_KERNEL);
+       info = kvrealloc(env->scc_info[scc], new_sz, GFP_KERNEL_ACCOUNT);
        if (!info)
                return NULL;
        env->scc_info[scc] = info;
@@ -2095,7 +2095,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
        struct bpf_verifier_stack_elem *elem;
        int err;
 
-       elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
+       elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
        if (!elem)
                goto err;
 
@@ -2862,7 +2862,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
        struct bpf_verifier_stack_elem *elem;
        struct bpf_func_state *frame;
 
-       elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
+       elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT);
        if (!elem)
                goto err;
 
@@ -2885,7 +2885,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
         */
        elem->st.branches = 1;
        elem->st.in_sleepable = is_sleepable;
-       frame = kzalloc(sizeof(*frame), GFP_KERNEL);
+       frame = kzalloc(sizeof(*frame), GFP_KERNEL_ACCOUNT);
        if (!frame)
                goto err;
        init_func_state(env, frame,
@@ -3237,7 +3237,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
                        return -EINVAL;
                }
 
-               tab = kzalloc(sizeof(*tab), GFP_KERNEL);
+               tab = kzalloc(sizeof(*tab), GFP_KERNEL_ACCOUNT);
                if (!tab)
                        return -ENOMEM;
                prog_aux->kfunc_tab = tab;
@@ -3253,7 +3253,7 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
                return 0;
 
        if (!btf_tab && offset) {
-               btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
+               btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL_ACCOUNT);
                if (!btf_tab)
                        return -ENOMEM;
                prog_aux->kfunc_btf_tab = btf_tab;
@@ -3939,7 +3939,7 @@ static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_st
 
        cnt++;
        alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
-       p = krealloc(cur->jmp_history, alloc_size, GFP_USER);
+       p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
        if (!p)
                return -ENOMEM;
        cur->jmp_history = p;
@@ -10356,7 +10356,7 @@ static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int calls
        }
 
        caller = state->frame[state->curframe];
-       callee = kzalloc(sizeof(*callee), GFP_KERNEL);
+       callee = kzalloc(sizeof(*callee), GFP_KERNEL_ACCOUNT);
        if (!callee)
                return -ENOMEM;
        state->frame[state->curframe + 1] = callee;
@@ -17693,17 +17693,18 @@ static int check_cfg(struct bpf_verifier_env *env)
        int *insn_stack, *insn_state, *insn_postorder;
        int ex_insn_beg, i, ret = 0;
 
-       insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
+       insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
        if (!insn_state)
                return -ENOMEM;
 
-       insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
+       insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
        if (!insn_stack) {
                kvfree(insn_state);
                return -ENOMEM;
        }
 
-       insn_postorder = env->cfg.insn_postorder = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
+       insn_postorder = env->cfg.insn_postorder =
+               kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
        if (!insn_postorder) {
                kvfree(insn_state);
                kvfree(insn_stack);
@@ -17837,7 +17838,7 @@ static int check_btf_func_early(struct bpf_verifier_env *env,
        urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
        min_size = min_t(u32, krec_size, urec_size);
 
-       krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
+       krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!krecord)
                return -ENOMEM;
 
@@ -17937,7 +17938,7 @@ static int check_btf_func(struct bpf_verifier_env *env,
        urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
 
        krecord = prog->aux->func_info;
-       info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
+       info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!info_aux)
                return -ENOMEM;
 
@@ -18023,7 +18024,7 @@ static int check_btf_line(struct bpf_verifier_env *env,
         * pass in a smaller bpf_line_info object.
         */
        linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
-                        GFP_KERNEL | __GFP_NOWARN);
+                        GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (!linfo)
                return -ENOMEM;
 
@@ -19408,7 +19409,7 @@ hit:
                        if (loop) {
                                struct bpf_scc_backedge *backedge;
 
-                               backedge = kzalloc(sizeof(*backedge), GFP_KERNEL);
+                               backedge = kzalloc(sizeof(*backedge), GFP_KERNEL_ACCOUNT);
                                if (!backedge)
                                        return -ENOMEM;
                                err = copy_verifier_state(&backedge->state, cur);
@@ -19472,7 +19473,7 @@ miss:
         * When looping the sl->state.branches will be > 0 and this state
         * will not be considered for equivalence until branches == 0.
         */
-       new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
+       new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL_ACCOUNT);
        if (!new_sl)
                return -ENOMEM;
        env->total_states++;
@@ -22976,13 +22977,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
        env->prev_linfo = NULL;
        env->pass_cnt++;
 
-       state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
+       state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL_ACCOUNT);
        if (!state)
                return -ENOMEM;
        state->curframe = 0;
        state->speculative = false;
        state->branches = 1;
-       state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
+       state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL_ACCOUNT);
        if (!state->frame[0]) {
                kfree(state);
                return -ENOMEM;
@@ -23208,7 +23209,7 @@ static void print_verification_stats(struct bpf_verifier_env *env)
 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
                               const struct bpf_ctx_arg_aux *info, u32 cnt)
 {
-       prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL);
+       prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL_ACCOUNT);
        prog->aux->ctx_arg_info_size = cnt;
 
        return prog->aux->ctx_arg_info ? 0 : -ENOMEM;
@@ -24152,7 +24153,7 @@ static int compute_live_registers(struct bpf_verifier_env *env)
         * - repeat the computation while {in,out} fields changes for
         *   any instruction.
         */
-       state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL);
+       state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL_ACCOUNT);
        if (!state) {
                err = -ENOMEM;
                goto out;
@@ -24244,10 +24245,10 @@ static int compute_scc(struct bpf_verifier_env *env)
         * - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n';
         * - 'dfs' DFS traversal stack, used to emulate explicit recursion.
         */
-       stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
-       pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
-       low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
-       dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL);
+       stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
+       pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
+       low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
+       dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT);
        if (!stack || !pre || !low || !dfs) {
                err = -ENOMEM;
                goto exit;
@@ -24381,7 +24382,7 @@ dfs_continue:
                        dfs_sz--;
                }
        }
-       env->scc_info = kvcalloc(next_scc_id, sizeof(*env->scc_info), GFP_KERNEL);
+       env->scc_info = kvcalloc(next_scc_id, sizeof(*env->scc_info), GFP_KERNEL_ACCOUNT);
        if (!env->scc_info) {
                err = -ENOMEM;
                goto exit;
@@ -24409,7 +24410,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
        /* 'struct bpf_verifier_env' can be global, but since it's not small,
         * allocate/free it every time bpf_check() is called
         */
-       env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
+       env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL_ACCOUNT);
        if (!env)
                return -ENOMEM;
 
@@ -24472,7 +24473,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 
        env->explored_states = kvcalloc(state_htab_size(env),
                                       sizeof(struct list_head),
-                                      GFP_USER);
+                                      GFP_KERNEL_ACCOUNT);
        ret = -ENOMEM;
        if (!env->explored_states)
                goto skip_full_check;
@@ -24603,7 +24604,7 @@ skip_full_check:
                /* if program passed verifier, update used_maps in bpf_prog_info */
                env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
                                                          sizeof(env->used_maps[0]),
-                                                         GFP_KERNEL);
+                                                         GFP_KERNEL_ACCOUNT);
 
                if (!env->prog->aux->used_maps) {
                        ret = -ENOMEM;
@@ -24618,7 +24619,7 @@ skip_full_check:
                /* if program passed verifier, update used_btfs in bpf_prog_aux */
                env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
                                                          sizeof(env->used_btfs[0]),
-                                                         GFP_KERNEL);
+                                                         GFP_KERNEL_ACCOUNT);
                if (!env->prog->aux->used_btfs) {
                        ret = -ENOMEM;
                        goto err_release_maps;