bpf: allow using bpf_kptr_xchg even if the NON_OWN_REF flag is set

When traversing an rbtree using bpf_rbtree_left/right, if bpf_kptr_xchg
is used to access the __kptr pointer contained in a node, it currently
requires first removing the node with bpf_rbtree_remove and clearing the
NON_OWN_REF flag, then re-adding the node to the original rbtree with
bpf_rbtree_add after usage. This process significantly degrades rbtree
traversal performance. The patch enables accessing __kptr pointers with
the NON_OWN_REF flag set while holding the lock, eliminating the need
for this remove-read-add sequence.

Signed-off-by: Kaitao Cheng <chengkaitao@kylinos.cn>
Signed-off-by: Feng Yang <yangfeng@kylinos.cn>
Link: https://lore.kernel.org/r/20260214124042.62229-3-pilgrimtao@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ee63f27aa5e4397932e906adbb701f9df841ff40..7426dba5e5d1c15c752ae7aaac9869e03c911d28 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -9309,7 +9309,8 @@ static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE }
 static const struct bpf_reg_types kptr_xchg_dest_types = {
        .types = {
                PTR_TO_MAP_VALUE,
-               PTR_TO_BTF_ID | MEM_ALLOC
+               PTR_TO_BTF_ID | MEM_ALLOC,
+               PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF,
        }
 };
 static const struct bpf_reg_types dynptr_types = {
@@ -9473,6 +9474,7 @@ found:
        }
        case PTR_TO_BTF_ID | MEM_ALLOC:
        case PTR_TO_BTF_ID | MEM_PERCPU | MEM_ALLOC:
+       case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF:
                if (meta->func_id != BPF_FUNC_spin_lock && meta->func_id != BPF_FUNC_spin_unlock &&
                    meta->func_id != BPF_FUNC_kptr_xchg) {
                        verifier_bug(env, "unimplemented handling of MEM_ALLOC");