1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
7 #include <uapi/linux/bpf.h>
8 #include <uapi/linux/filter.h>
10 #include <linux/workqueue.h>
11 #include <linux/file.h>
12 #include <linux/percpu.h>
13 #include <linux/err.h>
14 #include <linux/rbtree_latch.h>
15 #include <linux/numa.h>
16 #include <linux/mm_types.h>
17 #include <linux/wait.h>
18 #include <linux/refcount.h>
19 #include <linux/mutex.h>
20 #include <linux/module.h>
21 #include <linux/kallsyms.h>
22 #include <linux/capability.h>
23 #include <linux/sched/mm.h>
24 #include <linux/slab.h>
25 #include <linux/percpu-refcount.h>
26 #include <linux/stddef.h>
27 #include <linux/bpfptr.h>
28 #include <linux/btf.h>
29 #include <linux/rcupdate_trace.h>
30 #include <linux/static_call.h>
31 #include <linux/memcontrol.h>
33 struct bpf_verifier_env
;
34 struct bpf_verifier_log
;
43 struct exception_table_entry
;
44 struct seq_operations
;
45 struct bpf_iter_aux_info
;
46 struct bpf_local_storage
;
47 struct bpf_local_storage_map
;
51 struct bpf_func_state
;
55 extern struct idr btf_idr
;
56 extern spinlock_t btf_idr_lock
;
57 extern struct kobject
*btf_kobj
;
58 extern struct bpf_mem_alloc bpf_global_ma
, bpf_global_percpu_ma
;
59 extern bool bpf_global_ma_set
, bpf_global_percpu_ma_set
;
61 typedef u64 (*bpf_callback_t
)(u64
, u64
, u64
, u64
, u64
);
62 typedef int (*bpf_iter_init_seq_priv_t
)(void *private_data
,
63 struct bpf_iter_aux_info
*aux
);
64 typedef void (*bpf_iter_fini_seq_priv_t
)(void *private_data
);
65 typedef unsigned int (*bpf_func_t
)(const void *,
66 const struct bpf_insn
*);
67 struct bpf_iter_seq_info
{
68 const struct seq_operations
*seq_ops
;
69 bpf_iter_init_seq_priv_t init_seq_private
;
70 bpf_iter_fini_seq_priv_t fini_seq_private
;
74 /* map is generic key/value storage optionally accessible by eBPF programs */
76 /* funcs callable from userspace (via syscall) */
77 int (*map_alloc_check
)(union bpf_attr
*attr
);
78 struct bpf_map
*(*map_alloc
)(union bpf_attr
*attr
);
79 void (*map_release
)(struct bpf_map
*map
, struct file
*map_file
);
80 void (*map_free
)(struct bpf_map
*map
);
81 int (*map_get_next_key
)(struct bpf_map
*map
, void *key
, void *next_key
);
82 void (*map_release_uref
)(struct bpf_map
*map
);
83 void *(*map_lookup_elem_sys_only
)(struct bpf_map
*map
, void *key
);
84 int (*map_lookup_batch
)(struct bpf_map
*map
, const union bpf_attr
*attr
,
85 union bpf_attr __user
*uattr
);
86 int (*map_lookup_and_delete_elem
)(struct bpf_map
*map
, void *key
,
87 void *value
, u64 flags
);
88 int (*map_lookup_and_delete_batch
)(struct bpf_map
*map
,
89 const union bpf_attr
*attr
,
90 union bpf_attr __user
*uattr
);
91 int (*map_update_batch
)(struct bpf_map
*map
, struct file
*map_file
,
92 const union bpf_attr
*attr
,
93 union bpf_attr __user
*uattr
);
94 int (*map_delete_batch
)(struct bpf_map
*map
, const union bpf_attr
*attr
,
95 union bpf_attr __user
*uattr
);
97 /* funcs callable from userspace and from eBPF programs */
98 void *(*map_lookup_elem
)(struct bpf_map
*map
, void *key
);
99 long (*map_update_elem
)(struct bpf_map
*map
, void *key
, void *value
, u64 flags
);
100 long (*map_delete_elem
)(struct bpf_map
*map
, void *key
);
101 long (*map_push_elem
)(struct bpf_map
*map
, void *value
, u64 flags
);
102 long (*map_pop_elem
)(struct bpf_map
*map
, void *value
);
103 long (*map_peek_elem
)(struct bpf_map
*map
, void *value
);
104 void *(*map_lookup_percpu_elem
)(struct bpf_map
*map
, void *key
, u32 cpu
);
106 /* funcs called by prog_array and perf_event_array map */
107 void *(*map_fd_get_ptr
)(struct bpf_map
*map
, struct file
*map_file
,
109 void (*map_fd_put_ptr
)(void *ptr
);
110 int (*map_gen_lookup
)(struct bpf_map
*map
, struct bpf_insn
*insn_buf
);
111 u32 (*map_fd_sys_lookup_elem
)(void *ptr
);
112 void (*map_seq_show_elem
)(struct bpf_map
*map
, void *key
,
114 int (*map_check_btf
)(const struct bpf_map
*map
,
115 const struct btf
*btf
,
116 const struct btf_type
*key_type
,
117 const struct btf_type
*value_type
);
119 /* Prog poke tracking helpers. */
120 int (*map_poke_track
)(struct bpf_map
*map
, struct bpf_prog_aux
*aux
);
121 void (*map_poke_untrack
)(struct bpf_map
*map
, struct bpf_prog_aux
*aux
);
122 void (*map_poke_run
)(struct bpf_map
*map
, u32 key
, struct bpf_prog
*old
,
123 struct bpf_prog
*new);
125 /* Direct value access helpers. */
126 int (*map_direct_value_addr
)(const struct bpf_map
*map
,
128 int (*map_direct_value_meta
)(const struct bpf_map
*map
,
130 int (*map_mmap
)(struct bpf_map
*map
, struct vm_area_struct
*vma
);
131 __poll_t (*map_poll
)(struct bpf_map
*map
, struct file
*filp
,
132 struct poll_table_struct
*pts
);
134 /* Functions called by bpf_local_storage maps */
135 int (*map_local_storage_charge
)(struct bpf_local_storage_map
*smap
,
136 void *owner
, u32 size
);
137 void (*map_local_storage_uncharge
)(struct bpf_local_storage_map
*smap
,
138 void *owner
, u32 size
);
139 struct bpf_local_storage __rcu
** (*map_owner_storage_ptr
)(void *owner
);
142 long (*map_redirect
)(struct bpf_map
*map
, u64 key
, u64 flags
);
144 /* map_meta_equal must be implemented for maps that can be
145 * used as an inner map. It is a runtime check to ensure
146 * an inner map can be inserted to an outer map.
148 * Some properties of the inner map has been used during the
149 * verification time. When inserting an inner map at the runtime,
150 * map_meta_equal has to ensure the inserting map has the same
151 * properties that the verifier has used earlier.
153 bool (*map_meta_equal
)(const struct bpf_map
*meta0
,
154 const struct bpf_map
*meta1
);
157 int (*map_set_for_each_callback_args
)(struct bpf_verifier_env
*env
,
158 struct bpf_func_state
*caller
,
159 struct bpf_func_state
*callee
);
160 long (*map_for_each_callback
)(struct bpf_map
*map
,
161 bpf_callback_t callback_fn
,
162 void *callback_ctx
, u64 flags
);
164 u64 (*map_mem_usage
)(const struct bpf_map
*map
);
166 /* BTF id of struct allocated by map_alloc */
169 /* bpf_iter info used to open a seq_file */
170 const struct bpf_iter_seq_info
*iter_seq_info
;
174 /* Support at most 10 fields in a BTF type */
178 enum btf_field_type
{
179 BPF_SPIN_LOCK
= (1 << 0),
180 BPF_TIMER
= (1 << 1),
181 BPF_KPTR_UNREF
= (1 << 2),
182 BPF_KPTR_REF
= (1 << 3),
183 BPF_KPTR_PERCPU
= (1 << 4),
184 BPF_KPTR
= BPF_KPTR_UNREF
| BPF_KPTR_REF
| BPF_KPTR_PERCPU
,
185 BPF_LIST_HEAD
= (1 << 5),
186 BPF_LIST_NODE
= (1 << 6),
187 BPF_RB_ROOT
= (1 << 7),
188 BPF_RB_NODE
= (1 << 8),
189 BPF_GRAPH_NODE_OR_ROOT
= BPF_LIST_NODE
| BPF_LIST_HEAD
|
190 BPF_RB_NODE
| BPF_RB_ROOT
,
191 BPF_REFCOUNT
= (1 << 9),
194 typedef void (*btf_dtor_kfunc_t
)(void *);
196 struct btf_field_kptr
{
198 struct module
*module
;
199 /* dtor used if btf_is_kernel(btf), otherwise the type is
200 * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used
202 btf_dtor_kfunc_t dtor
;
206 struct btf_field_graph_root
{
210 struct btf_record
*value_rec
;
216 enum btf_field_type type
;
218 struct btf_field_kptr kptr
;
219 struct btf_field_graph_root graph_root
;
229 struct btf_field fields
[];
232 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
233 struct bpf_rb_node_kern
{
234 struct rb_node rb_node
;
236 } __attribute__((aligned(8)));
238 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
239 struct bpf_list_node_kern
{
240 struct list_head list_head
;
242 } __attribute__((aligned(8)));
245 /* The first two cachelines with read-mostly members of which some
246 * are also accessed in fast-path (e.g. ops, max_entries).
248 const struct bpf_map_ops
*ops ____cacheline_aligned
;
249 struct bpf_map
*inner_map_meta
;
250 #ifdef CONFIG_SECURITY
253 enum bpf_map_type map_type
;
257 u64 map_extra
; /* any per-map-type extra fields */
260 struct btf_record
*record
;
263 u32 btf_value_type_id
;
264 u32 btf_vmlinux_value_type_id
;
266 #ifdef CONFIG_MEMCG_KMEM
267 struct obj_cgroup
*objcg
;
269 char name
[BPF_OBJ_NAME_LEN
];
270 /* The 3rd and 4th cacheline with misc members to avoid false sharing
271 * particularly with refcounting.
273 atomic64_t refcnt ____cacheline_aligned
;
275 struct work_struct work
;
276 struct mutex freeze_mutex
;
278 /* 'Ownership' of program-containing map is claimed by the first program
279 * that is going to use this map or by the first program which FD is
280 * stored in the map to make sure that all callers and callees have the
281 * same prog type, JITed flag and xdp_has_frags flag.
285 enum bpf_prog_type type
;
290 bool frozen
; /* write-once; write-protected by freeze_mutex */
291 s64 __percpu
*elem_count
;
294 static inline const char *btf_field_type_name(enum btf_field_type type
)
298 return "bpf_spin_lock";
304 case BPF_KPTR_PERCPU
:
305 return "percpu_kptr";
307 return "bpf_list_head";
309 return "bpf_list_node";
311 return "bpf_rb_root";
313 return "bpf_rb_node";
315 return "bpf_refcount";
322 static inline u32
btf_field_type_size(enum btf_field_type type
)
326 return sizeof(struct bpf_spin_lock
);
328 return sizeof(struct bpf_timer
);
331 case BPF_KPTR_PERCPU
:
334 return sizeof(struct bpf_list_head
);
336 return sizeof(struct bpf_list_node
);
338 return sizeof(struct bpf_rb_root
);
340 return sizeof(struct bpf_rb_node
);
342 return sizeof(struct bpf_refcount
);
349 static inline u32
btf_field_type_align(enum btf_field_type type
)
353 return __alignof__(struct bpf_spin_lock
);
355 return __alignof__(struct bpf_timer
);
358 case BPF_KPTR_PERCPU
:
359 return __alignof__(u64
);
361 return __alignof__(struct bpf_list_head
);
363 return __alignof__(struct bpf_list_node
);
365 return __alignof__(struct bpf_rb_root
);
367 return __alignof__(struct bpf_rb_node
);
369 return __alignof__(struct bpf_refcount
);
376 static inline void bpf_obj_init_field(const struct btf_field
*field
, void *addr
)
378 memset(addr
, 0, field
->size
);
380 switch (field
->type
) {
382 refcount_set((refcount_t
*)addr
, 1);
385 RB_CLEAR_NODE((struct rb_node
*)addr
);
389 INIT_LIST_HEAD((struct list_head
*)addr
);
392 /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
397 case BPF_KPTR_PERCPU
:
405 static inline bool btf_record_has_field(const struct btf_record
*rec
, enum btf_field_type type
)
407 if (IS_ERR_OR_NULL(rec
))
409 return rec
->field_mask
& type
;
412 static inline void bpf_obj_init(const struct btf_record
*rec
, void *obj
)
416 if (IS_ERR_OR_NULL(rec
))
418 for (i
= 0; i
< rec
->cnt
; i
++)
419 bpf_obj_init_field(&rec
->fields
[i
], obj
+ rec
->fields
[i
].offset
);
422 /* 'dst' must be a temporary buffer and should not point to memory that is being
423 * used in parallel by a bpf program or bpf syscall, otherwise the access from
424 * the bpf program or bpf syscall may be corrupted by the reinitialization,
425 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
426 * allocator, it is still possible for 'dst' to be used in parallel by a bpf
427 * program or bpf syscall.
429 static inline void check_and_init_map_value(struct bpf_map
*map
, void *dst
)
431 bpf_obj_init(map
->record
, dst
);
434 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
435 * forced to use 'long' read/writes to try to atomically copy long counters.
436 * Best-effort only. No barriers here, since it _will_ race with concurrent
437 * updates from BPF programs. Called from bpf syscall and mostly used with
438 * size 8 or 16 bytes, so ask compiler to inline it.
440 static inline void bpf_long_memcpy(void *dst
, const void *src
, u32 size
)
442 const long *lsrc
= src
;
445 size
/= sizeof(long);
447 data_race(*ldst
++ = *lsrc
++);
450 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
451 static inline void bpf_obj_memcpy(struct btf_record
*rec
,
452 void *dst
, void *src
, u32 size
,
458 if (IS_ERR_OR_NULL(rec
)) {
460 bpf_long_memcpy(dst
, src
, round_up(size
, 8));
462 memcpy(dst
, src
, size
);
466 for (i
= 0; i
< rec
->cnt
; i
++) {
467 u32 next_off
= rec
->fields
[i
].offset
;
468 u32 sz
= next_off
- curr_off
;
470 memcpy(dst
+ curr_off
, src
+ curr_off
, sz
);
471 curr_off
+= rec
->fields
[i
].size
+ sz
;
473 memcpy(dst
+ curr_off
, src
+ curr_off
, size
- curr_off
);
476 static inline void copy_map_value(struct bpf_map
*map
, void *dst
, void *src
)
478 bpf_obj_memcpy(map
->record
, dst
, src
, map
->value_size
, false);
481 static inline void copy_map_value_long(struct bpf_map
*map
, void *dst
, void *src
)
483 bpf_obj_memcpy(map
->record
, dst
, src
, map
->value_size
, true);
486 static inline void bpf_obj_memzero(struct btf_record
*rec
, void *dst
, u32 size
)
491 if (IS_ERR_OR_NULL(rec
)) {
492 memset(dst
, 0, size
);
496 for (i
= 0; i
< rec
->cnt
; i
++) {
497 u32 next_off
= rec
->fields
[i
].offset
;
498 u32 sz
= next_off
- curr_off
;
500 memset(dst
+ curr_off
, 0, sz
);
501 curr_off
+= rec
->fields
[i
].size
+ sz
;
503 memset(dst
+ curr_off
, 0, size
- curr_off
);
506 static inline void zero_map_value(struct bpf_map
*map
, void *dst
)
508 bpf_obj_memzero(map
->record
, dst
, map
->value_size
);
511 void copy_map_value_locked(struct bpf_map
*map
, void *dst
, void *src
,
513 void bpf_timer_cancel_and_free(void *timer
);
514 void bpf_list_head_free(const struct btf_field
*field
, void *list_head
,
515 struct bpf_spin_lock
*spin_lock
);
516 void bpf_rb_root_free(const struct btf_field
*field
, void *rb_root
,
517 struct bpf_spin_lock
*spin_lock
);
520 int bpf_obj_name_cpy(char *dst
, const char *src
, unsigned int size
);
522 struct bpf_offload_dev
;
523 struct bpf_offloaded_map
;
525 struct bpf_map_dev_ops
{
526 int (*map_get_next_key
)(struct bpf_offloaded_map
*map
,
527 void *key
, void *next_key
);
528 int (*map_lookup_elem
)(struct bpf_offloaded_map
*map
,
529 void *key
, void *value
);
530 int (*map_update_elem
)(struct bpf_offloaded_map
*map
,
531 void *key
, void *value
, u64 flags
);
532 int (*map_delete_elem
)(struct bpf_offloaded_map
*map
, void *key
);
535 struct bpf_offloaded_map
{
537 struct net_device
*netdev
;
538 const struct bpf_map_dev_ops
*dev_ops
;
540 struct list_head offloads
;
543 static inline struct bpf_offloaded_map
*map_to_offmap(struct bpf_map
*map
)
545 return container_of(map
, struct bpf_offloaded_map
, map
);
548 static inline bool bpf_map_offload_neutral(const struct bpf_map
*map
)
550 return map
->map_type
== BPF_MAP_TYPE_PERF_EVENT_ARRAY
;
553 static inline bool bpf_map_support_seq_show(const struct bpf_map
*map
)
555 return (map
->btf_value_type_id
|| map
->btf_vmlinux_value_type_id
) &&
556 map
->ops
->map_seq_show_elem
;
559 int map_check_no_btf(const struct bpf_map
*map
,
560 const struct btf
*btf
,
561 const struct btf_type
*key_type
,
562 const struct btf_type
*value_type
);
564 bool bpf_map_meta_equal(const struct bpf_map
*meta0
,
565 const struct bpf_map
*meta1
);
567 extern const struct bpf_map_ops bpf_map_offload_ops
;
569 /* bpf_type_flag contains a set of flags that are applicable to the values of
570 * arg_type, ret_type and reg_type. For example, a pointer value may be null,
571 * or a memory is read-only. We classify types into two categories: base types
572 * and extended types. Extended types are base types combined with a type flag.
574 * Currently there are no more than 32 base types in arg_type, ret_type and
577 #define BPF_BASE_TYPE_BITS 8
580 /* PTR may be NULL. */
581 PTR_MAYBE_NULL
= BIT(0 + BPF_BASE_TYPE_BITS
),
583 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
584 * compatible with both mutable and immutable memory.
586 MEM_RDONLY
= BIT(1 + BPF_BASE_TYPE_BITS
),
588 /* MEM points to BPF ring buffer reservation. */
589 MEM_RINGBUF
= BIT(2 + BPF_BASE_TYPE_BITS
),
591 /* MEM is in user address space. */
592 MEM_USER
= BIT(3 + BPF_BASE_TYPE_BITS
),
594 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
595 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
596 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
597 * or bpf_this_cpu_ptr(), which will return the pointer corresponding
598 * to the specified cpu.
600 MEM_PERCPU
= BIT(4 + BPF_BASE_TYPE_BITS
),
602 /* Indicates that the argument will be released. */
603 OBJ_RELEASE
= BIT(5 + BPF_BASE_TYPE_BITS
),
605 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
606 * unreferenced and referenced kptr loaded from map value using a load
607 * instruction, so that they can only be dereferenced but not escape the
608 * BPF program into the kernel (i.e. cannot be passed as arguments to
609 * kfunc or bpf helpers).
611 PTR_UNTRUSTED
= BIT(6 + BPF_BASE_TYPE_BITS
),
613 MEM_UNINIT
= BIT(7 + BPF_BASE_TYPE_BITS
),
615 /* DYNPTR points to memory local to the bpf program. */
616 DYNPTR_TYPE_LOCAL
= BIT(8 + BPF_BASE_TYPE_BITS
),
618 /* DYNPTR points to a kernel-produced ringbuf record. */
619 DYNPTR_TYPE_RINGBUF
= BIT(9 + BPF_BASE_TYPE_BITS
),
621 /* Size is known at compile time. */
622 MEM_FIXED_SIZE
= BIT(10 + BPF_BASE_TYPE_BITS
),
624 /* MEM is of an allocated object of type in program BTF. This is used to
625 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
627 MEM_ALLOC
= BIT(11 + BPF_BASE_TYPE_BITS
),
629 /* PTR was passed from the kernel in a trusted context, and may be
630 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
631 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
632 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
633 * without invoking bpf_kptr_xchg(). What we really need to know is
634 * whether a pointer is safe to pass to a kfunc or BPF helper function.
635 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
636 * helpers, they do not cover all possible instances of unsafe
637 * pointers. For example, a pointer that was obtained from walking a
638 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
639 * fact that it may be NULL, invalid, etc. This is due to backwards
640 * compatibility requirements, as this was the behavior that was first
641 * introduced when kptrs were added. The behavior is now considered
642 * deprecated, and PTR_UNTRUSTED will eventually be removed.
644 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
645 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
646 * For example, pointers passed to tracepoint arguments are considered
647 * PTR_TRUSTED, as are pointers that are passed to struct_ops
648 * callbacks. As alluded to above, pointers that are obtained from
649 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
650 * struct task_struct *task is PTR_TRUSTED, then accessing
651 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
652 * in a BPF register. Similarly, pointers passed to certain programs
653 * types such as kretprobes are not guaranteed to be valid, as they may
654 * for example contain an object that was recently freed.
656 PTR_TRUSTED
= BIT(12 + BPF_BASE_TYPE_BITS
),
658 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
659 MEM_RCU
= BIT(13 + BPF_BASE_TYPE_BITS
),
661 /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
662 * Currently only valid for linked-list and rbtree nodes. If the nodes
663 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
665 NON_OWN_REF
= BIT(14 + BPF_BASE_TYPE_BITS
),
667 /* DYNPTR points to sk_buff */
668 DYNPTR_TYPE_SKB
= BIT(15 + BPF_BASE_TYPE_BITS
),
670 /* DYNPTR points to xdp_buff */
671 DYNPTR_TYPE_XDP
= BIT(16 + BPF_BASE_TYPE_BITS
),
674 __BPF_TYPE_LAST_FLAG
= __BPF_TYPE_FLAG_MAX
- 1,
677 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
680 /* Max number of base types. */
681 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
683 /* Max number of all types. */
684 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
686 /* function argument constraints */
688 ARG_DONTCARE
= 0, /* unused argument in helper function */
690 /* the following constraints used to prototype
691 * bpf_map_lookup/update/delete_elem() functions
693 ARG_CONST_MAP_PTR
, /* const argument used as pointer to bpf_map */
694 ARG_PTR_TO_MAP_KEY
, /* pointer to stack used as map key */
695 ARG_PTR_TO_MAP_VALUE
, /* pointer to stack used as map value */
697 /* Used to prototype bpf_memcmp() and other functions that access data
698 * on eBPF program stack
700 ARG_PTR_TO_MEM
, /* pointer to valid memory (stack, packet, map value) */
702 ARG_CONST_SIZE
, /* number of bytes accessed from memory */
703 ARG_CONST_SIZE_OR_ZERO
, /* number of bytes accessed from memory or 0 */
705 ARG_PTR_TO_CTX
, /* pointer to context */
706 ARG_ANYTHING
, /* any (initialized) argument is ok */
707 ARG_PTR_TO_SPIN_LOCK
, /* pointer to bpf_spin_lock */
708 ARG_PTR_TO_SOCK_COMMON
, /* pointer to sock_common */
709 ARG_PTR_TO_INT
, /* pointer to int */
710 ARG_PTR_TO_LONG
, /* pointer to long */
711 ARG_PTR_TO_SOCKET
, /* pointer to bpf_sock (fullsock) */
712 ARG_PTR_TO_BTF_ID
, /* pointer to in-kernel struct */
713 ARG_PTR_TO_RINGBUF_MEM
, /* pointer to dynamically reserved ringbuf memory */
714 ARG_CONST_ALLOC_SIZE_OR_ZERO
, /* number of allocated bytes requested */
715 ARG_PTR_TO_BTF_ID_SOCK_COMMON
, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
716 ARG_PTR_TO_PERCPU_BTF_ID
, /* pointer to in-kernel percpu type */
717 ARG_PTR_TO_FUNC
, /* pointer to a bpf program function */
718 ARG_PTR_TO_STACK
, /* pointer to stack */
719 ARG_PTR_TO_CONST_STR
, /* pointer to a null terminated read-only string */
720 ARG_PTR_TO_TIMER
, /* pointer to bpf_timer */
721 ARG_PTR_TO_KPTR
, /* pointer to referenced kptr */
722 ARG_PTR_TO_DYNPTR
, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
725 /* Extended arg_types. */
726 ARG_PTR_TO_MAP_VALUE_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_MAP_VALUE
,
727 ARG_PTR_TO_MEM_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_MEM
,
728 ARG_PTR_TO_CTX_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_CTX
,
729 ARG_PTR_TO_SOCKET_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_SOCKET
,
730 ARG_PTR_TO_STACK_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_STACK
,
731 ARG_PTR_TO_BTF_ID_OR_NULL
= PTR_MAYBE_NULL
| ARG_PTR_TO_BTF_ID
,
732 /* pointer to memory does not need to be initialized, helper function must fill
733 * all bytes or clear them in error case.
735 ARG_PTR_TO_UNINIT_MEM
= MEM_UNINIT
| ARG_PTR_TO_MEM
,
736 /* Pointer to valid memory of size known at compile time. */
737 ARG_PTR_TO_FIXED_SIZE_MEM
= MEM_FIXED_SIZE
| ARG_PTR_TO_MEM
,
739 /* This must be the last entry. Its purpose is to ensure the enum is
740 * wide enough to hold the higher bits reserved for bpf_type_flag.
742 __BPF_ARG_TYPE_LIMIT
= BPF_TYPE_LIMIT
,
744 static_assert(__BPF_ARG_TYPE_MAX
<= BPF_BASE_TYPE_LIMIT
);
746 /* type of values returned from helper functions */
747 enum bpf_return_type
{
748 RET_INTEGER
, /* function returns integer */
749 RET_VOID
, /* function doesn't return anything */
750 RET_PTR_TO_MAP_VALUE
, /* returns a pointer to map elem value */
751 RET_PTR_TO_SOCKET
, /* returns a pointer to a socket */
752 RET_PTR_TO_TCP_SOCK
, /* returns a pointer to a tcp_sock */
753 RET_PTR_TO_SOCK_COMMON
, /* returns a pointer to a sock_common */
754 RET_PTR_TO_MEM
, /* returns a pointer to memory */
755 RET_PTR_TO_MEM_OR_BTF_ID
, /* returns a pointer to a valid memory or a btf_id */
756 RET_PTR_TO_BTF_ID
, /* returns a pointer to a btf_id */
759 /* Extended ret_types. */
760 RET_PTR_TO_MAP_VALUE_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_MAP_VALUE
,
761 RET_PTR_TO_SOCKET_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_SOCKET
,
762 RET_PTR_TO_TCP_SOCK_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_TCP_SOCK
,
763 RET_PTR_TO_SOCK_COMMON_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_SOCK_COMMON
,
764 RET_PTR_TO_RINGBUF_MEM_OR_NULL
= PTR_MAYBE_NULL
| MEM_RINGBUF
| RET_PTR_TO_MEM
,
765 RET_PTR_TO_DYNPTR_MEM_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_MEM
,
766 RET_PTR_TO_BTF_ID_OR_NULL
= PTR_MAYBE_NULL
| RET_PTR_TO_BTF_ID
,
767 RET_PTR_TO_BTF_ID_TRUSTED
= PTR_TRUSTED
| RET_PTR_TO_BTF_ID
,
769 /* This must be the last entry. Its purpose is to ensure the enum is
770 * wide enough to hold the higher bits reserved for bpf_type_flag.
772 __BPF_RET_TYPE_LIMIT
= BPF_TYPE_LIMIT
,
774 static_assert(__BPF_RET_TYPE_MAX
<= BPF_BASE_TYPE_LIMIT
);
776 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
777 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
778 * instructions after verifying
780 struct bpf_func_proto
{
781 u64 (*func
)(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
);
785 enum bpf_return_type ret_type
;
788 enum bpf_arg_type arg1_type
;
789 enum bpf_arg_type arg2_type
;
790 enum bpf_arg_type arg3_type
;
791 enum bpf_arg_type arg4_type
;
792 enum bpf_arg_type arg5_type
;
794 enum bpf_arg_type arg_type
[5];
814 int *ret_btf_id
; /* return value btf_id */
815 bool (*allowed
)(const struct bpf_prog
*prog
);
818 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
819 * the first argument to eBPF programs.
820 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
824 enum bpf_access_type
{
829 /* types of values stored in eBPF registers */
830 /* Pointer types represent:
833 * pointer + (u16) var
834 * pointer + (u16) var + imm
835 * if (range > 0) then [ptr, ptr + range - off) is safe to access
836 * if (id > 0) means that some 'var' was added
837 * if (off > 0) means that 'imm' was added
840 NOT_INIT
= 0, /* nothing was written into register */
841 SCALAR_VALUE
, /* reg doesn't contain a valid pointer */
842 PTR_TO_CTX
, /* reg points to bpf_context */
843 CONST_PTR_TO_MAP
, /* reg points to struct bpf_map */
844 PTR_TO_MAP_VALUE
, /* reg points to map element value */
845 PTR_TO_MAP_KEY
, /* reg points to a map element key */
846 PTR_TO_STACK
, /* reg == frame_pointer + offset */
847 PTR_TO_PACKET_META
, /* skb->data - meta_len */
848 PTR_TO_PACKET
, /* reg points to skb->data */
849 PTR_TO_PACKET_END
, /* skb->data + headlen */
850 PTR_TO_FLOW_KEYS
, /* reg points to bpf_flow_keys */
851 PTR_TO_SOCKET
, /* reg points to struct bpf_sock */
852 PTR_TO_SOCK_COMMON
, /* reg points to sock_common */
853 PTR_TO_TCP_SOCK
, /* reg points to struct tcp_sock */
854 PTR_TO_TP_BUFFER
, /* reg points to a writable raw tp's buffer */
855 PTR_TO_XDP_SOCK
, /* reg points to struct xdp_sock */
856 /* PTR_TO_BTF_ID points to a kernel struct that does not need
857 * to be null checked by the BPF program. This does not imply the
858 * pointer is _not_ null and in practice this can easily be a null
859 * pointer when reading pointer chains. The assumption is program
860 * context will handle null pointer dereference typically via fault
861 * handling. The verifier must keep this in mind and can make no
862 * assumptions about null or non-null when doing branch analysis.
863 * Further, when passed into helpers the helpers can not, without
864 * additional context, assume the value is non-null.
867 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
868 * been checked for null. Used primarily to inform the verifier
869 * an explicit null check is required for this struct.
871 PTR_TO_MEM
, /* reg points to valid memory region */
872 PTR_TO_BUF
, /* reg points to a read/write buffer */
873 PTR_TO_FUNC
, /* reg points to a bpf program function */
874 CONST_PTR_TO_DYNPTR
, /* reg points to a const struct bpf_dynptr */
877 /* Extended reg_types. */
878 PTR_TO_MAP_VALUE_OR_NULL
= PTR_MAYBE_NULL
| PTR_TO_MAP_VALUE
,
879 PTR_TO_SOCKET_OR_NULL
= PTR_MAYBE_NULL
| PTR_TO_SOCKET
,
880 PTR_TO_SOCK_COMMON_OR_NULL
= PTR_MAYBE_NULL
| PTR_TO_SOCK_COMMON
,
881 PTR_TO_TCP_SOCK_OR_NULL
= PTR_MAYBE_NULL
| PTR_TO_TCP_SOCK
,
882 PTR_TO_BTF_ID_OR_NULL
= PTR_MAYBE_NULL
| PTR_TO_BTF_ID
,
884 /* This must be the last entry. Its purpose is to ensure the enum is
885 * wide enough to hold the higher bits reserved for bpf_type_flag.
887 __BPF_REG_TYPE_LIMIT
= BPF_TYPE_LIMIT
,
889 static_assert(__BPF_REG_TYPE_MAX
<= BPF_BASE_TYPE_LIMIT
);
891 /* The information passed from prog-specific *_is_valid_access
892 * back to the verifier.
894 struct bpf_insn_access_aux
{
895 enum bpf_reg_type reg_type
;
903 struct bpf_verifier_log
*log
; /* for verbose logs */
907 bpf_ctx_record_field_size(struct bpf_insn_access_aux
*aux
, u32 size
)
909 aux
->ctx_field_size
= size
;
912 static inline bool bpf_pseudo_func(const struct bpf_insn
*insn
)
914 return insn
->code
== (BPF_LD
| BPF_IMM
| BPF_DW
) &&
915 insn
->src_reg
== BPF_PSEUDO_FUNC
;
918 struct bpf_prog_ops
{
919 int (*test_run
)(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
920 union bpf_attr __user
*uattr
);
923 struct bpf_reg_state
;
924 struct bpf_verifier_ops
{
925 /* return eBPF function prototype for verification */
926 const struct bpf_func_proto
*
927 (*get_func_proto
)(enum bpf_func_id func_id
,
928 const struct bpf_prog
*prog
);
930 /* return true if 'size' wide access at offset 'off' within bpf_context
931 * with 'type' (read or write) is allowed
933 bool (*is_valid_access
)(int off
, int size
, enum bpf_access_type type
,
934 const struct bpf_prog
*prog
,
935 struct bpf_insn_access_aux
*info
);
936 int (*gen_prologue
)(struct bpf_insn
*insn
, bool direct_write
,
937 const struct bpf_prog
*prog
);
938 int (*gen_ld_abs
)(const struct bpf_insn
*orig
,
939 struct bpf_insn
*insn_buf
);
940 u32 (*convert_ctx_access
)(enum bpf_access_type type
,
941 const struct bpf_insn
*src
,
942 struct bpf_insn
*dst
,
943 struct bpf_prog
*prog
, u32
*target_size
);
944 int (*btf_struct_access
)(struct bpf_verifier_log
*log
,
945 const struct bpf_reg_state
*reg
,
949 struct bpf_prog_offload_ops
{
950 /* verifier basic callbacks */
951 int (*insn_hook
)(struct bpf_verifier_env
*env
,
952 int insn_idx
, int prev_insn_idx
);
953 int (*finalize
)(struct bpf_verifier_env
*env
);
954 /* verifier optimization callbacks (called after .finalize) */
955 int (*replace_insn
)(struct bpf_verifier_env
*env
, u32 off
,
956 struct bpf_insn
*insn
);
957 int (*remove_insns
)(struct bpf_verifier_env
*env
, u32 off
, u32 cnt
);
958 /* program management callbacks */
959 int (*prepare
)(struct bpf_prog
*prog
);
960 int (*translate
)(struct bpf_prog
*prog
);
961 void (*destroy
)(struct bpf_prog
*prog
);
964 struct bpf_prog_offload
{
965 struct bpf_prog
*prog
;
966 struct net_device
*netdev
;
967 struct bpf_offload_dev
*offdev
;
969 struct list_head offloads
;
976 enum bpf_cgroup_storage_type
{
977 BPF_CGROUP_STORAGE_SHARED
,
978 BPF_CGROUP_STORAGE_PERCPU
,
979 __BPF_CGROUP_STORAGE_MAX
982 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
984 /* The longest tracepoint has 12 args.
985 * See include/trace/bpf_probe.h
987 #define MAX_BPF_FUNC_ARGS 12
989 /* The maximum number of arguments passed through registers
990 * a single function may have.
992 #define MAX_BPF_FUNC_REG_ARGS 5
994 /* The argument is a structure. */
995 #define BTF_FMODEL_STRUCT_ARG BIT(0)
997 /* The argument is signed. */
998 #define BTF_FMODEL_SIGNED_ARG BIT(1)
1000 struct btf_func_model
{
1004 u8 arg_size
[MAX_BPF_FUNC_ARGS
];
1005 u8 arg_flags
[MAX_BPF_FUNC_ARGS
];
1008 /* Restore arguments before returning from trampoline to let original function
1009 * continue executing. This flag is used for fentry progs when there are no
1012 #define BPF_TRAMP_F_RESTORE_REGS BIT(0)
1013 /* Call original function after fentry progs, but before fexit progs.
1014 * Makes sense for fentry/fexit, normal calls and indirect calls.
1016 #define BPF_TRAMP_F_CALL_ORIG BIT(1)
1017 /* Skip current frame and return to parent. Makes sense for fentry/fexit
1018 * programs only. Should not be used with normal calls and indirect calls.
1020 #define BPF_TRAMP_F_SKIP_FRAME BIT(2)
1021 /* Store IP address of the caller on the trampoline stack,
1022 * so it's available for trampoline's programs.
1024 #define BPF_TRAMP_F_IP_ARG BIT(3)
1025 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1026 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
1028 /* Get original function from stack instead of from provided direct address.
1029 * Makes sense for trampolines with fexit or fmod_ret programs.
1031 #define BPF_TRAMP_F_ORIG_STACK BIT(5)
1033 /* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1034 * e.g., a live patch. This flag is set and cleared by ftrace call backs,
1036 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6)
1038 /* Indicate that current trampoline is in a tail call context. Then, it has to
1039 * cache and restore tail_call_cnt to avoid infinite tail call loop.
1041 #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7)
1043 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1047 #if defined(__s390x__)
1048 BPF_MAX_TRAMP_LINKS
= 27,
1050 BPF_MAX_TRAMP_LINKS
= 38,
1054 struct bpf_tramp_links
{
1055 struct bpf_tramp_link
*links
[BPF_MAX_TRAMP_LINKS
];
1059 struct bpf_tramp_run_ctx
;
1061 /* Different use cases for BPF trampoline:
1062 * 1. replace nop at the function entry (kprobe equivalent)
1063 * flags = BPF_TRAMP_F_RESTORE_REGS
1064 * fentry = a set of programs to run before returning from trampoline
1066 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1067 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1068 * orig_call = fentry_ip + MCOUNT_INSN_SIZE
1069 * fentry = a set of program to run before calling original function
1070 * fexit = a set of program to run after original function
1072 * 3. replace direct call instruction anywhere in the function body
1073 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1075 * fentry = a set of programs to run before returning from trampoline
1076 * With flags = BPF_TRAMP_F_CALL_ORIG
1077 * orig_call = original callback addr or direct function addr
1078 * fentry = a set of program to run before calling original function
1079 * fexit = a set of program to run after original function
1081 struct bpf_tramp_image
;
1082 int arch_prepare_bpf_trampoline(struct bpf_tramp_image
*tr
, void *image
, void *image_end
,
1083 const struct btf_func_model
*m
, u32 flags
,
1084 struct bpf_tramp_links
*tlinks
,
1086 u64 notrace
__bpf_prog_enter_sleepable_recur(struct bpf_prog
*prog
,
1087 struct bpf_tramp_run_ctx
*run_ctx
);
1088 void notrace
__bpf_prog_exit_sleepable_recur(struct bpf_prog
*prog
, u64 start
,
1089 struct bpf_tramp_run_ctx
*run_ctx
);
1090 void notrace
__bpf_tramp_enter(struct bpf_tramp_image
*tr
);
1091 void notrace
__bpf_tramp_exit(struct bpf_tramp_image
*tr
);
1092 typedef u64 (*bpf_trampoline_enter_t
)(struct bpf_prog
*prog
,
1093 struct bpf_tramp_run_ctx
*run_ctx
);
1094 typedef void (*bpf_trampoline_exit_t
)(struct bpf_prog
*prog
, u64 start
,
1095 struct bpf_tramp_run_ctx
*run_ctx
);
1096 bpf_trampoline_enter_t
bpf_trampoline_enter(const struct bpf_prog
*prog
);
1097 bpf_trampoline_exit_t
bpf_trampoline_exit(const struct bpf_prog
*prog
);
1100 unsigned long start
;
1102 char name
[KSYM_NAME_LEN
];
1103 struct list_head lnode
;
1104 struct latch_tree_node tnode
;
1108 enum bpf_tramp_prog_type
{
1111 BPF_TRAMP_MODIFY_RETURN
,
1113 BPF_TRAMP_REPLACE
, /* more than MAX */
1116 struct bpf_tramp_image
{
1118 struct bpf_ksym ksym
;
1119 struct percpu_ref pcref
;
1120 void *ip_after_call
;
1123 struct rcu_head rcu
;
1124 struct work_struct work
;
1128 struct bpf_trampoline
{
1129 /* hlist for trampoline_table */
1130 struct hlist_node hlist
;
1131 struct ftrace_ops
*fops
;
1132 /* serializes access to fields of this trampoline */
1138 struct btf_func_model model
;
1140 bool ftrace_managed
;
1142 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1143 * program by replacing one of its functions. func.addr is the address
1144 * of the function it replaced.
1146 struct bpf_prog
*extension_prog
;
1147 /* list of BPF programs using this trampoline */
1148 struct hlist_head progs_hlist
[BPF_TRAMP_MAX
];
1149 /* Number of attached programs. A counter per kind. */
1150 int progs_cnt
[BPF_TRAMP_MAX
];
1151 /* Executable image of trampoline */
1152 struct bpf_tramp_image
*cur_image
;
1156 struct bpf_attach_target_info
{
1157 struct btf_func_model fmodel
;
1159 struct module
*tgt_mod
;
1160 const char *tgt_name
;
1161 const struct btf_type
*tgt_type
;
1164 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1166 struct bpf_dispatcher_prog
{
1167 struct bpf_prog
*prog
;
1171 struct bpf_dispatcher
{
1172 /* dispatcher mutex */
1175 struct bpf_dispatcher_prog progs
[BPF_DISPATCHER_MAX
];
1180 struct bpf_ksym ksym
;
1181 #ifdef CONFIG_HAVE_STATIC_CALL
1182 struct static_call_key
*sc_key
;
1187 static __always_inline __nocfi
unsigned int bpf_dispatcher_nop_func(
1189 const struct bpf_insn
*insnsi
,
1190 bpf_func_t bpf_func
)
1192 return bpf_func(ctx
, insnsi
);
1195 /* the implementation of the opaque uapi struct bpf_dynptr */
1196 struct bpf_dynptr_kern
{
1198 /* Size represents the number of usable bytes of dynptr data.
1199 * If for example the offset is at 4 for a local dynptr whose data is
1200 * of type u64, the number of usable bytes is 4.
1202 * The upper 8 bits are reserved. It is as follows:
1203 * Bits 0 - 23 = size
1204 * Bits 24 - 30 = dynptr type
1205 * Bit 31 = whether dynptr is read-only
1211 enum bpf_dynptr_type
{
1212 BPF_DYNPTR_TYPE_INVALID
,
1213 /* Points to memory that is local to the bpf program */
1214 BPF_DYNPTR_TYPE_LOCAL
,
1215 /* Underlying data is a ringbuf record */
1216 BPF_DYNPTR_TYPE_RINGBUF
,
1217 /* Underlying data is a sk_buff */
1218 BPF_DYNPTR_TYPE_SKB
,
1219 /* Underlying data is a xdp_buff */
1220 BPF_DYNPTR_TYPE_XDP
,
1223 int bpf_dynptr_check_size(u32 size
);
1224 u32
__bpf_dynptr_size(const struct bpf_dynptr_kern
*ptr
);
1226 #ifdef CONFIG_BPF_JIT
1227 int bpf_trampoline_link_prog(struct bpf_tramp_link
*link
, struct bpf_trampoline
*tr
);
1228 int bpf_trampoline_unlink_prog(struct bpf_tramp_link
*link
, struct bpf_trampoline
*tr
);
1229 struct bpf_trampoline
*bpf_trampoline_get(u64 key
,
1230 struct bpf_attach_target_info
*tgt_info
);
1231 void bpf_trampoline_put(struct bpf_trampoline
*tr
);
1232 int arch_prepare_bpf_dispatcher(void *image
, void *buf
, s64
*funcs
, int num_funcs
);
1235 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1236 * indirection with a direct call to the bpf program. If the architecture does
1237 * not have STATIC_CALL, avoid a double-indirection.
1239 #ifdef CONFIG_HAVE_STATIC_CALL
1241 #define __BPF_DISPATCHER_SC_INIT(_name) \
1242 .sc_key = &STATIC_CALL_KEY(_name), \
1243 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1245 #define __BPF_DISPATCHER_SC(name) \
1246 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1248 #define __BPF_DISPATCHER_CALL(name) \
1249 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1251 #define __BPF_DISPATCHER_UPDATE(_d, _new) \
1252 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1255 #define __BPF_DISPATCHER_SC_INIT(name)
1256 #define __BPF_DISPATCHER_SC(name)
1257 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi)
1258 #define __BPF_DISPATCHER_UPDATE(_d, _new)
1261 #define BPF_DISPATCHER_INIT(_name) { \
1262 .mutex = __MUTEX_INITIALIZER(_name.mutex), \
1263 .func = &_name##_func, \
1270 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
1272 __BPF_DISPATCHER_SC_INIT(_name##_call) \
1275 #define DEFINE_BPF_DISPATCHER(name) \
1276 __BPF_DISPATCHER_SC(name); \
1277 noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \
1279 const struct bpf_insn *insnsi, \
1280 bpf_func_t bpf_func) \
1282 return __BPF_DISPATCHER_CALL(name); \
1284 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
1285 struct bpf_dispatcher bpf_dispatcher_##name = \
1286 BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1288 #define DECLARE_BPF_DISPATCHER(name) \
1289 unsigned int bpf_dispatcher_##name##_func( \
1291 const struct bpf_insn *insnsi, \
1292 bpf_func_t bpf_func); \
1293 extern struct bpf_dispatcher bpf_dispatcher_##name;
1295 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1296 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1297 void bpf_dispatcher_change_prog(struct bpf_dispatcher
*d
, struct bpf_prog
*from
,
1298 struct bpf_prog
*to
);
1299 /* Called only from JIT-enabled code, so there's no need for stubs. */
1300 void bpf_image_ksym_add(void *data
, struct bpf_ksym
*ksym
);
1301 void bpf_image_ksym_del(struct bpf_ksym
*ksym
);
1302 void bpf_ksym_add(struct bpf_ksym
*ksym
);
1303 void bpf_ksym_del(struct bpf_ksym
*ksym
);
1304 int bpf_jit_charge_modmem(u32 size
);
1305 void bpf_jit_uncharge_modmem(u32 size
);
1306 bool bpf_prog_has_trampoline(const struct bpf_prog
*prog
);
1308 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link
*link
,
1309 struct bpf_trampoline
*tr
)
1313 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link
*link
,
1314 struct bpf_trampoline
*tr
)
1318 static inline struct bpf_trampoline
*bpf_trampoline_get(u64 key
,
1319 struct bpf_attach_target_info
*tgt_info
)
1323 static inline void bpf_trampoline_put(struct bpf_trampoline
*tr
) {}
1324 #define DEFINE_BPF_DISPATCHER(name)
1325 #define DECLARE_BPF_DISPATCHER(name)
1326 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1327 #define BPF_DISPATCHER_PTR(name) NULL
1328 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher
*d
,
1329 struct bpf_prog
*from
,
1330 struct bpf_prog
*to
) {}
1331 static inline bool is_bpf_image_address(unsigned long address
)
1335 static inline bool bpf_prog_has_trampoline(const struct bpf_prog
*prog
)
1341 struct bpf_func_info_aux
{
1346 enum bpf_jit_poke_reason
{
1347 BPF_POKE_REASON_TAIL_CALL
,
1350 /* Descriptor of pokes pointing /into/ the JITed image. */
1351 struct bpf_jit_poke_descriptor
{
1352 void *tailcall_target
;
1353 void *tailcall_bypass
;
1358 struct bpf_map
*map
;
1362 bool tailcall_target_stable
;
1368 /* reg_type info for ctx arguments */
1369 struct bpf_ctx_arg_aux
{
1371 enum bpf_reg_type reg_type
;
1375 struct btf_mod_pair
{
1377 struct module
*module
;
1380 struct bpf_kfunc_desc_tab
;
1382 struct bpf_prog_aux
{
1391 u32 func_cnt
; /* used by non-func prog as the number of func progs */
1392 u32 real_func_cnt
; /* includes hidden progs, only used for JIT and freeing progs */
1393 u32 func_idx
; /* 0 for non-func prog, the index in func array for func prog */
1394 u32 attach_btf_id
; /* in-kernel BTF type id to attach to */
1395 u32 ctx_arg_info_size
;
1396 u32 max_rdonly_access
;
1397 u32 max_rdwr_access
;
1398 struct btf
*attach_btf
;
1399 const struct bpf_ctx_arg_aux
*ctx_arg_info
;
1400 struct mutex dst_mutex
; /* protects dst_* pointers below, *after* prog becomes visible */
1401 struct bpf_prog
*dst_prog
;
1402 struct bpf_trampoline
*dst_trampoline
;
1403 enum bpf_prog_type saved_dst_prog_type
;
1404 enum bpf_attach_type saved_dst_attach_type
;
1405 bool verifier_zext
; /* Zero extensions has been inserted by verifier. */
1406 bool dev_bound
; /* Program is bound to the netdev. */
1407 bool offload_requested
; /* Program is bound and offloaded to the netdev. */
1408 bool attach_btf_trace
; /* true if attaching to BTF-enabled raw tp */
1409 bool func_proto_unreliable
;
1411 bool tail_call_reachable
;
1414 bool exception_boundary
;
1415 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1416 const struct btf_type
*attach_func_proto
;
1417 /* function name for valid attach_btf_id */
1418 const char *attach_func_name
;
1419 struct bpf_prog
**func
;
1420 void *jit_data
; /* JIT specific data. arch dependent */
1421 struct bpf_jit_poke_descriptor
*poke_tab
;
1422 struct bpf_kfunc_desc_tab
*kfunc_tab
;
1423 struct bpf_kfunc_btf_tab
*kfunc_btf_tab
;
1425 struct bpf_ksym ksym
;
1426 const struct bpf_prog_ops
*ops
;
1427 struct bpf_map
**used_maps
;
1428 struct mutex used_maps_mutex
; /* mutex for used_maps and used_map_cnt */
1429 struct btf_mod_pair
*used_btfs
;
1430 struct bpf_prog
*prog
;
1431 struct user_struct
*user
;
1432 u64 load_time
; /* ns since boottime */
1434 int cgroup_atype
; /* enum cgroup_bpf_attach_type */
1435 struct bpf_map
*cgroup_storage
[MAX_BPF_CGROUP_STORAGE_TYPE
];
1436 char name
[BPF_OBJ_NAME_LEN
];
1437 unsigned int (*bpf_exception_cb
)(u64 cookie
, u64 sp
, u64 bp
);
1438 #ifdef CONFIG_SECURITY
1441 struct bpf_prog_offload
*offload
;
1443 struct bpf_func_info
*func_info
;
1444 struct bpf_func_info_aux
*func_info_aux
;
1445 /* bpf_line_info loaded from userspace. linfo->insn_off
1446 * has the xlated insn offset.
1447 * Both the main and sub prog share the same linfo.
1448 * The subprog can access its first linfo by
1449 * using the linfo_idx.
1451 struct bpf_line_info
*linfo
;
1452 /* jited_linfo is the jited addr of the linfo. It has a
1453 * one to one mapping to linfo:
1454 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1455 * Both the main and sub prog share the same jited_linfo.
1456 * The subprog can access its first jited_linfo by
1457 * using the linfo_idx.
1462 /* subprog can use linfo_idx to access its first linfo and
1464 * main prog always has linfo_idx == 0
1469 struct exception_table_entry
*extable
;
1471 struct work_struct work
;
1472 struct rcu_head rcu
;
1477 u16 pages
; /* Number of allocated pages */
1478 u16 jited
:1, /* Is our filter JIT'ed? */
1479 jit_requested
:1,/* archs need to JIT the prog */
1480 gpl_compatible
:1, /* Is filter GPL compatible? */
1481 cb_access
:1, /* Is control block accessed? */
1482 dst_needed
:1, /* Do we need dst entry? */
1483 blinding_requested
:1, /* needs constant blinding */
1484 blinded
:1, /* Was blinded */
1485 is_func
:1, /* program is a bpf function */
1486 kprobe_override
:1, /* Do we override a kprobe? */
1487 has_callchain_buf
:1, /* callchain buffer allocated? */
1488 enforce_expected_attach_type
:1, /* Enforce expected_attach_type checking at attach time */
1489 call_get_stack
:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1490 call_get_func_ip
:1, /* Do we call get_func_ip() */
1491 tstamp_type_access
:1; /* Accessed __sk_buff->tstamp_type */
1492 enum bpf_prog_type type
; /* Type of BPF program */
1493 enum bpf_attach_type expected_attach_type
; /* For some prog types */
1494 u32 len
; /* Number of filter blocks */
1495 u32 jited_len
; /* Size of jited insns in bytes */
1496 u8 tag
[BPF_TAG_SIZE
];
1497 struct bpf_prog_stats __percpu
*stats
;
1498 int __percpu
*active
;
1499 unsigned int (*bpf_func
)(const void *ctx
,
1500 const struct bpf_insn
*insn
);
1501 struct bpf_prog_aux
*aux
; /* Auxiliary fields */
1502 struct sock_fprog_kern
*orig_prog
; /* Original BPF program */
1503 /* Instructions for interpreter */
1505 DECLARE_FLEX_ARRAY(struct sock_filter
, insns
);
1506 DECLARE_FLEX_ARRAY(struct bpf_insn
, insnsi
);
1510 struct bpf_array_aux
{
1511 /* Programs with direct jumps into programs part of this array. */
1512 struct list_head poke_progs
;
1513 struct bpf_map
*map
;
1514 struct mutex poke_mutex
;
1515 struct work_struct work
;
1521 enum bpf_link_type type
;
1522 const struct bpf_link_ops
*ops
;
1523 struct bpf_prog
*prog
;
1524 struct work_struct work
;
1527 struct bpf_link_ops
{
1528 void (*release
)(struct bpf_link
*link
);
1529 void (*dealloc
)(struct bpf_link
*link
);
1530 int (*detach
)(struct bpf_link
*link
);
1531 int (*update_prog
)(struct bpf_link
*link
, struct bpf_prog
*new_prog
,
1532 struct bpf_prog
*old_prog
);
1533 void (*show_fdinfo
)(const struct bpf_link
*link
, struct seq_file
*seq
);
1534 int (*fill_link_info
)(const struct bpf_link
*link
,
1535 struct bpf_link_info
*info
);
1536 int (*update_map
)(struct bpf_link
*link
, struct bpf_map
*new_map
,
1537 struct bpf_map
*old_map
);
1540 struct bpf_tramp_link
{
1541 struct bpf_link link
;
1542 struct hlist_node tramp_hlist
;
1546 struct bpf_shim_tramp_link
{
1547 struct bpf_tramp_link link
;
1548 struct bpf_trampoline
*trampoline
;
1551 struct bpf_tracing_link
{
1552 struct bpf_tramp_link link
;
1553 enum bpf_attach_type attach_type
;
1554 struct bpf_trampoline
*trampoline
;
1555 struct bpf_prog
*tgt_prog
;
1558 struct bpf_link_primer
{
1559 struct bpf_link
*link
;
1565 struct bpf_struct_ops_value
;
1568 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1570 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1571 * define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1572 * of BPF_PROG_TYPE_STRUCT_OPS progs.
1573 * @verifier_ops: A structure of callbacks that are invoked by the verifier
1574 * when determining whether the struct_ops progs in the
1575 * struct_ops map are valid.
1576 * @init: A callback that is invoked a single time, and before any other
1577 * callback, to initialize the structure. A nonzero return value means
1578 * the subsystem could not be initialized.
1579 * @check_member: When defined, a callback invoked by the verifier to allow
1580 * the subsystem to determine if an entry in the struct_ops map
1581 * is valid. A nonzero return value means that the map is
1582 * invalid and should be rejected by the verifier.
1583 * @init_member: A callback that is invoked for each member of the struct_ops
1584 * map to allow the subsystem to initialize the member. A nonzero
1585 * value means the member could not be initialized. This callback
1586 * is exclusive with the @type, @type_id, @value_type, and
1588 * @reg: A callback that is invoked when the struct_ops map has been
1589 * initialized and is being attached to. Zero means the struct_ops map
1590 * has been successfully registered and is live. A nonzero return value
1591 * means the struct_ops map could not be registered.
1592 * @unreg: A callback that is invoked when the struct_ops map should be
1594 * @update: A callback that is invoked when the live struct_ops map is being
1595 * updated to contain new values. This callback is only invoked when
1596 * the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1597 * it is assumed that the struct_ops map cannot be updated.
1598 * @validate: A callback that is invoked after all of the members have been
1599 * initialized. This callback should perform static checks on the
1600 * map, meaning that it should either fail or succeed
1601 * deterministically. A struct_ops map that has been validated may
1602 * not necessarily succeed in being registered if the call to @reg
1603 * fails. For example, a valid struct_ops map may be loaded, but
1604 * then fail to be registered due to there being another active
1605 * struct_ops map on the system in the subsystem already. For this
1606 * reason, if this callback is not defined, the check is skipped as
1607 * the struct_ops map will have final verification performed in
1610 * @value_type: Value type.
1611 * @name: The name of the struct bpf_struct_ops object.
1612 * @func_models: Func models
1613 * @type_id: BTF type id.
1614 * @value_id: BTF value id.
1616 struct bpf_struct_ops
{
1617 const struct bpf_verifier_ops
*verifier_ops
;
1618 int (*init
)(struct btf
*btf
);
1619 int (*check_member
)(const struct btf_type
*t
,
1620 const struct btf_member
*member
,
1621 const struct bpf_prog
*prog
);
1622 int (*init_member
)(const struct btf_type
*t
,
1623 const struct btf_member
*member
,
1624 void *kdata
, const void *udata
);
1625 int (*reg
)(void *kdata
);
1626 void (*unreg
)(void *kdata
);
1627 int (*update
)(void *kdata
, void *old_kdata
);
1628 int (*validate
)(void *kdata
);
1629 const struct btf_type
*type
;
1630 const struct btf_type
*value_type
;
1632 struct btf_func_model func_models
[BPF_STRUCT_OPS_MAX_NR_MEMBERS
];
1637 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1638 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1639 const struct bpf_struct_ops
*bpf_struct_ops_find(u32 type_id
);
1640 void bpf_struct_ops_init(struct btf
*btf
, struct bpf_verifier_log
*log
);
1641 bool bpf_struct_ops_get(const void *kdata
);
1642 void bpf_struct_ops_put(const void *kdata
);
1643 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map
*map
, void *key
,
1645 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links
*tlinks
,
1646 struct bpf_tramp_link
*link
,
1647 const struct btf_func_model
*model
,
1648 void *image
, void *image_end
);
1649 static inline bool bpf_try_module_get(const void *data
, struct module
*owner
)
1651 if (owner
== BPF_MODULE_OWNER
)
1652 return bpf_struct_ops_get(data
);
1654 return try_module_get(owner
);
1656 static inline void bpf_module_put(const void *data
, struct module
*owner
)
1658 if (owner
== BPF_MODULE_OWNER
)
1659 bpf_struct_ops_put(data
);
1663 int bpf_struct_ops_link_create(union bpf_attr
*attr
);
1666 /* Define it here to avoid the use of forward declaration */
1667 struct bpf_dummy_ops_state
{
1671 struct bpf_dummy_ops
{
1672 int (*test_1
)(struct bpf_dummy_ops_state
*cb
);
1673 int (*test_2
)(struct bpf_dummy_ops_state
*cb
, int a1
, unsigned short a2
,
1674 char a3
, unsigned long a4
);
1675 int (*test_sleepable
)(struct bpf_dummy_ops_state
*cb
);
1678 int bpf_struct_ops_test_run(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
1679 union bpf_attr __user
*uattr
);
1682 static inline const struct bpf_struct_ops
*bpf_struct_ops_find(u32 type_id
)
1686 static inline void bpf_struct_ops_init(struct btf
*btf
,
1687 struct bpf_verifier_log
*log
)
1690 static inline bool bpf_try_module_get(const void *data
, struct module
*owner
)
1692 return try_module_get(owner
);
1694 static inline void bpf_module_put(const void *data
, struct module
*owner
)
1698 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map
*map
,
1704 static inline int bpf_struct_ops_link_create(union bpf_attr
*attr
)
1711 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1712 int bpf_trampoline_link_cgroup_shim(struct bpf_prog
*prog
,
1714 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog
*prog
);
1716 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog
*prog
,
1721 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog
*prog
)
1730 struct bpf_array_aux
*aux
;
1732 DECLARE_FLEX_ARRAY(char, value
) __aligned(8);
1733 DECLARE_FLEX_ARRAY(void *, ptrs
) __aligned(8);
1734 DECLARE_FLEX_ARRAY(void __percpu
*, pptrs
) __aligned(8);
1738 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
1739 #define MAX_TAIL_CALL_CNT 33
1741 /* Maximum number of loops for bpf_loop and bpf_iter_num.
1742 * It's enum to expose it (and thus make it discoverable) through BTF.
1745 BPF_MAX_LOOPS
= 8 * 1024 * 1024,
1748 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
1749 BPF_F_RDONLY_PROG | \
1753 #define BPF_MAP_CAN_READ BIT(0)
1754 #define BPF_MAP_CAN_WRITE BIT(1)
1756 /* Maximum number of user-producer ring buffer samples that can be drained in
1757 * a call to bpf_user_ringbuf_drain().
1759 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1761 static inline u32
bpf_map_flags_to_cap(struct bpf_map
*map
)
1763 u32 access_flags
= map
->map_flags
& (BPF_F_RDONLY_PROG
| BPF_F_WRONLY_PROG
);
1765 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
1768 if (access_flags
& BPF_F_RDONLY_PROG
)
1769 return BPF_MAP_CAN_READ
;
1770 else if (access_flags
& BPF_F_WRONLY_PROG
)
1771 return BPF_MAP_CAN_WRITE
;
1773 return BPF_MAP_CAN_READ
| BPF_MAP_CAN_WRITE
;
1776 static inline bool bpf_map_flags_access_ok(u32 access_flags
)
1778 return (access_flags
& (BPF_F_RDONLY_PROG
| BPF_F_WRONLY_PROG
)) !=
1779 (BPF_F_RDONLY_PROG
| BPF_F_WRONLY_PROG
);
1782 struct bpf_event_entry
{
1783 struct perf_event
*event
;
1784 struct file
*perf_file
;
1785 struct file
*map_file
;
1786 struct rcu_head rcu
;
1789 static inline bool map_type_contains_progs(struct bpf_map
*map
)
1791 return map
->map_type
== BPF_MAP_TYPE_PROG_ARRAY
||
1792 map
->map_type
== BPF_MAP_TYPE_DEVMAP
||
1793 map
->map_type
== BPF_MAP_TYPE_CPUMAP
;
1796 bool bpf_prog_map_compatible(struct bpf_map
*map
, const struct bpf_prog
*fp
);
1797 int bpf_prog_calc_tag(struct bpf_prog
*fp
);
1799 const struct bpf_func_proto
*bpf_get_trace_printk_proto(void);
1800 const struct bpf_func_proto
*bpf_get_trace_vprintk_proto(void);
1802 typedef unsigned long (*bpf_ctx_copy_t
)(void *dst
, const void *src
,
1803 unsigned long off
, unsigned long len
);
1804 typedef u32 (*bpf_convert_ctx_access_t
)(enum bpf_access_type type
,
1805 const struct bpf_insn
*src
,
1806 struct bpf_insn
*dst
,
1807 struct bpf_prog
*prog
,
1810 u64
bpf_event_output(struct bpf_map
*map
, u64 flags
, void *meta
, u64 meta_size
,
1811 void *ctx
, u64 ctx_size
, bpf_ctx_copy_t ctx_copy
);
1813 /* an array of programs to be executed under rcu_lock.
1816 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
1818 * the structure returned by bpf_prog_array_alloc() should be populated
1819 * with program pointers and the last pointer must be NULL.
1820 * The user has to keep refcnt on the program and make sure the program
1821 * is removed from the array before bpf_prog_put().
1822 * The 'struct bpf_prog_array *' should only be replaced with xchg()
1823 * since other cpus are walking the array of pointers in parallel.
1825 struct bpf_prog_array_item
{
1826 struct bpf_prog
*prog
;
1828 struct bpf_cgroup_storage
*cgroup_storage
[MAX_BPF_CGROUP_STORAGE_TYPE
];
1833 struct bpf_prog_array
{
1834 struct rcu_head rcu
;
1835 struct bpf_prog_array_item items
[];
1838 struct bpf_empty_prog_array
{
1839 struct bpf_prog_array hdr
;
1840 struct bpf_prog
*null_prog
;
1843 /* to avoid allocating empty bpf_prog_array for cgroups that
1844 * don't have bpf program attached use one global 'bpf_empty_prog_array'
1845 * It will not be modified the caller of bpf_prog_array_alloc()
1846 * (since caller requested prog_cnt == 0)
1847 * that pointer should be 'freed' by bpf_prog_array_free()
1849 extern struct bpf_empty_prog_array bpf_empty_prog_array
;
1851 struct bpf_prog_array
*bpf_prog_array_alloc(u32 prog_cnt
, gfp_t flags
);
1852 void bpf_prog_array_free(struct bpf_prog_array
*progs
);
1853 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
1854 void bpf_prog_array_free_sleepable(struct bpf_prog_array
*progs
);
1855 int bpf_prog_array_length(struct bpf_prog_array
*progs
);
1856 bool bpf_prog_array_is_empty(struct bpf_prog_array
*array
);
1857 int bpf_prog_array_copy_to_user(struct bpf_prog_array
*progs
,
1858 __u32 __user
*prog_ids
, u32 cnt
);
1860 void bpf_prog_array_delete_safe(struct bpf_prog_array
*progs
,
1861 struct bpf_prog
*old_prog
);
1862 int bpf_prog_array_delete_safe_at(struct bpf_prog_array
*array
, int index
);
1863 int bpf_prog_array_update_at(struct bpf_prog_array
*array
, int index
,
1864 struct bpf_prog
*prog
);
1865 int bpf_prog_array_copy_info(struct bpf_prog_array
*array
,
1866 u32
*prog_ids
, u32 request_cnt
,
1868 int bpf_prog_array_copy(struct bpf_prog_array
*old_array
,
1869 struct bpf_prog
*exclude_prog
,
1870 struct bpf_prog
*include_prog
,
1872 struct bpf_prog_array
**new_array
);
1874 struct bpf_run_ctx
{};
1876 struct bpf_cg_run_ctx
{
1877 struct bpf_run_ctx run_ctx
;
1878 const struct bpf_prog_array_item
*prog_item
;
1882 struct bpf_trace_run_ctx
{
1883 struct bpf_run_ctx run_ctx
;
1888 struct bpf_tramp_run_ctx
{
1889 struct bpf_run_ctx run_ctx
;
1891 struct bpf_run_ctx
*saved_run_ctx
;
1894 static inline struct bpf_run_ctx
*bpf_set_run_ctx(struct bpf_run_ctx
*new_ctx
)
1896 struct bpf_run_ctx
*old_ctx
= NULL
;
1898 #ifdef CONFIG_BPF_SYSCALL
1899 old_ctx
= current
->bpf_ctx
;
1900 current
->bpf_ctx
= new_ctx
;
1905 static inline void bpf_reset_run_ctx(struct bpf_run_ctx
*old_ctx
)
1907 #ifdef CONFIG_BPF_SYSCALL
1908 current
->bpf_ctx
= old_ctx
;
1912 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
1913 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
1914 /* BPF program asks to set CN on the packet. */
1915 #define BPF_RET_SET_CN (1 << 0)
1917 typedef u32 (*bpf_prog_run_fn
)(const struct bpf_prog
*prog
, const void *ctx
);
1919 static __always_inline u32
1920 bpf_prog_run_array(const struct bpf_prog_array
*array
,
1921 const void *ctx
, bpf_prog_run_fn run_prog
)
1923 const struct bpf_prog_array_item
*item
;
1924 const struct bpf_prog
*prog
;
1925 struct bpf_run_ctx
*old_run_ctx
;
1926 struct bpf_trace_run_ctx run_ctx
;
1929 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
1931 if (unlikely(!array
))
1934 run_ctx
.is_uprobe
= false;
1937 old_run_ctx
= bpf_set_run_ctx(&run_ctx
.run_ctx
);
1938 item
= &array
->items
[0];
1939 while ((prog
= READ_ONCE(item
->prog
))) {
1940 run_ctx
.bpf_cookie
= item
->bpf_cookie
;
1941 ret
&= run_prog(prog
, ctx
);
1944 bpf_reset_run_ctx(old_run_ctx
);
1949 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
1951 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
1952 * overall. As a result, we must use the bpf_prog_array_free_sleepable
1953 * in order to use the tasks_trace rcu grace period.
1955 * When a non-sleepable program is inside the array, we take the rcu read
1956 * section and disable preemption for that program alone, so it can access
1957 * rcu-protected dynamically sized maps.
1959 static __always_inline u32
1960 bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu
*array_rcu
,
1961 const void *ctx
, bpf_prog_run_fn run_prog
)
1963 const struct bpf_prog_array_item
*item
;
1964 const struct bpf_prog
*prog
;
1965 const struct bpf_prog_array
*array
;
1966 struct bpf_run_ctx
*old_run_ctx
;
1967 struct bpf_trace_run_ctx run_ctx
;
1972 rcu_read_lock_trace();
1975 run_ctx
.is_uprobe
= true;
1977 array
= rcu_dereference_check(array_rcu
, rcu_read_lock_trace_held());
1978 if (unlikely(!array
))
1980 old_run_ctx
= bpf_set_run_ctx(&run_ctx
.run_ctx
);
1981 item
= &array
->items
[0];
1982 while ((prog
= READ_ONCE(item
->prog
))) {
1983 if (!prog
->aux
->sleepable
)
1986 run_ctx
.bpf_cookie
= item
->bpf_cookie
;
1987 ret
&= run_prog(prog
, ctx
);
1990 if (!prog
->aux
->sleepable
)
1993 bpf_reset_run_ctx(old_run_ctx
);
1996 rcu_read_unlock_trace();
2000 #ifdef CONFIG_BPF_SYSCALL
2001 DECLARE_PER_CPU(int, bpf_prog_active
);
2002 extern struct mutex bpf_stats_enabled_mutex
;
2005 * Block execution of BPF programs attached to instrumentation (perf,
2006 * kprobes, tracepoints) to prevent deadlocks on map operations as any of
2007 * these events can happen inside a region which holds a map bucket lock
2008 * and can deadlock on it.
2010 static inline void bpf_disable_instrumentation(void)
2013 this_cpu_inc(bpf_prog_active
);
2016 static inline void bpf_enable_instrumentation(void)
2018 this_cpu_dec(bpf_prog_active
);
2022 extern const struct file_operations bpf_map_fops
;
2023 extern const struct file_operations bpf_prog_fops
;
2024 extern const struct file_operations bpf_iter_fops
;
2026 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2027 extern const struct bpf_prog_ops _name ## _prog_ops; \
2028 extern const struct bpf_verifier_ops _name ## _verifier_ops;
2029 #define BPF_MAP_TYPE(_id, _ops) \
2030 extern const struct bpf_map_ops _ops;
2031 #define BPF_LINK_TYPE(_id, _name)
2032 #include <linux/bpf_types.h>
2033 #undef BPF_PROG_TYPE
2035 #undef BPF_LINK_TYPE
2037 extern const struct bpf_prog_ops bpf_offload_prog_ops
;
2038 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops
;
2039 extern const struct bpf_verifier_ops xdp_analyzer_ops
;
2041 struct bpf_prog
*bpf_prog_get(u32 ufd
);
2042 struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
, enum bpf_prog_type type
,
2044 void bpf_prog_add(struct bpf_prog
*prog
, int i
);
2045 void bpf_prog_sub(struct bpf_prog
*prog
, int i
);
2046 void bpf_prog_inc(struct bpf_prog
*prog
);
2047 struct bpf_prog
* __must_check
bpf_prog_inc_not_zero(struct bpf_prog
*prog
);
2048 void bpf_prog_put(struct bpf_prog
*prog
);
2050 void bpf_prog_free_id(struct bpf_prog
*prog
);
2051 void bpf_map_free_id(struct bpf_map
*map
);
2053 struct btf_field
*btf_record_find(const struct btf_record
*rec
,
2054 u32 offset
, u32 field_mask
);
2055 void btf_record_free(struct btf_record
*rec
);
2056 void bpf_map_free_record(struct bpf_map
*map
);
2057 struct btf_record
*btf_record_dup(const struct btf_record
*rec
);
2058 bool btf_record_equal(const struct btf_record
*rec_a
, const struct btf_record
*rec_b
);
2059 void bpf_obj_free_timer(const struct btf_record
*rec
, void *obj
);
2060 void bpf_obj_free_fields(const struct btf_record
*rec
, void *obj
);
2061 void __bpf_obj_drop_impl(void *p
, const struct btf_record
*rec
, bool percpu
);
2063 struct bpf_map
*bpf_map_get(u32 ufd
);
2064 struct bpf_map
*bpf_map_get_with_uref(u32 ufd
);
2065 struct bpf_map
*__bpf_map_get(struct fd f
);
2066 void bpf_map_inc(struct bpf_map
*map
);
2067 void bpf_map_inc_with_uref(struct bpf_map
*map
);
2068 struct bpf_map
*__bpf_map_inc_not_zero(struct bpf_map
*map
, bool uref
);
2069 struct bpf_map
* __must_check
bpf_map_inc_not_zero(struct bpf_map
*map
);
2070 void bpf_map_put_with_uref(struct bpf_map
*map
);
2071 void bpf_map_put(struct bpf_map
*map
);
2072 void *bpf_map_area_alloc(u64 size
, int numa_node
);
2073 void *bpf_map_area_mmapable_alloc(u64 size
, int numa_node
);
2074 void bpf_map_area_free(void *base
);
2075 bool bpf_map_write_active(const struct bpf_map
*map
);
2076 void bpf_map_init_from_attr(struct bpf_map
*map
, union bpf_attr
*attr
);
2077 int generic_map_lookup_batch(struct bpf_map
*map
,
2078 const union bpf_attr
*attr
,
2079 union bpf_attr __user
*uattr
);
2080 int generic_map_update_batch(struct bpf_map
*map
, struct file
*map_file
,
2081 const union bpf_attr
*attr
,
2082 union bpf_attr __user
*uattr
);
2083 int generic_map_delete_batch(struct bpf_map
*map
,
2084 const union bpf_attr
*attr
,
2085 union bpf_attr __user
*uattr
);
2086 struct bpf_map
*bpf_map_get_curr_or_next(u32
*id
);
2087 struct bpf_prog
*bpf_prog_get_curr_or_next(u32
*id
);
2089 #ifdef CONFIG_MEMCG_KMEM
2090 void *bpf_map_kmalloc_node(const struct bpf_map
*map
, size_t size
, gfp_t flags
,
2092 void *bpf_map_kzalloc(const struct bpf_map
*map
, size_t size
, gfp_t flags
);
2093 void *bpf_map_kvcalloc(struct bpf_map
*map
, size_t n
, size_t size
,
2095 void __percpu
*bpf_map_alloc_percpu(const struct bpf_map
*map
, size_t size
,
2096 size_t align
, gfp_t flags
);
2098 static inline void *
2099 bpf_map_kmalloc_node(const struct bpf_map
*map
, size_t size
, gfp_t flags
,
2102 return kmalloc_node(size
, flags
, node
);
2105 static inline void *
2106 bpf_map_kzalloc(const struct bpf_map
*map
, size_t size
, gfp_t flags
)
2108 return kzalloc(size
, flags
);
2111 static inline void *
2112 bpf_map_kvcalloc(struct bpf_map
*map
, size_t n
, size_t size
, gfp_t flags
)
2114 return kvcalloc(n
, size
, flags
);
2117 static inline void __percpu
*
2118 bpf_map_alloc_percpu(const struct bpf_map
*map
, size_t size
, size_t align
,
2121 return __alloc_percpu_gfp(size
, align
, flags
);
2126 bpf_map_init_elem_count(struct bpf_map
*map
)
2128 size_t size
= sizeof(*map
->elem_count
), align
= size
;
2129 gfp_t flags
= GFP_USER
| __GFP_NOWARN
;
2131 map
->elem_count
= bpf_map_alloc_percpu(map
, size
, align
, flags
);
2132 if (!map
->elem_count
)
2139 bpf_map_free_elem_count(struct bpf_map
*map
)
2141 free_percpu(map
->elem_count
);
2144 static inline void bpf_map_inc_elem_count(struct bpf_map
*map
)
2146 this_cpu_inc(*map
->elem_count
);
2149 static inline void bpf_map_dec_elem_count(struct bpf_map
*map
)
2151 this_cpu_dec(*map
->elem_count
);
2154 extern int sysctl_unprivileged_bpf_disabled
;
2156 static inline bool bpf_allow_ptr_leaks(void)
2158 return perfmon_capable();
2161 static inline bool bpf_allow_uninit_stack(void)
2163 return perfmon_capable();
2166 static inline bool bpf_bypass_spec_v1(void)
2168 return cpu_mitigations_off() || perfmon_capable();
2171 static inline bool bpf_bypass_spec_v4(void)
2173 return cpu_mitigations_off() || perfmon_capable();
2176 int bpf_map_new_fd(struct bpf_map
*map
, int flags
);
2177 int bpf_prog_new_fd(struct bpf_prog
*prog
);
2179 void bpf_link_init(struct bpf_link
*link
, enum bpf_link_type type
,
2180 const struct bpf_link_ops
*ops
, struct bpf_prog
*prog
);
2181 int bpf_link_prime(struct bpf_link
*link
, struct bpf_link_primer
*primer
);
2182 int bpf_link_settle(struct bpf_link_primer
*primer
);
2183 void bpf_link_cleanup(struct bpf_link_primer
*primer
);
2184 void bpf_link_inc(struct bpf_link
*link
);
2185 void bpf_link_put(struct bpf_link
*link
);
2186 int bpf_link_new_fd(struct bpf_link
*link
);
2187 struct bpf_link
*bpf_link_get_from_fd(u32 ufd
);
2188 struct bpf_link
*bpf_link_get_curr_or_next(u32
*id
);
2190 int bpf_obj_pin_user(u32 ufd
, int path_fd
, const char __user
*pathname
);
2191 int bpf_obj_get_user(int path_fd
, const char __user
*pathname
, int flags
);
2193 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2194 #define DEFINE_BPF_ITER_FUNC(target, args...) \
2195 extern int bpf_iter_ ## target(args); \
2196 int __init bpf_iter_ ## target(args) { return 0; }
2199 * The task type of iterators.
2201 * For BPF task iterators, they can be parameterized with various
2202 * parameters to visit only some of tasks.
2204 * BPF_TASK_ITER_ALL (default)
2205 * Iterate over resources of every task.
2208 * Iterate over resources of a task/tid.
2210 * BPF_TASK_ITER_TGID
2211 * Iterate over resources of every task of a process / task group.
2213 enum bpf_iter_task_type
{
2214 BPF_TASK_ITER_ALL
= 0,
2219 struct bpf_iter_aux_info
{
2220 /* for map_elem iter */
2221 struct bpf_map
*map
;
2223 /* for cgroup iter */
2225 struct cgroup
*start
; /* starting cgroup */
2226 enum bpf_cgroup_iter_order order
;
2229 enum bpf_iter_task_type type
;
2234 typedef int (*bpf_iter_attach_target_t
)(struct bpf_prog
*prog
,
2235 union bpf_iter_link_info
*linfo
,
2236 struct bpf_iter_aux_info
*aux
);
2237 typedef void (*bpf_iter_detach_target_t
)(struct bpf_iter_aux_info
*aux
);
2238 typedef void (*bpf_iter_show_fdinfo_t
) (const struct bpf_iter_aux_info
*aux
,
2239 struct seq_file
*seq
);
2240 typedef int (*bpf_iter_fill_link_info_t
)(const struct bpf_iter_aux_info
*aux
,
2241 struct bpf_link_info
*info
);
2242 typedef const struct bpf_func_proto
*
2243 (*bpf_iter_get_func_proto_t
)(enum bpf_func_id func_id
,
2244 const struct bpf_prog
*prog
);
2246 enum bpf_iter_feature
{
2247 BPF_ITER_RESCHED
= BIT(0),
2250 #define BPF_ITER_CTX_ARG_MAX 2
2251 struct bpf_iter_reg
{
2253 bpf_iter_attach_target_t attach_target
;
2254 bpf_iter_detach_target_t detach_target
;
2255 bpf_iter_show_fdinfo_t show_fdinfo
;
2256 bpf_iter_fill_link_info_t fill_link_info
;
2257 bpf_iter_get_func_proto_t get_func_proto
;
2258 u32 ctx_arg_info_size
;
2260 struct bpf_ctx_arg_aux ctx_arg_info
[BPF_ITER_CTX_ARG_MAX
];
2261 const struct bpf_iter_seq_info
*seq_info
;
2264 struct bpf_iter_meta
{
2265 __bpf_md_ptr(struct seq_file
*, seq
);
2270 struct bpf_iter__bpf_map_elem
{
2271 __bpf_md_ptr(struct bpf_iter_meta
*, meta
);
2272 __bpf_md_ptr(struct bpf_map
*, map
);
2273 __bpf_md_ptr(void *, key
);
2274 __bpf_md_ptr(void *, value
);
2277 int bpf_iter_reg_target(const struct bpf_iter_reg
*reg_info
);
2278 void bpf_iter_unreg_target(const struct bpf_iter_reg
*reg_info
);
2279 bool bpf_iter_prog_supported(struct bpf_prog
*prog
);
2280 const struct bpf_func_proto
*
2281 bpf_iter_get_func_proto(enum bpf_func_id func_id
, const struct bpf_prog
*prog
);
2282 int bpf_iter_link_attach(const union bpf_attr
*attr
, bpfptr_t uattr
, struct bpf_prog
*prog
);
2283 int bpf_iter_new_fd(struct bpf_link
*link
);
2284 bool bpf_link_is_iter(struct bpf_link
*link
);
2285 struct bpf_prog
*bpf_iter_get_info(struct bpf_iter_meta
*meta
, bool in_stop
);
2286 int bpf_iter_run_prog(struct bpf_prog
*prog
, void *ctx
);
2287 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info
*aux
,
2288 struct seq_file
*seq
);
2289 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info
*aux
,
2290 struct bpf_link_info
*info
);
2292 int map_set_for_each_callback_args(struct bpf_verifier_env
*env
,
2293 struct bpf_func_state
*caller
,
2294 struct bpf_func_state
*callee
);
2296 int bpf_percpu_hash_copy(struct bpf_map
*map
, void *key
, void *value
);
2297 int bpf_percpu_array_copy(struct bpf_map
*map
, void *key
, void *value
);
2298 int bpf_percpu_hash_update(struct bpf_map
*map
, void *key
, void *value
,
2300 int bpf_percpu_array_update(struct bpf_map
*map
, void *key
, void *value
,
2303 int bpf_stackmap_copy(struct bpf_map
*map
, void *key
, void *value
);
2305 int bpf_fd_array_map_update_elem(struct bpf_map
*map
, struct file
*map_file
,
2306 void *key
, void *value
, u64 map_flags
);
2307 int bpf_fd_array_map_lookup_elem(struct bpf_map
*map
, void *key
, u32
*value
);
2308 int bpf_fd_htab_map_update_elem(struct bpf_map
*map
, struct file
*map_file
,
2309 void *key
, void *value
, u64 map_flags
);
2310 int bpf_fd_htab_map_lookup_elem(struct bpf_map
*map
, void *key
, u32
*value
);
2312 int bpf_get_file_flag(int flags
);
2313 int bpf_check_uarg_tail_zero(bpfptr_t uaddr
, size_t expected_size
,
2314 size_t actual_size
);
2316 /* verify correctness of eBPF program */
2317 int bpf_check(struct bpf_prog
**fp
, union bpf_attr
*attr
, bpfptr_t uattr
, u32 uattr_size
);
2319 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
2320 void bpf_patch_call_args(struct bpf_insn
*insn
, u32 stack_depth
);
2323 struct btf
*bpf_get_btf_vmlinux(void);
2328 struct bpf_dtab_netdev
;
2329 struct bpf_cpu_map_entry
;
2331 void __dev_flush(void);
2332 int dev_xdp_enqueue(struct net_device
*dev
, struct xdp_frame
*xdpf
,
2333 struct net_device
*dev_rx
);
2334 int dev_map_enqueue(struct bpf_dtab_netdev
*dst
, struct xdp_frame
*xdpf
,
2335 struct net_device
*dev_rx
);
2336 int dev_map_enqueue_multi(struct xdp_frame
*xdpf
, struct net_device
*dev_rx
,
2337 struct bpf_map
*map
, bool exclude_ingress
);
2338 int dev_map_generic_redirect(struct bpf_dtab_netdev
*dst
, struct sk_buff
*skb
,
2339 struct bpf_prog
*xdp_prog
);
2340 int dev_map_redirect_multi(struct net_device
*dev
, struct sk_buff
*skb
,
2341 struct bpf_prog
*xdp_prog
, struct bpf_map
*map
,
2342 bool exclude_ingress
);
2344 void __cpu_map_flush(void);
2345 int cpu_map_enqueue(struct bpf_cpu_map_entry
*rcpu
, struct xdp_frame
*xdpf
,
2346 struct net_device
*dev_rx
);
2347 int cpu_map_generic_redirect(struct bpf_cpu_map_entry
*rcpu
,
2348 struct sk_buff
*skb
);
2350 /* Return map's numa specified by userspace */
2351 static inline int bpf_map_attr_numa_node(const union bpf_attr
*attr
)
2353 return (attr
->map_flags
& BPF_F_NUMA_NODE
) ?
2354 attr
->numa_node
: NUMA_NO_NODE
;
2357 struct bpf_prog
*bpf_prog_get_type_path(const char *name
, enum bpf_prog_type type
);
2358 int array_map_alloc_check(union bpf_attr
*attr
);
2360 int bpf_prog_test_run_xdp(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
2361 union bpf_attr __user
*uattr
);
2362 int bpf_prog_test_run_skb(struct bpf_prog
*prog
, const union bpf_attr
*kattr
,
2363 union bpf_attr __user
*uattr
);
2364 int bpf_prog_test_run_tracing(struct bpf_prog
*prog
,
2365 const union bpf_attr
*kattr
,
2366 union bpf_attr __user
*uattr
);
2367 int bpf_prog_test_run_flow_dissector(struct bpf_prog
*prog
,
2368 const union bpf_attr
*kattr
,
2369 union bpf_attr __user
*uattr
);
2370 int bpf_prog_test_run_raw_tp(struct bpf_prog
*prog
,
2371 const union bpf_attr
*kattr
,
2372 union bpf_attr __user
*uattr
);
2373 int bpf_prog_test_run_sk_lookup(struct bpf_prog
*prog
,
2374 const union bpf_attr
*kattr
,
2375 union bpf_attr __user
*uattr
);
2376 int bpf_prog_test_run_nf(struct bpf_prog
*prog
,
2377 const union bpf_attr
*kattr
,
2378 union bpf_attr __user
*uattr
);
2379 bool btf_ctx_access(int off
, int size
, enum bpf_access_type type
,
2380 const struct bpf_prog
*prog
,
2381 struct bpf_insn_access_aux
*info
);
2383 static inline bool bpf_tracing_ctx_access(int off
, int size
,
2384 enum bpf_access_type type
)
2386 if (off
< 0 || off
>= sizeof(__u64
) * MAX_BPF_FUNC_ARGS
)
2388 if (type
!= BPF_READ
)
2390 if (off
% size
!= 0)
2395 static inline bool bpf_tracing_btf_ctx_access(int off
, int size
,
2396 enum bpf_access_type type
,
2397 const struct bpf_prog
*prog
,
2398 struct bpf_insn_access_aux
*info
)
2400 if (!bpf_tracing_ctx_access(off
, size
, type
))
2402 return btf_ctx_access(off
, size
, type
, prog
, info
);
2405 int btf_struct_access(struct bpf_verifier_log
*log
,
2406 const struct bpf_reg_state
*reg
,
2407 int off
, int size
, enum bpf_access_type atype
,
2408 u32
*next_btf_id
, enum bpf_type_flag
*flag
, const char **field_name
);
2409 bool btf_struct_ids_match(struct bpf_verifier_log
*log
,
2410 const struct btf
*btf
, u32 id
, int off
,
2411 const struct btf
*need_btf
, u32 need_type_id
,
2414 int btf_distill_func_proto(struct bpf_verifier_log
*log
,
2416 const struct btf_type
*func_proto
,
2417 const char *func_name
,
2418 struct btf_func_model
*m
);
2420 struct bpf_reg_state
;
2421 int btf_check_subprog_arg_match(struct bpf_verifier_env
*env
, int subprog
,
2422 struct bpf_reg_state
*regs
);
2423 int btf_check_subprog_call(struct bpf_verifier_env
*env
, int subprog
,
2424 struct bpf_reg_state
*regs
);
2425 int btf_prepare_func_args(struct bpf_verifier_env
*env
, int subprog
,
2426 struct bpf_reg_state
*reg
, bool is_ex_cb
);
2427 int btf_check_type_match(struct bpf_verifier_log
*log
, const struct bpf_prog
*prog
,
2428 struct btf
*btf
, const struct btf_type
*t
);
2429 const char *btf_find_decl_tag_value(const struct btf
*btf
, const struct btf_type
*pt
,
2430 int comp_idx
, const char *tag_key
);
2432 struct bpf_prog
*bpf_prog_by_id(u32 id
);
2433 struct bpf_link
*bpf_link_by_id(u32 id
);
2435 const struct bpf_func_proto
*bpf_base_func_proto(enum bpf_func_id func_id
);
2436 void bpf_task_storage_free(struct task_struct
*task
);
2437 void bpf_cgrp_storage_free(struct cgroup
*cgroup
);
2438 bool bpf_prog_has_kfunc_call(const struct bpf_prog
*prog
);
2439 const struct btf_func_model
*
2440 bpf_jit_find_kfunc_model(const struct bpf_prog
*prog
,
2441 const struct bpf_insn
*insn
);
2442 int bpf_get_kfunc_addr(const struct bpf_prog
*prog
, u32 func_id
,
2443 u16 btf_fd_idx
, u8
**func_addr
);
2445 struct bpf_core_ctx
{
2446 struct bpf_verifier_log
*log
;
2447 const struct btf
*btf
;
2450 bool btf_nested_type_is_trusted(struct bpf_verifier_log
*log
,
2451 const struct bpf_reg_state
*reg
,
2452 const char *field_name
, u32 btf_id
, const char *suffix
);
2454 bool btf_type_ids_nocast_alias(struct bpf_verifier_log
*log
,
2455 const struct btf
*reg_btf
, u32 reg_id
,
2456 const struct btf
*arg_btf
, u32 arg_id
);
2458 int bpf_core_apply(struct bpf_core_ctx
*ctx
, const struct bpf_core_relo
*relo
,
2459 int relo_idx
, void *insn
);
2461 static inline bool unprivileged_ebpf_enabled(void)
2463 return !sysctl_unprivileged_bpf_disabled
;
2466 /* Not all bpf prog type has the bpf_ctx.
2467 * For the bpf prog type that has initialized the bpf_ctx,
2468 * this function can be used to decide if a kernel function
2469 * is called by a bpf program.
2471 static inline bool has_current_bpf_ctx(void)
2473 return !!current
->bpf_ctx
;
2476 void notrace
bpf_prog_inc_misses_counter(struct bpf_prog
*prog
);
2478 void bpf_dynptr_init(struct bpf_dynptr_kern
*ptr
, void *data
,
2479 enum bpf_dynptr_type type
, u32 offset
, u32 size
);
2480 void bpf_dynptr_set_null(struct bpf_dynptr_kern
*ptr
);
2481 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern
*ptr
);
2483 bool dev_check_flush(void);
2484 bool cpu_map_check_flush(void);
2485 #else /* !CONFIG_BPF_SYSCALL */
2486 static inline struct bpf_prog
*bpf_prog_get(u32 ufd
)
2488 return ERR_PTR(-EOPNOTSUPP
);
2491 static inline struct bpf_prog
*bpf_prog_get_type_dev(u32 ufd
,
2492 enum bpf_prog_type type
,
2495 return ERR_PTR(-EOPNOTSUPP
);
2498 static inline void bpf_prog_add(struct bpf_prog
*prog
, int i
)
2502 static inline void bpf_prog_sub(struct bpf_prog
*prog
, int i
)
2506 static inline void bpf_prog_put(struct bpf_prog
*prog
)
2510 static inline void bpf_prog_inc(struct bpf_prog
*prog
)
2514 static inline struct bpf_prog
*__must_check
2515 bpf_prog_inc_not_zero(struct bpf_prog
*prog
)
2517 return ERR_PTR(-EOPNOTSUPP
);
2520 static inline void bpf_link_init(struct bpf_link
*link
, enum bpf_link_type type
,
2521 const struct bpf_link_ops
*ops
,
2522 struct bpf_prog
*prog
)
2526 static inline int bpf_link_prime(struct bpf_link
*link
,
2527 struct bpf_link_primer
*primer
)
2532 static inline int bpf_link_settle(struct bpf_link_primer
*primer
)
2537 static inline void bpf_link_cleanup(struct bpf_link_primer
*primer
)
2541 static inline void bpf_link_inc(struct bpf_link
*link
)
2545 static inline void bpf_link_put(struct bpf_link
*link
)
2549 static inline int bpf_obj_get_user(const char __user
*pathname
, int flags
)
2554 static inline void __dev_flush(void)
2559 struct bpf_dtab_netdev
;
2560 struct bpf_cpu_map_entry
;
2563 int dev_xdp_enqueue(struct net_device
*dev
, struct xdp_frame
*xdpf
,
2564 struct net_device
*dev_rx
)
2570 int dev_map_enqueue(struct bpf_dtab_netdev
*dst
, struct xdp_frame
*xdpf
,
2571 struct net_device
*dev_rx
)
2577 int dev_map_enqueue_multi(struct xdp_frame
*xdpf
, struct net_device
*dev_rx
,
2578 struct bpf_map
*map
, bool exclude_ingress
)
2585 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev
*dst
,
2586 struct sk_buff
*skb
,
2587 struct bpf_prog
*xdp_prog
)
2593 int dev_map_redirect_multi(struct net_device
*dev
, struct sk_buff
*skb
,
2594 struct bpf_prog
*xdp_prog
, struct bpf_map
*map
,
2595 bool exclude_ingress
)
2600 static inline void __cpu_map_flush(void)
2604 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry
*rcpu
,
2605 struct xdp_frame
*xdpf
,
2606 struct net_device
*dev_rx
)
2611 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry
*rcpu
,
2612 struct sk_buff
*skb
)
2617 static inline struct bpf_prog
*bpf_prog_get_type_path(const char *name
,
2618 enum bpf_prog_type type
)
2620 return ERR_PTR(-EOPNOTSUPP
);
2623 static inline int bpf_prog_test_run_xdp(struct bpf_prog
*prog
,
2624 const union bpf_attr
*kattr
,
2625 union bpf_attr __user
*uattr
)
2630 static inline int bpf_prog_test_run_skb(struct bpf_prog
*prog
,
2631 const union bpf_attr
*kattr
,
2632 union bpf_attr __user
*uattr
)
2637 static inline int bpf_prog_test_run_tracing(struct bpf_prog
*prog
,
2638 const union bpf_attr
*kattr
,
2639 union bpf_attr __user
*uattr
)
2644 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog
*prog
,
2645 const union bpf_attr
*kattr
,
2646 union bpf_attr __user
*uattr
)
2651 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog
*prog
,
2652 const union bpf_attr
*kattr
,
2653 union bpf_attr __user
*uattr
)
2658 static inline void bpf_map_put(struct bpf_map
*map
)
2662 static inline struct bpf_prog
*bpf_prog_by_id(u32 id
)
2664 return ERR_PTR(-ENOTSUPP
);
2667 static inline int btf_struct_access(struct bpf_verifier_log
*log
,
2668 const struct bpf_reg_state
*reg
,
2669 int off
, int size
, enum bpf_access_type atype
,
2670 u32
*next_btf_id
, enum bpf_type_flag
*flag
,
2671 const char **field_name
)
2676 static inline const struct bpf_func_proto
*
2677 bpf_base_func_proto(enum bpf_func_id func_id
)
2682 static inline void bpf_task_storage_free(struct task_struct
*task
)
2686 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog
*prog
)
2691 static inline const struct btf_func_model
*
2692 bpf_jit_find_kfunc_model(const struct bpf_prog
*prog
,
2693 const struct bpf_insn
*insn
)
2699 bpf_get_kfunc_addr(const struct bpf_prog
*prog
, u32 func_id
,
2700 u16 btf_fd_idx
, u8
**func_addr
)
2705 static inline bool unprivileged_ebpf_enabled(void)
2710 static inline bool has_current_bpf_ctx(void)
2715 static inline void bpf_prog_inc_misses_counter(struct bpf_prog
*prog
)
2719 static inline void bpf_cgrp_storage_free(struct cgroup
*cgroup
)
2723 static inline void bpf_dynptr_init(struct bpf_dynptr_kern
*ptr
, void *data
,
2724 enum bpf_dynptr_type type
, u32 offset
, u32 size
)
2728 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern
*ptr
)
2732 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern
*ptr
)
2735 #endif /* CONFIG_BPF_SYSCALL */
2737 static __always_inline
int
2738 bpf_probe_read_kernel_common(void *dst
, u32 size
, const void *unsafe_ptr
)
2742 if (IS_ENABLED(CONFIG_BPF_EVENTS
))
2743 ret
= copy_from_kernel_nofault(dst
, unsafe_ptr
, size
);
2744 if (unlikely(ret
< 0))
2745 memset(dst
, 0, size
);
2749 void __bpf_free_used_btfs(struct bpf_prog_aux
*aux
,
2750 struct btf_mod_pair
*used_btfs
, u32 len
);
2752 static inline struct bpf_prog
*bpf_prog_get_type(u32 ufd
,
2753 enum bpf_prog_type type
)
2755 return bpf_prog_get_type_dev(ufd
, type
, false);
2758 void __bpf_free_used_maps(struct bpf_prog_aux
*aux
,
2759 struct bpf_map
**used_maps
, u32 len
);
2761 bool bpf_prog_get_ok(struct bpf_prog
*, enum bpf_prog_type
*, bool);
2763 int bpf_prog_offload_compile(struct bpf_prog
*prog
);
2764 void bpf_prog_dev_bound_destroy(struct bpf_prog
*prog
);
2765 int bpf_prog_offload_info_fill(struct bpf_prog_info
*info
,
2766 struct bpf_prog
*prog
);
2768 int bpf_map_offload_info_fill(struct bpf_map_info
*info
, struct bpf_map
*map
);
2770 int bpf_map_offload_lookup_elem(struct bpf_map
*map
, void *key
, void *value
);
2771 int bpf_map_offload_update_elem(struct bpf_map
*map
,
2772 void *key
, void *value
, u64 flags
);
2773 int bpf_map_offload_delete_elem(struct bpf_map
*map
, void *key
);
2774 int bpf_map_offload_get_next_key(struct bpf_map
*map
,
2775 void *key
, void *next_key
);
2777 bool bpf_offload_prog_map_match(struct bpf_prog
*prog
, struct bpf_map
*map
);
2779 struct bpf_offload_dev
*
2780 bpf_offload_dev_create(const struct bpf_prog_offload_ops
*ops
, void *priv
);
2781 void bpf_offload_dev_destroy(struct bpf_offload_dev
*offdev
);
2782 void *bpf_offload_dev_priv(struct bpf_offload_dev
*offdev
);
2783 int bpf_offload_dev_netdev_register(struct bpf_offload_dev
*offdev
,
2784 struct net_device
*netdev
);
2785 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev
*offdev
,
2786 struct net_device
*netdev
);
2787 bool bpf_offload_dev_match(struct bpf_prog
*prog
, struct net_device
*netdev
);
2789 void unpriv_ebpf_notify(int new_state
);
2791 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
2792 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log
*log
,
2793 struct bpf_prog_aux
*prog_aux
);
2794 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog
*prog
, u32 func_id
);
2795 int bpf_prog_dev_bound_init(struct bpf_prog
*prog
, union bpf_attr
*attr
);
2796 int bpf_prog_dev_bound_inherit(struct bpf_prog
*new_prog
, struct bpf_prog
*old_prog
);
2797 void bpf_dev_bound_netdev_unregister(struct net_device
*dev
);
2799 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux
*aux
)
2801 return aux
->dev_bound
;
2804 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux
*aux
)
2806 return aux
->offload_requested
;
2809 bool bpf_prog_dev_bound_match(const struct bpf_prog
*lhs
, const struct bpf_prog
*rhs
);
2811 static inline bool bpf_map_is_offloaded(struct bpf_map
*map
)
2813 return unlikely(map
->ops
== &bpf_map_offload_ops
);
2816 struct bpf_map
*bpf_map_offload_map_alloc(union bpf_attr
*attr
);
2817 void bpf_map_offload_map_free(struct bpf_map
*map
);
2818 u64
bpf_map_offload_map_mem_usage(const struct bpf_map
*map
);
2819 int bpf_prog_test_run_syscall(struct bpf_prog
*prog
,
2820 const union bpf_attr
*kattr
,
2821 union bpf_attr __user
*uattr
);
2823 int sock_map_get_from_fd(const union bpf_attr
*attr
, struct bpf_prog
*prog
);
2824 int sock_map_prog_detach(const union bpf_attr
*attr
, enum bpf_prog_type ptype
);
2825 int sock_map_update_elem_sys(struct bpf_map
*map
, void *key
, void *value
, u64 flags
);
2826 int sock_map_bpf_prog_query(const union bpf_attr
*attr
,
2827 union bpf_attr __user
*uattr
);
2829 void sock_map_unhash(struct sock
*sk
);
2830 void sock_map_destroy(struct sock
*sk
);
2831 void sock_map_close(struct sock
*sk
, long timeout
);
2833 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log
*log
,
2834 struct bpf_prog_aux
*prog_aux
)
2839 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog
*prog
,
2845 static inline int bpf_prog_dev_bound_init(struct bpf_prog
*prog
,
2846 union bpf_attr
*attr
)
2851 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog
*new_prog
,
2852 struct bpf_prog
*old_prog
)
2857 static inline void bpf_dev_bound_netdev_unregister(struct net_device
*dev
)
2861 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux
*aux
)
2866 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux
*aux
)
2871 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog
*lhs
, const struct bpf_prog
*rhs
)
2876 static inline bool bpf_map_is_offloaded(struct bpf_map
*map
)
2881 static inline struct bpf_map
*bpf_map_offload_map_alloc(union bpf_attr
*attr
)
2883 return ERR_PTR(-EOPNOTSUPP
);
2886 static inline void bpf_map_offload_map_free(struct bpf_map
*map
)
2890 static inline u64
bpf_map_offload_map_mem_usage(const struct bpf_map
*map
)
2895 static inline int bpf_prog_test_run_syscall(struct bpf_prog
*prog
,
2896 const union bpf_attr
*kattr
,
2897 union bpf_attr __user
*uattr
)
2902 #ifdef CONFIG_BPF_SYSCALL
2903 static inline int sock_map_get_from_fd(const union bpf_attr
*attr
,
2904 struct bpf_prog
*prog
)
2909 static inline int sock_map_prog_detach(const union bpf_attr
*attr
,
2910 enum bpf_prog_type ptype
)
2915 static inline int sock_map_update_elem_sys(struct bpf_map
*map
, void *key
, void *value
,
2921 static inline int sock_map_bpf_prog_query(const union bpf_attr
*attr
,
2922 union bpf_attr __user
*uattr
)
2926 #endif /* CONFIG_BPF_SYSCALL */
2927 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
2929 static __always_inline
void
2930 bpf_prog_inc_misses_counters(const struct bpf_prog_array
*array
)
2932 const struct bpf_prog_array_item
*item
;
2933 struct bpf_prog
*prog
;
2935 if (unlikely(!array
))
2938 item
= &array
->items
[0];
2939 while ((prog
= READ_ONCE(item
->prog
))) {
2940 bpf_prog_inc_misses_counter(prog
);
2945 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
2946 void bpf_sk_reuseport_detach(struct sock
*sk
);
2947 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map
*map
, void *key
,
2949 int bpf_fd_reuseport_array_update_elem(struct bpf_map
*map
, void *key
,
2950 void *value
, u64 map_flags
);
2952 static inline void bpf_sk_reuseport_detach(struct sock
*sk
)
2956 #ifdef CONFIG_BPF_SYSCALL
2957 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map
*map
,
2958 void *key
, void *value
)
2963 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map
*map
,
2964 void *key
, void *value
,
2969 #endif /* CONFIG_BPF_SYSCALL */
2970 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
2972 /* verifier prototypes for helper functions called from eBPF programs */
2973 extern const struct bpf_func_proto bpf_map_lookup_elem_proto
;
2974 extern const struct bpf_func_proto bpf_map_update_elem_proto
;
2975 extern const struct bpf_func_proto bpf_map_delete_elem_proto
;
2976 extern const struct bpf_func_proto bpf_map_push_elem_proto
;
2977 extern const struct bpf_func_proto bpf_map_pop_elem_proto
;
2978 extern const struct bpf_func_proto bpf_map_peek_elem_proto
;
2979 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto
;
2981 extern const struct bpf_func_proto bpf_get_prandom_u32_proto
;
2982 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto
;
2983 extern const struct bpf_func_proto bpf_get_numa_node_id_proto
;
2984 extern const struct bpf_func_proto bpf_tail_call_proto
;
2985 extern const struct bpf_func_proto bpf_ktime_get_ns_proto
;
2986 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto
;
2987 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto
;
2988 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto
;
2989 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto
;
2990 extern const struct bpf_func_proto bpf_get_current_comm_proto
;
2991 extern const struct bpf_func_proto bpf_get_stackid_proto
;
2992 extern const struct bpf_func_proto bpf_get_stack_proto
;
2993 extern const struct bpf_func_proto bpf_get_task_stack_proto
;
2994 extern const struct bpf_func_proto bpf_get_stackid_proto_pe
;
2995 extern const struct bpf_func_proto bpf_get_stack_proto_pe
;
2996 extern const struct bpf_func_proto bpf_sock_map_update_proto
;
2997 extern const struct bpf_func_proto bpf_sock_hash_update_proto
;
2998 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto
;
2999 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto
;
3000 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto
;
3001 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto
;
3002 extern const struct bpf_func_proto bpf_msg_redirect_map_proto
;
3003 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto
;
3004 extern const struct bpf_func_proto bpf_sk_redirect_map_proto
;
3005 extern const struct bpf_func_proto bpf_spin_lock_proto
;
3006 extern const struct bpf_func_proto bpf_spin_unlock_proto
;
3007 extern const struct bpf_func_proto bpf_get_local_storage_proto
;
3008 extern const struct bpf_func_proto bpf_strtol_proto
;
3009 extern const struct bpf_func_proto bpf_strtoul_proto
;
3010 extern const struct bpf_func_proto bpf_tcp_sock_proto
;
3011 extern const struct bpf_func_proto bpf_jiffies64_proto
;
3012 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto
;
3013 extern const struct bpf_func_proto bpf_event_output_data_proto
;
3014 extern const struct bpf_func_proto bpf_ringbuf_output_proto
;
3015 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto
;
3016 extern const struct bpf_func_proto bpf_ringbuf_submit_proto
;
3017 extern const struct bpf_func_proto bpf_ringbuf_discard_proto
;
3018 extern const struct bpf_func_proto bpf_ringbuf_query_proto
;
3019 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto
;
3020 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto
;
3021 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto
;
3022 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto
;
3023 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto
;
3024 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto
;
3025 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto
;
3026 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto
;
3027 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto
;
3028 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto
;
3029 extern const struct bpf_func_proto bpf_copy_from_user_proto
;
3030 extern const struct bpf_func_proto bpf_snprintf_btf_proto
;
3031 extern const struct bpf_func_proto bpf_snprintf_proto
;
3032 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto
;
3033 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto
;
3034 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto
;
3035 extern const struct bpf_func_proto bpf_sock_from_file_proto
;
3036 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto
;
3037 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto
;
3038 extern const struct bpf_func_proto bpf_task_storage_get_proto
;
3039 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto
;
3040 extern const struct bpf_func_proto bpf_task_storage_delete_proto
;
3041 extern const struct bpf_func_proto bpf_for_each_map_elem_proto
;
3042 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto
;
3043 extern const struct bpf_func_proto bpf_sk_setsockopt_proto
;
3044 extern const struct bpf_func_proto bpf_sk_getsockopt_proto
;
3045 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto
;
3046 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto
;
3047 extern const struct bpf_func_proto bpf_find_vma_proto
;
3048 extern const struct bpf_func_proto bpf_loop_proto
;
3049 extern const struct bpf_func_proto bpf_copy_from_user_task_proto
;
3050 extern const struct bpf_func_proto bpf_set_retval_proto
;
3051 extern const struct bpf_func_proto bpf_get_retval_proto
;
3052 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto
;
3053 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto
;
3054 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto
;
3056 const struct bpf_func_proto
*tracing_prog_func_proto(
3057 enum bpf_func_id func_id
, const struct bpf_prog
*prog
);
3059 /* Shared helpers among cBPF and eBPF. */
3060 void bpf_user_rnd_init_once(void);
3061 u64
bpf_user_rnd_u32(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
);
3062 u64
bpf_get_raw_cpu_id(u64 r1
, u64 r2
, u64 r3
, u64 r4
, u64 r5
);
3064 #if defined(CONFIG_NET)
3065 bool bpf_sock_common_is_valid_access(int off
, int size
,
3066 enum bpf_access_type type
,
3067 struct bpf_insn_access_aux
*info
);
3068 bool bpf_sock_is_valid_access(int off
, int size
, enum bpf_access_type type
,
3069 struct bpf_insn_access_aux
*info
);
3070 u32
bpf_sock_convert_ctx_access(enum bpf_access_type type
,
3071 const struct bpf_insn
*si
,
3072 struct bpf_insn
*insn_buf
,
3073 struct bpf_prog
*prog
,
3075 int bpf_dynptr_from_skb_rdonly(struct sk_buff
*skb
, u64 flags
,
3076 struct bpf_dynptr_kern
*ptr
);
3078 static inline bool bpf_sock_common_is_valid_access(int off
, int size
,
3079 enum bpf_access_type type
,
3080 struct bpf_insn_access_aux
*info
)
3084 static inline bool bpf_sock_is_valid_access(int off
, int size
,
3085 enum bpf_access_type type
,
3086 struct bpf_insn_access_aux
*info
)
3090 static inline u32
bpf_sock_convert_ctx_access(enum bpf_access_type type
,
3091 const struct bpf_insn
*si
,
3092 struct bpf_insn
*insn_buf
,
3093 struct bpf_prog
*prog
,
3098 static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff
*skb
, u64 flags
,
3099 struct bpf_dynptr_kern
*ptr
)
3106 struct sk_reuseport_kern
{
3107 struct sk_buff
*skb
;
3109 struct sock
*selected_sk
;
3110 struct sock
*migrating_sk
;
3116 bool bpf_tcp_sock_is_valid_access(int off
, int size
, enum bpf_access_type type
,
3117 struct bpf_insn_access_aux
*info
);
3119 u32
bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type
,
3120 const struct bpf_insn
*si
,
3121 struct bpf_insn
*insn_buf
,
3122 struct bpf_prog
*prog
,
3125 bool bpf_xdp_sock_is_valid_access(int off
, int size
, enum bpf_access_type type
,
3126 struct bpf_insn_access_aux
*info
);
3128 u32
bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type
,
3129 const struct bpf_insn
*si
,
3130 struct bpf_insn
*insn_buf
,
3131 struct bpf_prog
*prog
,
3134 static inline bool bpf_tcp_sock_is_valid_access(int off
, int size
,
3135 enum bpf_access_type type
,
3136 struct bpf_insn_access_aux
*info
)
3141 static inline u32
bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type
,
3142 const struct bpf_insn
*si
,
3143 struct bpf_insn
*insn_buf
,
3144 struct bpf_prog
*prog
,
3149 static inline bool bpf_xdp_sock_is_valid_access(int off
, int size
,
3150 enum bpf_access_type type
,
3151 struct bpf_insn_access_aux
*info
)
3156 static inline u32
bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type
,
3157 const struct bpf_insn
*si
,
3158 struct bpf_insn
*insn_buf
,
3159 struct bpf_prog
*prog
,
3164 #endif /* CONFIG_INET */
3166 enum bpf_text_poke_type
{
3171 int bpf_arch_text_poke(void *ip
, enum bpf_text_poke_type t
,
3172 void *addr1
, void *addr2
);
3174 void *bpf_arch_text_copy(void *dst
, void *src
, size_t len
);
3175 int bpf_arch_text_invalidate(void *dst
, size_t len
);
3178 bool btf_id_set_contains(const struct btf_id_set
*set
, u32 id
);
3180 #define MAX_BPRINTF_VARARGS 12
3181 #define MAX_BPRINTF_BUF 1024
3183 struct bpf_bprintf_data
{
3190 int bpf_bprintf_prepare(char *fmt
, u32 fmt_size
, const u64
*raw_args
,
3191 u32 num_args
, struct bpf_bprintf_data
*data
);
3192 void bpf_bprintf_cleanup(struct bpf_bprintf_data
*data
);
3194 #ifdef CONFIG_BPF_LSM
3195 void bpf_cgroup_atype_get(u32 attach_btf_id
, int cgroup_atype
);
3196 void bpf_cgroup_atype_put(int cgroup_atype
);
3198 static inline void bpf_cgroup_atype_get(u32 attach_btf_id
, int cgroup_atype
) {}
3199 static inline void bpf_cgroup_atype_put(int cgroup_atype
) {}
3200 #endif /* CONFIG_BPF_LSM */
3209 #endif /* CONFIG_KEYS */
3211 static inline bool type_is_alloc(u32 type
)
3213 return type
& MEM_ALLOC
;
3216 static inline gfp_t
bpf_memcg_flags(gfp_t flags
)
3218 if (memcg_bpf_enabled())
3219 return flags
| __GFP_ACCOUNT
;
3223 static inline bool bpf_is_subprog(const struct bpf_prog
*prog
)
3225 return prog
->aux
->func_idx
!= 0;
3228 #endif /* _LINUX_BPF_H */