The BPF selftest headers include functionality that is
specific to arenas and is required by libarena. Keep libarena
self-contained by moving all functionality into its include/
directory. Also add libarena/include to the standard include
paths for the selftests to make the moved headers easy to
access by existing selftests.
Some functionality is required by libarena but not strictly
arena-related. We still move it to the libarena/include path,
which is an upgrade from directly accessing them from the
selftests/bpf directory using relative paths.
A new bpf_may_goto.h file is split off of bpf_experimental.h.
bpf_arena_spin_lock.h and bpf_arena_common.h are moved to
libarena/include. bpf_atomic.h is also moved to libarena
because it is necessary for arena spinlocks.
For bpf_arena_spin_lock.h, mark the spinlock state array as __weak
to define the spinlock state array in the header while also
being compatible with multi-compilation unit programs. While
we're at it, we remove unnecessary definitions from existing
test programs.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
Link: https://lore.kernel.org/r/20260426190338.4615-4-emil@etsalapatis.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
-Wno-unused-but-set-variable \
$(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \
-I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
- -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT)
+ -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT) \
+ -I$(CURDIR)/libarena/include
LDFLAGS += $(SAN_LDFLAGS)
LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \
-I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \
+ -I$(CURDIR)/libarena/include \
-I$(abspath $(OUTPUT)/../usr/include) \
-std=gnu11 \
-fno-strict-aliasing \
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#ifndef __round_mask
#define __round_mask(x, y) ((__typeof__(x))((y)-1))
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#pragma once
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
struct arena_list_node;
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#pragma once
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
__noinline int bpf_arena_strlen(const char __arena *s __arg_arena)
{
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
+#include <bpf_may_goto.h>
#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
})
#endif
-/*
- * Note that cond_break can only be portably used in the body of a breakable
- * construct, whereas can_loop can be used anywhere.
- */
-#ifdef __BPF_FEATURE_MAY_GOTO
-#define can_loop \
- ({ __label__ l_break, l_continue; \
- bool ret = true; \
- asm volatile goto("may_goto %l[l_break]" \
- :::: l_break); \
- goto l_continue; \
- l_break: ret = false; \
- l_continue:; \
- ret; \
- })
-
-#define __cond_break(expr) \
- ({ __label__ l_break, l_continue; \
- asm volatile goto("may_goto %l[l_break]" \
- :::: l_break); \
- goto l_continue; \
- l_break: expr; \
- l_continue:; \
- })
-#else
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define can_loop \
- ({ __label__ l_break, l_continue; \
- bool ret = true; \
- asm volatile goto("1:.byte 0xe5; \
- .byte 0; \
- .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
- .short 0" \
- :::: l_break); \
- goto l_continue; \
- l_break: ret = false; \
- l_continue:; \
- ret; \
- })
-
-#define __cond_break(expr) \
- ({ __label__ l_break, l_continue; \
- asm volatile goto("1:.byte 0xe5; \
- .byte 0; \
- .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
- .short 0" \
- :::: l_break); \
- goto l_continue; \
- l_break: expr; \
- l_continue:; \
- })
-#else
-#define can_loop \
- ({ __label__ l_break, l_continue; \
- bool ret = true; \
- asm volatile goto("1:.byte 0xe5; \
- .byte 0; \
- .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
- .short 0" \
- :::: l_break); \
- goto l_continue; \
- l_break: ret = false; \
- l_continue:; \
- ret; \
- })
-
-#define __cond_break(expr) \
- ({ __label__ l_break, l_continue; \
- asm volatile goto("1:.byte 0xe5; \
- .byte 0; \
- .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
- .short 0" \
- :::: l_break); \
- goto l_continue; \
- l_break: expr; \
- l_continue:; \
- })
-#endif
-#endif
-
-#define cond_break __cond_break(break)
-#define cond_break_label(label) __cond_break(goto label)
-
#ifndef bpf_nop_mov
#define bpf_nop_mov(var) \
asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
-#include "bpf_atomic.h"
+#include <bpf_atomic.h>
#define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label)
#define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1)
#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
-struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
+/*
+ * The qnodes are marked __weak so we can define them in the header
+ * while still ensuring all compilation units use the same struct
+ * instance.
+ */
+struct arena_qnode __weak __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
static inline u32 encode_tail(int cpu, int idx)
{
return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
}
-__noinline
+__noinline __weak
int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val)
{
struct arena_mcs_spinlock __arena *prev, *next, *node0, *node;
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
-#include "bpf_experimental.h"
+#include <bpf_may_goto.h>
extern bool CONFIG_X86_64 __kconfig __weak;
--- /dev/null
+#pragma once
+
+/*
+ * Note that cond_break can only be portably used in the body of a breakable
+ * construct, whereas can_loop can be used anywhere.
+ */
+#ifdef __BPF_FEATURE_MAY_GOTO
+#define can_loop \
+ ({ __label__ l_break, l_continue; \
+ bool ret = true; \
+ asm volatile goto("may_goto %l[l_break]" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: ret = false; \
+ l_continue:; \
+ ret; \
+ })
+
+#define __cond_break(expr) \
+ ({ __label__ l_break, l_continue; \
+ asm volatile goto("may_goto %l[l_break]" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: expr; \
+ l_continue:; \
+ })
+#else
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define can_loop \
+ ({ __label__ l_break, l_continue; \
+ bool ret = true; \
+ asm volatile goto("1:.byte 0xe5; \
+ .byte 0; \
+ .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
+ .short 0" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: ret = false; \
+ l_continue:; \
+ ret; \
+ })
+
+#define __cond_break(expr) \
+ ({ __label__ l_break, l_continue; \
+ asm volatile goto("1:.byte 0xe5; \
+ .byte 0; \
+ .long ((%l[l_break] - 1b - 8) / 8) & 0xffff; \
+ .short 0" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: expr; \
+ l_continue:; \
+ })
+#else
+#define can_loop \
+ ({ __label__ l_break, l_continue; \
+ bool ret = true; \
+ asm volatile goto("1:.byte 0xe5; \
+ .byte 0; \
+ .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
+ .short 0" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: ret = false; \
+ l_continue:; \
+ ret; \
+ })
+
+#define __cond_break(expr) \
+ ({ __label__ l_break, l_continue; \
+ asm volatile goto("1:.byte 0xe5; \
+ .byte 0; \
+ .long (((%l[l_break] - 1b - 8) / 8) & 0xffff) << 16; \
+ .short 0" \
+ :::: l_break); \
+ goto l_continue; \
+ l_break: expr; \
+ l_continue:; \
+ })
+#endif
+#endif
+
+#define cond_break __cond_break(break)
+#define cond_break_label(label) __cond_break(goto label)
#include <vmlinux.h>
-#include "../../bpf_arena_common.h"
-#include "../../progs/bpf_arena_spin_lock.h"
+#include <bpf_arena_common.h>
+#include <bpf_arena_spin_lock.h>
#include <asm-generic/errno.h>
#include <sys/sysinfo.h>
struct __qspinlock { int val; };
-typedef struct __qspinlock arena_spinlock_t;
-
-struct arena_qnode {
- unsigned long next;
- int count;
- int locked;
-};
#include "arena_spin_lock.skel.h"
#include <bpf/bpf_tracing.h>
#include <stdbool.h>
#include <stdatomic.h>
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
-#include "bpf_arena_spin_lock.h"
+#include <bpf_arena_spin_lock.h>
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "../../../include/linux/filter.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#include "bpf_misc.h"
struct {
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include "bpf_misc.h"
-#include "bpf_atomic.h"
+#include <bpf_atomic.h>
#include "progs/lpm_trie.h"
#define BPF_OBJ_NAME_LEN 16U
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
struct arr_elem {
struct bpf_res_spin_lock lock;
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8)))
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_experimental.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#include "bpf_misc.h"
#define ARENA_PAGES (1UL<< (32 - __builtin_ffs(__PAGE_SIZE) + 1))
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#define ARENA_PAGES (32)
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#define ARENA_SIZE (1ull << 32)
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
-#include "bpf_arena_common.h"
+#include <bpf_arena_common.h>
#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \