htab_bucket_t *buckets;
int n_buckets;
};
-typedef struct htab __arena htab_t;
-static inline htab_bucket_t *__select_bucket(htab_t *htab, __u32 hash)
+static inline htab_bucket_t *__select_bucket(struct htab __arena *htab, __u32 hash)
{
htab_bucket_t *b = htab->buckets;
return &b[hash & (htab->n_buckets - 1)];
}
-static inline arena_list_head_t *select_bucket(htab_t *htab, __u32 hash)
+static inline arena_list_head_t *select_bucket(struct htab __arena *htab, __u32 hash)
{
return &__select_bucket(htab, hash)->head;
}
return key;
}
-__weak int htab_lookup_elem(htab_t *htab __arg_arena, int key)
+__weak int htab_lookup_elem(struct htab __arena *htab, int key)
{
hashtab_elem_t *l_old;
arena_list_head_t *head;
return 0;
}
-__weak int htab_update_elem(htab_t *htab __arg_arena, int key, int value)
+__weak int htab_update_elem(struct htab __arena *htab, int key, int value)
{
hashtab_elem_t *l_new = NULL, *l_old;
arena_list_head_t *head;
return 0;
}
-void htab_init(htab_t *htab)
+void htab_init(struct htab __arena *htab)
{
void __arena *buckets = bpf_arena_alloc_pages(&arena, NULL, 2, NUMA_NO_NODE, 0);
#pragma once
#include <bpf_arena_common.h>
-__noinline int bpf_arena_strlen(const char __arena *s __arg_arena)
+__noinline int bpf_arena_strlen(const char __arena *s)
{
const char __arena *sc;
*
* An opening bracket without a matching close is matched literally.
*/
-__noinline bool glob_match(char const __arena *pat __arg_arena, char const __arena *str __arg_arena)
+__noinline bool glob_match(char const __arena *pat, char const __arena *str)
{
/*
* Backtrack to previous * on mismatch and retry starting one
#define cast_kern(ptr) /* nop for bpf prog. emitted by LLVM */
#define cast_user(ptr) /* nop for bpf prog. emitted by LLVM */
#else
-#define __arena
+#define __arena __attribute__((btf_type_tag("arena")))
#define __arena_global SEC(".addr_space.1")
#define cast_kern(ptr) bpf_addr_space_cast(ptr, 0, 1)
#define cast_user(ptr) bpf_addr_space_cast(ptr, 1, 0)
#else /* when compiled as user space code */
#define __arena
-#define __arg_arena
#define cast_kern(ptr) /* nop for user space */
#define cast_user(ptr) /* nop for user space */
__weak char arena[1];
#define EOPNOTSUPP 95
#define ETIMEDOUT 110
-#ifndef __arena
-#define __arena __attribute__((address_space(1)))
-#endif
-
extern unsigned long CONFIG_NR_CPUS __kconfig;
/*
}
__noinline __weak
-int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val)
+int arena_spin_lock_slowpath(arena_spinlock_t __arena *lock, u32 val)
{
struct arena_mcs_spinlock __arena *prev, *next, *node0, *node;
int ret = -ETIMEDOUT;
#ifdef BPF_ARENA_ASAN
-typedef s8 __arena s8a;
-
static inline
-s8a *mem_to_shadow(void __arena __arg_arena *addr)
+s8 __arena *mem_to_shadow(void __arena *addr)
{
- return (s8a *)(((u32)(u64)addr >> ASAN_SHADOW_SHIFT) +
+ return (s8 __arena *)(((u32)(u64)addr >> ASAN_SHADOW_SHIFT) +
__asan_shadow_memory_dynamic_address);
}
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#pragma once
-struct buddy_chunk;
-typedef struct buddy_chunk __arena buddy_chunk_t;
-
-struct buddy_header;
-typedef struct buddy_header __arena buddy_header_t;
-
enum buddy_consts {
/*
* Minimum allocation is 1 << BUDDY_MIN_ALLOC_SHIFT.
u8 allocated[BUDDY_CHUNK_ITEMS / 8];
/* Freelists for O(1) allocation. */
u64 freelists[BUDDY_CHUNK_NUM_ORDERS];
- buddy_chunk_t *next;
+ struct buddy_chunk __arena *next;
};
struct buddy {
- buddy_chunk_t *first_chunk; /* Pointer to the chunk linked list. */
+ struct buddy_chunk __arena *first_chunk; /* Pointer to the chunk linked list. */
arena_spinlock_t lock; /* Allocator lock */
u64 vaddr; /* Allocation into reserved vaddr */
};
-typedef struct buddy __arena buddy_t;
-
#ifdef __BPF__
-int buddy_init(buddy_t *buddy);
-int buddy_destroy(buddy_t *buddy);
-int buddy_free_internal(buddy_t *buddy, u64 free);
-#define buddy_free(buddy, ptr) buddy_free_internal((buddy), (u64)(ptr))
-u64 buddy_alloc_internal(buddy_t *buddy, size_t size);
+int buddy_init(struct buddy __arena *buddy);
+int buddy_destroy(struct buddy __arena *buddy);
+int buddy_free(struct buddy __arena *buddy, void __arena *free);
+u64 buddy_alloc_internal(struct buddy __arena *buddy, size_t size);
#define buddy_alloc(alloc, size) ((void __arena *)buddy_alloc_internal((alloc), (size)))
-
#endif /* __BPF__ */
/* Required for parsing the ASAN call stacks. */
#include "test_progs_compat.h"
-extern buddy_t buddy;
+extern struct buddy __arena buddy;
#ifdef BPF_ARENA_ASAN
* Factored out because asan_validate_addr is complex enough to cause
* verification failures if verified with the rest of asan_test_buddy_uaf_single.
*/
-__weak int asan_test_buddy_byte(u8 __arena __arg_arena *mem, int i, bool freed)
+__weak int asan_test_buddy_byte(u8 __arena *mem, int i, bool freed)
{
int ret;
{
arena_stdout("%s:%d ASAN %p -> (val: %x gran: %x set: [%s])",
__func__, __LINE__, addr,
- *(s8a *)(addr), ASAN_GRANULE(addr),
+ *(s8 __arena *)(addr), ASAN_GRANULE(addr),
asan_shadow_set(addr) ? "yes" : "no");
}
#include <libarena/asan.h>
#include <libarena/buddy.h>
-extern buddy_t buddy;
+extern struct buddy __arena buddy;
struct segarr_entry {
u8 __arena *block;
* to exit due to a missing implementation. Provide a simple implementation
* just for memset to use it for poisoning/unpoisoning the map.
*/
-__weak int asan_memset(s8a __arg_arena *dst, s8 val, size_t size)
+__weak int asan_memset(s8 __arena *dst, s8 val, size_t size)
{
size_t i;
}
/* Validate a 1-byte access, always within a single byte. */
-static __always_inline bool memory_is_poisoned_1(s8a *addr)
+static __always_inline bool memory_is_poisoned_1(s8 __arena *addr)
{
- s8 shadow_value = *(s8a *)mem_to_shadow(addr);
+ s8 shadow_value = *(s8 __arena *)mem_to_shadow(addr);
/* Byte is 0, access is valid. */
if (likely(!shadow_value))
}
/* Validate a 2- 4-, 8-byte access, shadow spans up to 2 bytes. */
-static __always_inline bool memory_is_poisoned_2_4_8(s8a *addr, u64 size)
+static __always_inline bool memory_is_poisoned_2_4_8(s8 __arena *addr, u64 size)
{
u64 end = (u64)addr + size - 1;
* overflow above ASAN_GRANULE).
*/
if (likely(ASAN_GRANULE(end) >= size - 1))
- return memory_is_poisoned_1((s8a *)end);
+ return memory_is_poisoned_1((s8 __arena *)end);
/*
* Otherwise first byte must be fully unpoisoned, and second byte
* must be unpoisoned up to the end of the accessed region.
*/
- return *(s8a *)mem_to_shadow(addr) || memory_is_poisoned_1((s8a *)end);
+ return *(s8 __arena *)mem_to_shadow(addr) || memory_is_poisoned_1((s8 __arena *)end);
}
-__weak bool asan_shadow_set(void __arena __arg_arena *addr)
+__weak bool asan_shadow_set(void __arena *addr)
{
return memory_is_poisoned_1(addr);
}
static __always_inline u64 first_nonzero_byte(u64 addr, size_t size)
{
while (size && can_loop) {
- if (unlikely(*(s8a *)addr))
+ if (unlikely(*(s8 __arena *)addr))
return addr;
addr += 1;
size -= 1;
return SHADOW_ALL_ZEROES;
}
-static __always_inline bool memory_is_poisoned_n(s8a *addr, u64 size)
+static __always_inline bool memory_is_poisoned_n(s8 __arena *addr, u64 size)
{
u64 ret;
u64 start;
if (likely(ret == SHADOW_ALL_ZEROES))
return false;
- return unlikely(ret != end || ASAN_GRANULE(addr + size - 1) >= *(s8a *)end);
+ return unlikely(ret != end || ASAN_GRANULE(addr + size - 1) >= *(s8 __arena *)end);
}
-__weak int asan_report(s8a __arg_arena *addr, size_t sz, u32 flags)
+__weak int asan_report(s8 __arena *addr, size_t sz, u32 flags)
{
u32 reported = __sync_val_compare_and_swap(&asan_reported, false, true);
return 0;
}
-static __always_inline bool check_asan_args(s8a *addr, size_t size,
+static __always_inline bool check_asan_args(s8 __arena *addr, size_t size,
bool *result)
{
bool valid = true;
static __always_inline bool check_region_inline(intptr_t ptr, size_t size,
u32 flags)
{
- s8a *addr = (s8a *)(u64)ptr;
+ s8 __arena *addr = (s8 __arena *)(u64)ptr;
bool is_poisoned, is_valid;
if (check_asan_args(addr, size, &is_valid)) {
} \
__hidden void __asan_report_store##size(intptr_t addr) \
{ \
- asan_report((s8a *)addr, size, ASAN_WRITE); \
+ asan_report((s8 __arena *)addr, size, ASAN_WRITE); \
} \
__hidden void __asan_report_store##size##_noabort(intptr_t addr) \
{ \
- asan_report((s8a *)addr, size, ASAN_WRITE); \
+ asan_report((s8 __arena *)addr, size, ASAN_WRITE); \
} \
__hidden void __asan_report_load##size(intptr_t addr) \
{ \
- asan_report((s8a *)addr, size, ASAN_READ); \
+ asan_report((s8 __arena *)addr, size, ASAN_READ); \
} \
__hidden void __asan_report_load##size##_noabort(intptr_t addr) \
{ \
- asan_report((s8a *)addr, size, ASAN_READ); \
+ asan_report((s8 __arena *)addr, size, ASAN_READ); \
}
DEFINE_ASAN_LOAD_STORE(1);
*/
__hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size)
{
- s8a *shadow;
+ s8 __arena *shadow;
size_t len;
/*
__hidden __noasan int asan_unpoison(void __arena *addr, size_t size)
{
size_t partial = size & ASAN_GRANULE_MASK;
- s8a *shadow;
+ s8 __arena *shadow;
size_t len;
/*
BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE
};
-static inline int buddy_lock(buddy_t *buddy)
+static inline int buddy_lock(struct buddy __arena *buddy)
{
return arena_spin_lock(&buddy->lock);
}
-static inline void buddy_unlock(buddy_t *buddy)
+static inline void buddy_unlock(struct buddy __arena *buddy)
{
arena_spin_unlock(&buddy->lock);
}
* page alloc kfuncs do not support aligning to a boundary (in this
* case 1 MiB, see buddy.h on how this is derived).
*/
-static int buddy_reserve_arena_vaddr(buddy_t *buddy)
+static int buddy_reserve_arena_vaddr(struct buddy __arena *buddy)
{
buddy->vaddr = 0;
/*
* Free up any unused address space. Used only during teardown.
*/
-static void buddy_unreserve_arena_vaddr(buddy_t *buddy)
+static void buddy_unreserve_arena_vaddr(struct buddy __arena *buddy)
{
bpf_arena_free_pages(
&arena, (void __arena *)(BUDDY_VADDR_OFFSET + buddy->vaddr),
* However, bump allocation must still be atomic because this function
* is called without the buddy lock from multiple threads concurrently.
*/
-__weak int buddy_alloc_arena_vaddr(buddy_t __arg_arena *buddy, u64 *vaddrp)
+__weak int buddy_alloc_arena_vaddr(struct buddy __arena *buddy, u64 *vaddrp)
{
u64 vaddr, old, new;
}
__weak
-int idx_set_allocated(buddy_chunk_t __arg_arena *chunk, u64 idx, bool allocated)
+int idx_set_allocated(struct buddy_chunk __arena *chunk, u64 idx, bool allocated)
{
bool already_allocated;
return 0;
}
-static int idx_is_allocated(buddy_chunk_t *chunk, u64 idx, bool *allocated)
+static int idx_is_allocated(struct buddy_chunk __arena *chunk, u64 idx, bool *allocated)
{
if (unlikely(idx >= BUDDY_CHUNK_ITEMS)) {
arena_stderr("getting state of invalid idx (%llu, max %d)\n", idx,
}
__weak
-int idx_set_order(buddy_chunk_t __arg_arena *chunk, u64 idx, u8 order)
+int idx_set_order(struct buddy_chunk __arena *chunk, u64 idx, u8 order)
{
u8 prev_order;
return 0;
}
-static u8 idx_get_order(buddy_chunk_t *chunk, u64 idx)
+static u8 idx_get_order(struct buddy_chunk __arena *chunk, u64 idx)
{
u8 result;
return (idx & 0x1) ? (result & 0xf) : (result >> 4);
}
-static void __arena *idx_to_addr(buddy_chunk_t *chunk, size_t idx)
+static void __arena *idx_to_addr(struct buddy_chunk __arena *chunk, size_t idx)
{
u64 address;
return (void __arena *)address;
}
-static buddy_header_t *idx_to_header(buddy_chunk_t *chunk, size_t idx)
+static struct buddy_header __arena *idx_to_header(struct buddy_chunk __arena *chunk, size_t idx)
{
bool allocated;
u64 address;
* less probable.
*/
- return (buddy_header_t *)(address + BUDDY_HEADER_OFF);
+ return (struct buddy_header __arena *)(address + BUDDY_HEADER_OFF);
}
-static void header_add_freelist(buddy_chunk_t *chunk, buddy_header_t *header,
+static void header_add_freelist(struct buddy_chunk __arena *chunk, struct buddy_header __arena *header,
u64 idx, u8 order)
{
- buddy_header_t *tmp_header;
+ struct buddy_header __arena *tmp_header;
idx_set_order(chunk, idx, order);
chunk->freelists[order] = idx;
}
-static void header_remove_freelist(buddy_chunk_t *chunk,
- buddy_header_t *header, u8 order)
+static void header_remove_freelist(struct buddy_chunk __arena *chunk,
+ struct buddy_header __arena *header, u8 order)
{
- buddy_header_t *tmp_header;
+ struct buddy_header __arena *tmp_header;
if (header->prev_index != BUDDY_CHUNK_ITEMS) {
tmp_header = idx_to_header(chunk, header->prev_index);
}
__weak
-int add_leftovers_to_freelist(buddy_chunk_t __arg_arena *chunk, u32 cur_idx,
+int add_leftovers_to_freelist(struct buddy_chunk __arena *chunk, u32 cur_idx,
u64 min_order, u64 max_order)
{
- buddy_header_t *header;
+ struct buddy_header __arena *header;
u64 ord;
u32 idx;
return 0;
}
-static buddy_chunk_t *buddy_chunk_get(buddy_t *buddy)
+static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy)
{
u64 order, ord, min_order, max_order;
- buddy_chunk_t *chunk;
+ struct buddy_chunk __arena *chunk;
size_t left;
int power2;
u64 vaddr;
return chunk;
}
-__weak int buddy_init(buddy_t __arg_arena *buddy)
+__weak int buddy_init(struct buddy __arena *buddy)
{
- buddy_chunk_t *chunk;
+ struct buddy_chunk __arena *chunk;
int ret;
if (!asan_ready())
* We do not take a lock because we are freeing arena pages, and nobody should
* be using the allocator at that point in the execution.
*/
-__weak int buddy_destroy(buddy_t __arg_arena *buddy)
+__weak int buddy_destroy(struct buddy __arena *buddy)
{
- buddy_chunk_t *chunk, *next;
+ struct buddy_chunk __arena *chunk, *next;
if (!buddy)
return -EINVAL;
return 0;
}
-__weak u64 buddy_chunk_alloc(buddy_chunk_t __arg_arena *chunk, int order_req)
+__weak u64 buddy_chunk_alloc(struct buddy_chunk __arena *chunk, int order_req)
{
- buddy_header_t *header, *tmp_header, *next_header;
+ struct buddy_header __arena *header, *tmp_header, *next_header;
u32 idx, tmpidx, retidx;
u64 address;
u64 order = 0;
}
/* Scan the existing chunks for available memory. */
-static u64 buddy_alloc_from_existing_chunks(buddy_t *buddy, int order)
+static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int order)
{
- buddy_chunk_t *chunk;
+ struct buddy_chunk __arena *chunk;
u64 address;
for (chunk = buddy->first_chunk; chunk != NULL && can_loop;
* Try an allocation from a newly allocated chunk. Also
* incorporate the chunk into the linked list.
*/
-static u64 buddy_alloc_from_new_chunk(buddy_t *buddy, buddy_chunk_t *chunk, int order)
+static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order)
{
u64 address;
return (u64)address;
}
__weak
-u64 buddy_alloc_internal(buddy_t __arg_arena *buddy, size_t size)
+u64 buddy_alloc_internal(struct buddy __arena *buddy, size_t size)
{
- buddy_chunk_t *chunk;
u64 address = (u64)NULL;
+ struct buddy_chunk __arena *chunk;
int order;
if (!buddy)
* data is smaller than the header, we must poison any
* unused bytes that were part of the header.
*/
- if (size < BUDDY_HEADER_OFF + sizeof(buddy_header_t))
- asan_poison((u8 __arena *)address + BUDDY_HEADER_OFF,
- BUDDY_POISONED, sizeof(buddy_header_t));
+ if (size < BUDDY_HEADER_OFF + sizeof(struct buddy_header __arena))
+ asan_poison((u8 __arena *)address + BUDDY_HEADER_OFF, BUDDY_POISONED,
+ sizeof(struct buddy_header __arena));
asan_unpoison((u8 __arena *)address, size);
return address;
}
-static __always_inline int buddy_free_unlocked(buddy_t *buddy, u64 addr)
+static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 addr)
{
- buddy_header_t *header, *buddy_header;
+ struct buddy_header __arena *header, *buddy_header;
u64 idx, buddy_idx, tmp_idx;
- buddy_chunk_t *chunk;
+ struct buddy_chunk __arena *chunk;
bool allocated;
u8 order;
int ret;
return 0;
}
-__weak int buddy_free_internal(buddy_t __arg_arena *buddy, u64 addr)
+__weak int buddy_free(struct buddy __arena *buddy, void __arena *addr)
{
int ret;
if (ret)
return ret;
- ret = buddy_free_unlocked(buddy, addr);
+ ret = buddy_free_unlocked(buddy, (u64)addr);
buddy_unlock(buddy);
const volatile u32 zero = 0;
-buddy_t buddy;
+struct buddy __arena buddy;
int arena_fls(__u64 word)
{
return buddy_alloc_internal(&buddy, size);
}
-__weak void arena_free(void __arg_arena __arena *ptr)
+__weak void arena_free(void __arena *ptr)
{
- buddy_free_internal(&buddy, (u64)ptr);
+ buddy_free(&buddy, ptr);
}
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
#include "bpf_misc.h"
+#include <bpf_arena_common.h>
#include <bpf_arena_spin_lock.h>
struct {