/*! \file isc/mem.h */
+#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
+#include <strings.h>
#include <isc/attributes.h>
#include <isc/mutex.h>
*/
/*%
+ * Flags that can be passed to the isc_mem_*x() variants of the macros.
+ *
* The definitions of the macros have been pulled directly from jemalloc.h
* and checked for consistency in mem.c.
*
- *\li ISC__MEM_ZERO - fill the memory with zeroes before returning
+ *\li ISC_MEM_ALIGN(alignment) - use when you need aligned allocation,
+ *
+ * NOTE: Set the matching flag, when freeing aligned memory allocation.
+ *
+ *\li ISC_MEM_ZERO - fill the memory with zeroes before returning
*/
-#define ISC__MEM_ZERO ((int)0x40)
+#if __SIZEOF_POINTER__ == 4
+#define ISC_MEM_ALIGN(a) ((int)(ffs((int)(a)) - 1))
+#else
+#define ISC_MEM_ALIGN(a) \
+ ((int)(((size_t)(a) < (size_t)INT_MAX) \
+ ? ffs((int)(a)) - 1 \
+ : ffs((int)(((size_t)(a)) >> 32)) + 31))
+#endif
+#define ISC_MEM_ZERO ((int)0x40)
#define isc_mem_get(c, s) isc__mem_get((c), (s), 0 _ISC_MEM_FILELINE)
#define isc_mem_cget(c, n, s) \
isc__mem_get((c), ISC_CHECKED_MUL((n), (s)), \
- ISC__MEM_ZERO _ISC_MEM_FILELINE)
+ ISC_MEM_ZERO _ISC_MEM_FILELINE)
+#define isc_mem_getx(c, s, f) isc__mem_get((c), (s), (f)_ISC_MEM_FILELINE)
#define isc_mem_reget(c, p, o, n) \
isc__mem_reget((c), (p), (o), (n), 0 _ISC_MEM_FILELINE)
#define isc_mem_creget(c, p, o, n, s) \
isc__mem_reget((c), (p), ISC_CHECKED_MUL((o), (s)), \
ISC_CHECKED_MUL((n), (s)), \
- ISC__MEM_ZERO _ISC_MEM_FILELINE)
+ ISC_MEM_ZERO _ISC_MEM_FILELINE)
+#define isc_mem_regetx(c, p, o, n, f) \
+ isc__mem_reget((c), (p), (o), (n), (f)_ISC_MEM_FILELINE)
#define isc_mem_allocate(c, s) isc__mem_allocate((c), (s), 0 _ISC_MEM_FILELINE)
+#define isc_mem_allocatex(c, s, f) \
+ isc__mem_allocate((c), (s), (f)_ISC_MEM_FILELINE)
#define isc_mem_callocate(c, n, s) \
isc__mem_allocate((c), ISC_CHECKED_MUL((n), (s)), \
- ISC__MEM_ZERO _ISC_MEM_FILELINE)
+ ISC_MEM_ZERO _ISC_MEM_FILELINE)
#define isc_mem_reallocate(c, p, s) \
isc__mem_reallocate((c), (p), (s), 0 _ISC_MEM_FILELINE)
+#define isc_mem_reallocatex(c, p, s, f) \
+ isc__mem_reallocate((c), (p), (s), (f)_ISC_MEM_FILELINE)
#define isc_mem_strdup(c, p) isc__mem_strdup((c), (p)_ISC_MEM_FILELINE)
#define isc_mempool_get(c) isc__mempool_get((c)_ISC_MEM_FILELINE)
#define isc_mem_cput(c, p, n, s) \
do { \
isc__mem_put((c), (p), ISC_CHECKED_MUL((n), (s)), \
- ISC__MEM_ZERO _ISC_MEM_FILELINE); \
+ ISC_MEM_ZERO _ISC_MEM_FILELINE); \
(p) = NULL; \
} while (0)
+#define isc_mem_putx(c, p, s, f) \
+ do { \
+ isc__mem_put((c), (p), (s), (f)_ISC_MEM_FILELINE); \
+ (p) = NULL; \
+ } while (0)
#define isc_mem_putanddetach(c, p, s) \
do { \
isc__mem_putanddetach((c), (p), (s), 0 _ISC_MEM_FILELINE); \
(p) = NULL; \
} while (0)
+#define isc_mem_putanddetachx(c, p, s, f) \
+ do { \
+ isc__mem_putanddetach((c), (p), (s), (f)_ISC_MEM_FILELINE); \
+ (p) = NULL; \
+ } while (0)
#define isc_mem_free(c, p) \
do { \
isc__mem_free((c), (p), 0 _ISC_MEM_FILELINE); \
(p) = NULL; \
} while (0)
+#define isc_mem_freex(c, p, f) \
+ do { \
+ isc__mem_free((c), (p), (f)_ISC_MEM_FILELINE); \
+ (p) = NULL; \
+ } while (0)
#define isc_mempool_put(c, p) \
do { \
isc__mempool_put((c), (p)_ISC_MEM_FILELINE); \
#if !defined(HAVE_JEMALLOC)
+#include <limits.h>
#include <stddef.h>
+#include <stdint.h>
#include <stdlib.h>
#include <string.h>
+#include <strings.h>
#include <isc/overflow.h>
#include <isc/util.h>
const char *malloc_conf = NULL;
+static size_t
+get_aligned_size(size_t alignment, size_t size) {
+ /*
+ * C23 dropped the requirement that size be an integral multiple of
+ * alignment, but some implementations still enforce it at runtime,
+ * so round the size up.
+ */
+ return ISC_CHECKED_ADD(size, alignment - 1) & ~(alignment - 1);
+}
+
+#ifndef HAVE_FREE_SIZED
+static void
+free_sized(void *ptr, size_t size ISC_ATTR_UNUSED) {
+ free(ptr);
+}
+#endif
+
+#ifndef HAVE_FREE_ALIGNED_SIZED
+static void
+free_aligned_sized(void *ptr, size_t alignment ISC_ATTR_UNUSED,
+ size_t size ISC_ATTR_UNUSED) {
+ free(ptr);
+}
+#endif
+
/*
- * The MALLOCX_ZERO and MALLOCX_ZERO_GET macros were taken literal from
- * jemalloc_macros.h and jemalloc_internal_types.h headers respectively.
+ * The MALLOCX_ZERO, MALLOCX_ALIGN and MALLOCX_*_GET macros were taken
+ * literal from jemalloc_macros.h and jemalloc_internal_types.h headers
+ * respectively.
*/
#define MALLOCX_ZERO ((int)0x40)
#define MALLOCX_ZERO_GET(flags) ((bool)(flags & MALLOCX_ZERO))
-#if defined(HAVE_MALLOC_SIZE) || defined(HAVE_MALLOC_USABLE_SIZE)
-
-#ifdef HAVE_MALLOC_SIZE
-
-#include <malloc/malloc.h>
+#define MALLOCX_LG_ALIGN_MASK ((int)0x3f)
+#if __SIZEOF_POINTER__ == 4
+#define MALLOCX_ALIGN(a) ((int)(ffs((int)(a)) - 1))
+#else
+#define MALLOCX_ALIGN(a) \
+ ((int)(((size_t)(a) < (size_t)INT_MAX) \
+ ? ffs((int)(a)) - 1 \
+ : ffs((int)(((size_t)(a)) >> 32)) + 31))
+#endif
+#define MALLOCX_ALIGN_GET(flags) \
+ (((size_t)1) << (flags & MALLOCX_LG_ALIGN_MASK))
-static inline size_t
-sallocx(void *ptr, int flags ISC_ATTR_UNUSED) {
- return malloc_size(ptr);
-}
+/*
+ * malloc() already guarantees alignment suitable for any fundamental
+ * type; only larger alignment requests need the aligned paths below.
+ */
+#define MALLOCX_NEEDS_ALIGN(flags) \
+ (MALLOCX_ALIGN_GET(flags) > _Alignof(max_align_t))
-#elif HAVE_MALLOC_USABLE_SIZE
+typedef union {
+ struct {
+ size_t size;
+ int flags;
+ };
+ max_align_t __alignment;
+} size_info;
-#ifdef __DragonFly__
/*
- * On DragonFly BSD 'man 3 malloc' advises us to include the following
- * header to have access to malloc_usable_size().
+ * The size_info header sits a multiple of the requested alignment before the
+ * returned pointer. The caller passes the matching MALLOCX_ALIGN() flag on
+ * deallocation too, so the base address is always recomputable from the flags
+ * and nothing needs to be stored.
*/
-#include <malloc_np.h>
-#else
-#include <malloc.h>
-#endif
static inline size_t
-sallocx(void *ptr, int flags ISC_ATTR_UNUSED) {
- return malloc_usable_size(ptr);
+get_header_size(int flags) {
+ return get_aligned_size(MALLOCX_ALIGN_GET(flags), sizeof(size_info));
}
-#endif /* HAVE_MALLOC_SIZE */
-
-static inline void *
-mallocx(size_t size, int flags) {
- void *ptr = malloc(size);
- INSIST(ptr != NULL);
-
- if ((flags & MALLOCX_ZERO) != 0) {
- memset(ptr, 0, size);
- }
-
- return ptr;
-}
-
-static inline void
-sdallocx(void *ptr, size_t size ISC_ATTR_UNUSED, int flags ISC_ATTR_UNUSED) {
- free(ptr);
+static inline size_info *
+get_size_info(void *ptr, int flags) {
+ return (size_info *)((uint8_t *)ptr - get_header_size(flags));
}
-static inline void *
-rallocx(void *ptr, size_t size, int flags) {
- REQUIRE(size != 0);
- REQUIRE((flags & MALLOCX_ZERO) == 0);
-
- ptr = realloc(ptr, size);
- INSIST(ptr != NULL);
-
- return ptr;
-}
-
-#else
-
-typedef union {
- size_t size;
- max_align_t __alignment;
-} size_info;
-
static inline void *
mallocx(size_t size, int flags) {
- void *ptr = NULL;
-
- size_t bytes = ISC_CHECKED_ADD(size, sizeof(size_info));
- size_info *si = malloc(bytes);
+ size_t alignment = MALLOCX_ALIGN_GET(flags);
+ size_t header_size = get_header_size(flags);
+ size_info *si;
+
+ if (MALLOCX_NEEDS_ALIGN(flags)) {
+ size_t bytes = ISC_CHECKED_ADD(
+ get_aligned_size(alignment, size), header_size);
+ si = aligned_alloc(alignment, bytes);
+ } else {
+ si = malloc(ISC_CHECKED_ADD(size, header_size));
+ }
if (si == NULL) {
return NULL;
}
-
si->size = size;
- ptr = &si[1];
+ si->flags = flags;
+ void *ptr = (uint8_t *)si + header_size;
if (MALLOCX_ZERO_GET(flags)) {
memset(ptr, 0, size);
}
}
static inline void
-sdallocx(void *ptr, size_t size ISC_ATTR_UNUSED, int flags ISC_ATTR_UNUSED) {
- size_info *si = &(((size_info *)ptr)[-1]);
-
- free(si);
+sdallocx(void *ptr, size_t size, int flags) {
+ size_info *si = get_size_info(ptr, flags);
+ size_t alignment = MALLOCX_ALIGN_GET(flags);
+ size_t header_size = get_header_size(flags);
+
+ INSIST((flags & MALLOCX_LG_ALIGN_MASK) ==
+ (si->flags & MALLOCX_LG_ALIGN_MASK));
+
+ if (MALLOCX_NEEDS_ALIGN(flags)) {
+ size_t bytes = ISC_CHECKED_ADD(
+ get_aligned_size(alignment, size), header_size);
+ free_aligned_sized(si, alignment, bytes);
+ } else {
+ free_sized(si, ISC_CHECKED_ADD(si->size, header_size));
+ }
}
static inline size_t
-sallocx(void *ptr, int flags ISC_ATTR_UNUSED) {
- size_info *si = &(((size_info *)ptr)[-1]);
+sallocx(void *ptr, int flags) {
+ size_info *si = get_size_info(ptr, flags);
return si[0].size;
}
static inline void *
rallocx(void *ptr, size_t size, int flags) {
- size_t bytes = ISC_CHECKED_ADD(size, sizeof(size_info));
- size_info *si = realloc(&(((size_info *)ptr)[-1]), bytes);
+ size_t header_size = get_header_size(flags);
+ size_info *si = get_size_info(ptr, flags);
+ size_t old_size = si->size;
+
+ INSIST((flags & MALLOCX_LG_ALIGN_MASK) ==
+ (si->flags & MALLOCX_LG_ALIGN_MASK));
+
+ if (MALLOCX_NEEDS_ALIGN(flags)) {
+ /*
+ * realloc() cannot preserve the requested alignment;
+ * move the data to a fresh aligned allocation.
+ */
+ void *new_ptr = mallocx(size, flags & ~MALLOCX_ZERO);
+ if (new_ptr == NULL) {
+ return NULL;
+ }
+ memmove(new_ptr, ptr, ISC_MIN(old_size, size));
+ if (MALLOCX_ZERO_GET(flags) && size > old_size) {
+ memset((uint8_t *)new_ptr + old_size, 0,
+ size - old_size);
+ }
+ sdallocx(ptr, old_size, flags);
+
+ si = get_size_info(new_ptr, flags);
+ si->flags = flags;
+
+ return new_ptr;
+ }
+
+ si = realloc(si, ISC_CHECKED_ADD(size, header_size));
if (si == NULL) {
return NULL;
}
+ si->size = size;
+ si->flags = flags;
- if (MALLOCX_ZERO_GET(flags) && size > si->size) {
- memset((uint8_t *)si + sizeof(*si) + si->size, 0,
- size - si->size);
+ ptr = (uint8_t *)si + header_size;
+ if (MALLOCX_ZERO_GET(flags) && size > old_size) {
+ memset((uint8_t *)ptr + old_size, 0, size - old_size);
}
- si->size = size;
- ptr = &si[1];
-
return ptr;
}
-#endif /* defined(HAVE_MALLOC_SIZE) || defined(HAVE_MALLOC_USABLE_SIZE) */
-
#endif /* !defined(HAVE_JEMALLOC) */
#include <jemalloc/jemalloc.h>
#define JEMALLOC_API_SUPPORTED 1
#else
+#if defined(__GLIBC__)
+#include <malloc.h>
+#endif
#include "jemalloc_shim.h"
#endif
void
isc__mem_initialize(void) {
-/*
- * Check if the values copied from jemalloc still match
- */
-#ifdef JEMALLOC_API_SUPPORTED
- RUNTIME_CHECK(ISC__MEM_ZERO == MALLOCX_ZERO);
+ /*
+ * Check if the values copied from jemalloc still match; the
+ * shim defines the MALLOCX_* macros too, so this holds on
+ * every allocator path.
+ */
+ RUNTIME_CHECK(ISC_MEM_ZERO == MALLOCX_ZERO);
+ RUNTIME_CHECK(ISC_MEM_ALIGN(sizeof(void *)) ==
+ MALLOCX_ALIGN(sizeof(void *)));
+ RUNTIME_CHECK(ISC_MEM_ALIGN(ISC_OS_CACHELINE_SIZE) ==
+ MALLOCX_ALIGN(ISC_OS_CACHELINE_SIZE));
+#ifdef JEMALLOC_API_SUPPORTED
/*
* ignore errors — volumetric-based purge in mem_put handles the rest
* regardless
REQUIRE(ctxp != NULL && *ctxp == NULL);
REQUIRE(name != NULL);
- ctx = mallocx(sizeof(*ctx), jemalloc_flags);
+ ctx = mallocx(sizeof(*ctx),
+ jemalloc_flags | ISC_MEM_ALIGN(isc_os_cacheline()));
CHECK_OOM(ctx, sizeof(*ctx));
*ctx = (isc_mem_t){
INSIST(!ctx->checkfree || dl->ptr == NULL);
ISC_LIST_UNLINK(ctx->debuglist[i], dl, link);
- sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags);
+ sdallocx(dl, dl->dlsize, ctx->jemalloc_flags);
}
}
}
#endif /* ISC_MEM_TRACKLINES */
- sdallocx(ctx, sizeof(*ctx), ctx->jemalloc_flags);
+ sdallocx(ctx, sizeof(*ctx),
+ ctx->jemalloc_flags | ISC_MEM_ALIGN(isc_os_cacheline()));
}
#if ISC_MEM_TRACE
ADJUST_ZERO_ALLOCATION_SIZE(new_size);
-#ifdef JEMALLOC_API_SUPPORTED
new_ptr = mem_realloc(ctx, old_ptr, new_size, flags);
-#else
- new_ptr = mem_realloc(ctx, old_ptr, new_size,
- flags & ~ISC__MEM_ZERO);
-
- if ((flags & ISC__MEM_ZERO) != 0) {
- if (new_size > old_size) {
- memset((uint8_t *)new_ptr + old_size, 0,
- new_size - old_size);
- }
- }
-#endif
mem_getstats(ctx, new_size);
ADD_TRACE(ctx, new_ptr, new_size, func, file, line);
# CSPRNG
'arc4random': '#include <stdlib.h>',
+ # C23 sized frees
+ 'free_sized': '#include <stdlib.h>',
+ 'free_aligned_sized': '#include <stdlib.h>',
+
# Misc.
'chroot': '#include <unistd.h>',
'clock_gettime': '#include <time.h>',
endif
endif
-foreach fn : [
- 'malloc_usable_size',
-]
- if cc.has_function(
- fn,
- prefix: '#include <malloc.h>',
- args: sys_defines,
- dependencies: thread_dep,
- )
- config.set('HAVE_@0@'.format(fn.to_upper()), 1)
- endif
-endforeach
-
-foreach fn : [
- 'malloc_size',
-]
- if cc.has_function(
- fn,
- prefix: '#include <malloc/malloc.h>',
- args: sys_defines,
- dependencies: thread_dep,
- )
- config.set('HAVE_@0@'.format(fn.to_upper()), 1)
- endif
-endforeach
-
## dnstap
dnstap_dep = null_dep # Will be filled later
isc_mempool_destroy(&mp1);
}
+/* aligned memory system tests */
+ISC_RUN_TEST_IMPL(isc_mem_get_align) {
+ isc_mem_t *mctx2 = NULL;
+ void *ptr;
+ size_t alignment;
+ uintptr_t aligned;
+
+ /* Check different alignment sizes up to the page size */
+ for (alignment = sizeof(void *); alignment <= 4096; alignment *= 2) {
+ size_t size = alignment / 2 - 1;
+ ptr = isc_mem_getx(isc_g_mctx, size, ISC_MEM_ALIGN(alignment));
+
+ /* Check if the pointer is properly aligned */
+ aligned = (((uintptr_t)ptr / alignment) * alignment);
+ assert_ptr_equal(aligned, (uintptr_t)ptr);
+
+ /* Check if we can resize to <alignment, 2*alignment> range */
+ ptr = isc_mem_regetx(isc_g_mctx, ptr, size,
+ size * 2 + alignment,
+ ISC_MEM_ALIGN(alignment));
+
+ /* Check if the pointer is still properly aligned */
+ aligned = (((uintptr_t)ptr / alignment) * alignment);
+ assert_ptr_equal(aligned, (uintptr_t)ptr);
+
+ isc_mem_putx(isc_g_mctx, ptr, size * 2 + alignment,
+ ISC_MEM_ALIGN(alignment));
+
+ /* Check whether isc_mem_putanddetachx() also works */
+ isc_mem_create("mctx2", &mctx2);
+ ptr = isc_mem_getx(mctx2, size, ISC_MEM_ALIGN(alignment));
+ isc_mem_putanddetachx(&mctx2, ptr, size,
+ ISC_MEM_ALIGN(alignment));
+ }
+}
+
+/* aligned memory system tests */
+ISC_RUN_TEST_IMPL(isc_mem_allocate_align) {
+ void *ptr;
+ size_t alignment;
+ uintptr_t aligned;
+
+ /* Check different alignment sizes up to the page size */
+ for (alignment = sizeof(void *); alignment <= 4096; alignment *= 2) {
+ size_t size = alignment / 2 - 1;
+ ptr = isc_mem_allocatex(isc_g_mctx, size,
+ ISC_MEM_ALIGN(alignment));
+
+ /* Check if the pointer is properly aligned */
+ aligned = (((uintptr_t)ptr / alignment) * alignment);
+ assert_ptr_equal(aligned, (uintptr_t)ptr);
+
+ /* Check if we can resize to <alignment, 2*alignment> range */
+ ptr = isc_mem_reallocatex(isc_g_mctx, ptr, size * 2 + alignment,
+ ISC_MEM_ALIGN(alignment));
+
+ /* Check if the pointer is still properly aligned */
+ aligned = (((uintptr_t)ptr / alignment) * alignment);
+ assert_ptr_equal(aligned, (uintptr_t)ptr);
+
+ isc_mem_freex(isc_g_mctx, ptr, ISC_MEM_ALIGN(alignment));
+ }
+}
+
/* zeroed memory system tests */
ISC_RUN_TEST_IMPL(isc_mem_cget_zero) {
uint8_t *ptr;
ISC_TEST_LIST_START
ISC_TEST_ENTRY(isc_mem_get)
+ISC_TEST_ENTRY(isc_mem_get_align)
+ISC_TEST_ENTRY(isc_mem_allocate_align)
ISC_TEST_ENTRY(isc_mem_cget_zero)
ISC_TEST_ENTRY(isc_mem_callocate_zero)
ISC_TEST_ENTRY(isc_mem_inuse)
projects / thirdparty / bind9.git / commitdiff
