From: Olivier Houchard Date: Wed, 24 Jan 2018 17:38:31 +0000 (+0100) Subject: MINOR: pools/threads: Implement lockless memory pools. X-Git-Tag: v1.9-dev1~449 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cf975d4;p=thirdparty%2Fhaproxy.git MINOR: pools/threads: Implement lockless memory pools. On CPUs that support a double-width compare-and-swap, implement lockless pools. --- diff --git a/include/common/buffer.h b/include/common/buffer.h index 976085e5eb..20070ccfe8 100644 --- a/include/common/buffer.h +++ b/include/common/buffer.h @@ -735,13 +735,17 @@ static inline struct buffer *b_alloc_margin(struct buffer **buf, int margin) return *buf; *buf = &buf_wanted; +#ifndef HA_HAVE_CAS_DW HA_SPIN_LOCK(POOL_LOCK, &pool_head_buffer->lock); +#endif /* fast path */ if ((pool_head_buffer->allocated - pool_head_buffer->used) > margin) { b = __pool_get_first(pool_head_buffer); if (likely(b)) { +#ifndef HA_HAVE_CAS_DW HA_SPIN_UNLOCK(POOL_LOCK, &pool_head_buffer->lock); +#endif b->size = pool_head_buffer->size - sizeof(struct buffer); b_reset(b); *buf = b; @@ -752,7 +756,9 @@ static inline struct buffer *b_alloc_margin(struct buffer **buf, int margin) /* slow path, uses malloc() */ b = __pool_refill_alloc(pool_head_buffer, margin); +#ifndef HA_HAVE_CAS_DW HA_SPIN_UNLOCK(POOL_LOCK, &pool_head_buffer->lock); +#endif if (b) { b->size = pool_head_buffer->size - sizeof(struct buffer); diff --git a/include/common/memory.h b/include/common/memory.h index 80078558b6..9ec9fc7ef5 100644 --- a/include/common/memory.h +++ b/include/common/memory.h @@ -47,10 +47,20 @@ #define POOL_LINK(pool, item) ((void **)(item)) #endif +#ifdef HA_HAVE_CAS_DW +struct pool_free_list { + void **free_list; + uintptr_t seq; +}; +#endif + struct pool_head { - __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */ void **free_list; - struct list list; /* list of all known pools */ +#ifdef HA_HAVE_CAS_DW + uintptr_t seq; +#else + __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */ +#endif unsigned int used; /* how many chunks are currently in use */ unsigned int allocated; /* how many chunks have been allocated */ unsigned int limit; /* hard limit on the number of chunks */ @@ -59,6 +69,7 @@ struct pool_head { unsigned int flags; /* MEM_F_* */ unsigned int users; /* number of pools sharing this zone */ unsigned int failed; /* failed allocations */ + struct list list; /* list of all known pools */ char name[12]; /* name of the pool */ } __attribute__((aligned(64))); @@ -111,6 +122,110 @@ void pool_gc(struct pool_head *pool_ctx); */ void *pool_destroy(struct pool_head *pool); +#ifdef HA_HAVE_CAS_DW +/* + * Returns a pointer to type taken from the pool if + * available, otherwise returns NULL. No malloc() is attempted, and poisonning + * is never performed. The purpose is to get the fastest possible allocation. + */ +static inline void *__pool_get_first(struct pool_head *pool) +{ + struct pool_free_list cmp, new; + + cmp.seq = pool->seq; + __ha_barrier_load(); + + cmp.free_list = pool->free_list; + do { + if (cmp.free_list == NULL) + return NULL; + new.seq = cmp.seq + 1; + __ha_barrier_load(); + new.free_list = *POOL_LINK(pool, cmp.free_list); + } while (__ha_cas_dw((void *)&pool->free_list, (void *)&cmp, (void *)&new) == 0); +end: + HA_ATOMIC_ADD(&pool->used, 1); +#ifdef DEBUG_MEMORY_POOLS + /* keep track of where the element was allocated from */ + *POOL_LINK(pool, cmp.free_list) = (void *)pool; +#endif + return cmp.free_list; +} + +static inline void *pool_get_first(struct pool_head *pool) +{ + void *ret; + + ret = __pool_get_first(pool); + return ret; +} +/* + * Returns a pointer to type taken from the pool or + * dynamically allocated. In the first case, is updated to point to + * the next element in the list. No memory poisonning is ever performed on the + * returned area. + */ +static inline void *pool_alloc_dirty(struct pool_head *pool) +{ + void *p; + + if ((p = __pool_get_first(pool)) == NULL) + p = __pool_refill_alloc(pool, 0); + return p; +} + +/* + * Returns a pointer to type taken from the pool or + * dynamically allocated. In the first case, is updated to point to + * the next element in the list. Memory poisonning is performed if enabled. + */ +static inline void *pool_alloc(struct pool_head *pool) +{ + void *p; + + p = pool_alloc_dirty(pool); +#ifdef DEBUG_MEMORY_POOLS + if (p) { + /* keep track of where the element was allocated from */ + *POOL_LINK(pool, p) = (void *)pool; + } +#endif + if (p && mem_poison_byte >= 0) { + memset(p, mem_poison_byte, pool->size); + } + + return p; +} + +/* + * Puts a memory area back to the corresponding pool. + * Items are chained directly through a pointer that + * is written in the beginning of the memory area, so + * there's no need for any carrier cell. This implies + * that each memory area is at least as big as one + * pointer. Just like with the libc's free(), nothing + * is done if is NULL. + */ +static inline void pool_free(struct pool_head *pool, void *ptr) +{ + if (likely(ptr != NULL)) { + void *free_list; +#ifdef DEBUG_MEMORY_POOLS + /* we'll get late corruption if we refill to the wrong pool or double-free */ + if (*POOL_LINK(pool, ptr) != (void *)pool) + *(volatile int *)0 = 0; +#endif + free_list = pool->free_list; + do { + *POOL_LINK(pool, ptr) = (void *)free_list; + __ha_barrier_store(); + } while (!HA_ATOMIC_CAS(&pool->free_list, (void *)&free_list, ptr)); +end: + HA_ATOMIC_SUB(&pool->used, 1); + } +} + +#else /* * Returns a pointer to type taken from the pool if * available, otherwise returns NULL. No malloc() is attempted, and poisonning @@ -261,6 +376,7 @@ static inline void pool_free(struct pool_head *pool, void *ptr) HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock); } } +#endif /* HA_HAVE_CAS_DW */ #endif /* _COMMON_MEMORY_H */ /* diff --git a/src/memory.c b/src/memory.c index 4290c4b3fe..929a04ae31 100644 --- a/src/memory.c +++ b/src/memory.c @@ -93,10 +93,136 @@ struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags) LIST_ADDQ(start, &pool->list); } pool->users++; +#ifndef HA_HAVE_CAS_DW HA_SPIN_INIT(&pool->lock); +#endif return pool; } +#ifdef HA_HAVE_CAS_DW +/* Allocates new entries for pool until there are at least + 1 + * available, then returns the last one for immediate use, so that at least + * are left available in the pool upon return. NULL is returned if the + * last entry could not be allocated. It's important to note that at least one + * allocation is always performed even if there are enough entries in the pool. + * A call to the garbage collector is performed at most once in case malloc() + * returns an error, before returning NULL. + */ +void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail) +{ + void *ptr = NULL, *free_list; + int failed = 0; + int size = pool->size; + int limit = pool->limit; + int allocated = pool->allocated, allocated_orig = allocated; + + /* stop point */ + avail += pool->used; + + while (1) { + if (limit && allocated >= limit) { + HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig); + return NULL; + } + + ptr = malloc(size + POOL_EXTRA); + if (!ptr) { + HA_ATOMIC_ADD(&pool->failed, 1); + if (failed) + return NULL; + failed++; + pool_gc(pool); + continue; + } + if (++allocated > avail) + break; + + free_list = pool->free_list; + do { + *POOL_LINK(pool, ptr) = free_list; + __ha_barrier_store(); + } while (HA_ATOMIC_CAS(&pool->free_list, (void *)&free_list, ptr) == 0); + } + + HA_ATOMIC_ADD(&pool->allocated, allocated - allocated_orig); + HA_ATOMIC_ADD(&pool->used, 1); + +#ifdef DEBUG_MEMORY_POOLS + /* keep track of where the element was allocated from */ + *POOL_LINK(pool, ptr) = (void *)pool; +#endif + return ptr; +} +void *pool_refill_alloc(struct pool_head *pool, unsigned int avail) +{ + void *ptr; + + ptr = __pool_refill_alloc(pool, avail); + return ptr; +} +/* + * This function frees whatever can be freed in pool . + */ +void pool_flush(struct pool_head *pool) +{ + void *next, *temp; + int removed = 0; + + if (!pool) + return; + do { + next = pool->free_list; + } while (!HA_ATOMIC_CAS(&pool->free_list, (void *)&next, NULL)); + while (next) { + temp = next; + next = *POOL_LINK(pool, temp); + removed++; + free(temp); + } + pool->free_list = next; + HA_ATOMIC_SUB(&pool->allocated, removed); + /* here, we should have pool->allocate == pool->used */ +} + +/* + * This function frees whatever can be freed in all pools, but respecting + * the minimum thresholds imposed by owners. It takes care of avoiding + * recursion because it may be called from a signal handler. + * + * is unused + */ +void pool_gc(struct pool_head *pool_ctx) +{ + static int recurse; + int cur_recurse = 0; + struct pool_head *entry; + + if (recurse || !HA_ATOMIC_CAS(&recurse, &cur_recurse, 1)) + return; + + list_for_each_entry(entry, &pools, list) { + while ((int)((volatile int)entry->allocated - (volatile int)entry->used) > (int)entry->minavail) { + struct pool_free_list cmp, new; + + cmp.seq = entry->seq; + __ha_barrier_load(); + cmp.free_list = entry->free_list; + __ha_barrier_load(); + if (cmp.free_list == NULL) + break; + new.free_list = *POOL_LINK(entry, cmp.free_list); + new.seq = cmp.seq + 1; + if (__ha_cas_dw(&entry->free_list, &cmp, &new) == 0) + continue; + free(cmp.free_list); + HA_ATOMIC_SUB(&entry->allocated, 1); + } + } + + HA_ATOMIC_STORE(&recurse, 0); +} +#else + /* Allocates new entries for pool until there are at least + 1 * available, then returns the last one for immediate use, so that at least * are left available in the pool upon return. NULL is returned if the @@ -208,6 +334,7 @@ void pool_gc(struct pool_head *pool_ctx) HA_ATOMIC_STORE(&recurse, 0); } +#endif /* * This function destroys a pool by freeing it completely, unless it's still @@ -225,7 +352,9 @@ void *pool_destroy(struct pool_head *pool) pool->users--; if (!pool->users) { LIST_DEL(&pool->list); +#ifndef HA_HAVE_CAS_DW HA_SPIN_DESTROY(&pool->lock); +#endif free(pool); } } @@ -242,7 +371,9 @@ void dump_pools_to_trash() allocated = used = nbpools = 0; chunk_printf(&trash, "Dumping pools usage. Use SIGQUIT to flush them.\n"); list_for_each_entry(entry, &pools, list) { +#ifndef HA_HAVE_CAS_DW HA_SPIN_LOCK(POOL_LOCK, &entry->lock); +#endif chunk_appendf(&trash, " - Pool %s (%d bytes) : %d allocated (%u bytes), %d used, %d failures, %d users%s\n", entry->name, entry->size, entry->allocated, entry->size * entry->allocated, entry->used, entry->failed, @@ -251,7 +382,9 @@ void dump_pools_to_trash() allocated += entry->allocated * entry->size; used += entry->used * entry->size; nbpools++; +#ifndef HA_HAVE_CAS_DW HA_SPIN_UNLOCK(POOL_LOCK, &entry->lock); +#endif } chunk_appendf(&trash, "Total: %d pools, %lu bytes allocated, %lu used.\n", nbpools, allocated, used);