mempool-allocfree: Add linked-list malloc allocation

This is useful when you want to actually free memory from pool too

diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index b6d223168266967b5a4a1555c078179524cb560f..b8628318316e7b771e899996554a440a6fbf6186 100644 (file)
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -99,6 +99,7 @@ liblib_la_SOURCES = \
        md5.c \
        memarea.c \
        mempool.c \
+       mempool-allocfree.c \
        mempool-alloconly.c \
        mempool-datastack.c \
        mempool-system.c \
@@ -368,6 +369,7 @@ test_lib_SOURCES = \
        test-malloc-overflow.c \
        test-memarea.c \
        test-mempool.c \
+       test-mempool-allocfree.c \
        test-mempool-alloconly.c \
        test-pkcs5.c \
        test-net.c \

diff --git a/src/lib/mempool-allocfree.c b/src/lib/mempool-allocfree.c
new file mode 100644 (file)
index 0000000..15908dc
--- /dev/null
+++ b/src/lib/mempool-allocfree.c
@@ -0,0 +1,252 @@
+/* Copyright (c) 2017 Dovecot authors, see the included COPYING file */
+
+/* @UNSAFE: whole file */
+#include "lib.h"
+#include "safe-memset.h"
+#include "mempool.h"
+#include "llist.h"
+
+#define MAX_ALLOC_SIZE SSIZE_T_MAX
+
+struct allocfree_pool {
+       struct pool pool;
+       int refcount;
+       size_t total_alloc_count;
+       size_t total_alloc_used;
+
+       struct pool_block *blocks;
+#ifdef DEBUG
+       char *name;
+#endif
+       bool clean_frees;
+};
+
+struct pool_block {
+       struct pool_block *prev,*next;
+
+       size_t size;
+       unsigned char *block;
+};
+
+#define SIZEOF_ALLOCFREE_POOL MEM_ALIGN(sizeof(struct allocfree_pool))
+#define SIZEOF_POOLBLOCK (MEM_ALIGN(sizeof(struct pool_block)))
+
+static const char *pool_allocfree_get_name(pool_t pool);
+static void pool_allocfree_ref(pool_t pool);
+static void pool_allocfree_unref(pool_t *pool);
+static void *pool_allocfree_malloc(pool_t pool, size_t size);
+static void pool_allocfree_free(pool_t pool, void *mem);
+static void *pool_allocfree_realloc(pool_t pool, void *mem,
+                                   size_t old_size, size_t new_size);
+static void pool_allocfree_clear(pool_t pool);
+static size_t pool_allocfree_get_max_easy_alloc_size(pool_t pool);
+
+static const struct pool_vfuncs static_allocfree_pool_vfuncs = {
+       pool_allocfree_get_name,
+
+       pool_allocfree_ref,
+       pool_allocfree_unref,
+
+       pool_allocfree_malloc,
+       pool_allocfree_free,
+
+       pool_allocfree_realloc,
+
+       pool_allocfree_clear,
+       pool_allocfree_get_max_easy_alloc_size
+};
+
+static const struct pool static_allocfree_pool = {
+       .v = &static_allocfree_pool_vfuncs,
+
+       .alloconly_pool = FALSE,
+       .datastack_pool = FALSE
+};
+
+pool_t pool_allocfree_create(const char *name ATTR_UNUSED)
+{
+       struct allocfree_pool *pool;
+       pool = calloc(1, SIZEOF_ALLOCFREE_POOL);
+       if (pool == NULL)
+               i_fatal_status(FATAL_OUTOFMEM, "calloc(1, %"PRIuSIZE_T")",
+                              SIZEOF_ALLOCFREE_POOL);
+#ifdef DEBUG
+       pool->name = strdup(name);
+#endif
+       pool->pool = static_allocfree_pool;
+       pool->refcount = 1;
+       return &pool->pool;
+}
+
+pool_t pool_allocfree_create_clean(const char *name)
+{
+       struct allocfree_pool *apool;
+       pool_t pool;
+
+       pool = pool_allocfree_create(name);
+       apool = (struct allocfree_pool *)pool;
+       apool->clean_frees = TRUE;
+       return pool;
+}
+
+static void pool_allocfree_destroy(struct allocfree_pool *apool)
+{
+       pool_allocfree_clear(&apool->pool);
+       if (apool->clean_frees)
+               safe_memset(apool, 0, SIZEOF_ALLOCFREE_POOL);
+#ifdef DEBUG
+       free(apool->name);
+#endif
+       free(apool);
+}
+
+static const char *pool_allocfree_get_name(pool_t pool ATTR_UNUSED)
+{
+#ifdef DEBUG
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       return apool->name;
+#else
+       return "alloc";
+#endif
+}
+
+static void pool_allocfree_ref(pool_t pool)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       i_assert(apool->refcount > 0);
+
+       apool->refcount++;
+}
+
+static void pool_allocfree_unref(pool_t *_pool)
+{
+       pool_t pool = *_pool;
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       i_assert(apool->refcount > 0);
+
+       /* erase the pointer before freeing anything, as the pointer may
+          exist inside the pool's memory area */
+       *_pool = NULL;
+
+       if (--apool->refcount > 0)
+               return;
+
+       pool_allocfree_destroy(apool);
+}
+
+static void *pool_block_attach(struct allocfree_pool *apool, struct pool_block *block)
+{
+       i_assert(block->size > 0);
+       DLLIST_PREPEND(&apool->blocks, block);
+       block->block = PTR_OFFSET(block,SIZEOF_POOLBLOCK);
+       apool->total_alloc_used += block->size;
+       apool->total_alloc_count++;
+       return block->block;
+}
+
+static struct pool_block *
+pool_block_detach(struct allocfree_pool *apool, unsigned char *mem)
+{
+       struct pool_block *block = PTR_OFFSET(mem, -SIZEOF_POOLBLOCK);
+
+       /* make sure the block we are dealing with is correct */
+       i_assert(block->block == mem);
+       i_assert((block->prev == NULL || block->prev->next == block) &&
+                (block->next == NULL || block->next->prev == block));
+
+       i_assert(apool->total_alloc_used >= block->size);
+       i_assert(apool->total_alloc_count > 0);
+       DLLIST_REMOVE(&apool->blocks, block);
+       apool->total_alloc_used -= block->size;
+       apool->total_alloc_count--;
+
+       return block;
+}
+
+static void *pool_allocfree_malloc(pool_t pool, size_t size)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+
+       if (unlikely(size == 0 || size > SSIZE_T_MAX))
+               i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
+
+       struct pool_block *block = calloc(1, SIZEOF_POOLBLOCK + size);
+       if (block == NULL)
+               i_fatal_status(FATAL_OUTOFMEM, "calloc(1, %"PRIuSIZE_T")",
+                              SIZEOF_POOLBLOCK + size);
+       block->size = size;
+       return pool_block_attach(apool, block);
+}
+
+static void pool_allocfree_free(pool_t pool, void *mem)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       if (mem == NULL)
+               return;
+       struct pool_block *block = pool_block_detach(apool, mem);
+       if (apool->clean_frees)
+               safe_memset(block, 0, SIZEOF_POOLBLOCK+block->size);
+       free(block);
+}
+
+static void *pool_allocfree_realloc(pool_t pool, void *mem,
+                                   size_t old_size, size_t new_size)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       unsigned char *new_mem;
+
+       if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
+               i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
+
+       if (mem == NULL)
+               return pool_allocfree_malloc(pool, new_size);
+
+       struct pool_block *block = pool_block_detach(apool, mem);
+       if ((new_mem = realloc(block, SIZEOF_POOLBLOCK+new_size)) == NULL)
+               i_fatal_status(FATAL_OUTOFMEM, "realloc(block, %"PRIuSIZE_T")",
+                              SIZEOF_POOLBLOCK+new_size);
+
+       /* zero out new memory */
+       if (new_size > old_size)
+               memset(new_mem + SIZEOF_POOLBLOCK + old_size, 0,
+                      new_size - old_size);
+       block = (struct pool_block*)new_mem;
+       block->size = new_size;
+       return pool_block_attach(apool, block);
+}
+
+static void pool_allocfree_clear(pool_t pool)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+
+       while(apool->blocks != NULL)
+               pool_allocfree_free(pool, apool->blocks->block);
+       i_assert(apool->total_alloc_used == 0 && apool->total_alloc_count == 0);
+}
+
+static size_t pool_allocfree_get_max_easy_alloc_size(pool_t pool ATTR_UNUSED)
+{
+       return 0;
+}
+
+size_t pool_allocfree_get_total_used_size(pool_t pool)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       return apool->total_alloc_used;
+}
+
+size_t pool_allocfree_get_total_alloc_size(pool_t pool)
+{
+       struct allocfree_pool *apool =
+               container_of(pool, struct allocfree_pool, pool);
+       return apool->total_alloc_used +
+              SIZEOF_POOLBLOCK*apool->total_alloc_count + sizeof(*apool);
+}

diff --git a/src/lib/mempool.h b/src/lib/mempool.h
index 8814707de1970024175e3172a9e9bb64e49cd8a0..023beedad2a545b3034e3f4e25debe93e29c9a65 100644 (file)
--- a/src/lib/mempool.h
+++ b/src/lib/mempool.h
@@ -68,6 +68,14 @@ pool_t pool_alloconly_create_clean(const char *name, size_t size);
    that the stack frame is the same. This should make it quite safe to use. */
 pool_t pool_datastack_create(void);
 
+/* Create new alloc pool. This is very similar to system pool, but it
+   will deallocate all memory on deinit. */
+pool_t pool_allocfree_create(const char *name);
+
+/* Like alloc pool, but all memory is cleaned before freeing.
+   See pool_alloconly_create_clean. */
+pool_t pool_allocfree_create_clean(const char *name);
+
 /* Similar to nearest_power(), but try not to exceed buffer's easy
    allocation size. If you don't have any explicit minimum size, use
    old_size + 1. */
@@ -144,6 +152,11 @@ size_t pool_alloconly_get_total_used_size(pool_t pool);
 /* Returns how much system memory has been allocated for this pool. */
 size_t pool_alloconly_get_total_alloc_size(pool_t pool);
 
+/* Returns how much memory has been allocated from this pool. */
+size_t pool_allocfree_get_total_used_size(pool_t pool);
+/* Returns how much system memory has been allocated for this pool. */
+size_t pool_allocfree_get_total_alloc_size(pool_t pool);
+
 /* private: */
 void pool_system_free(pool_t pool, void *mem);
 

diff --git a/src/lib/test-lib.inc b/src/lib/test-lib.inc
index 255c5ab66cf7a04a94aadddd565076f10f86223f..294225e6d462105e3510d61c027b4f98ccb6d3ad 100644 (file)
--- a/src/lib/test-lib.inc
+++ b/src/lib/test-lib.inc
@@ -54,6 +54,8 @@ TEST(test_mempool)
 FATAL(fatal_mempool)
 TEST(test_mempool_alloconly)
 FATAL(fatal_mempool_alloconly)
+TEST(test_mempool_allocfree)
+FATAL(fatal_mempool_allocfree)
 TEST(test_net)
 TEST(test_numpack)
 TEST(test_ostream_buffer)

diff --git a/src/lib/test-mempool-allocfree.c b/src/lib/test-mempool-allocfree.c
new file mode 100644 (file)
index 0000000..9a5e179
--- /dev/null
+++ b/src/lib/test-mempool-allocfree.c
@@ -0,0 +1,130 @@
+/* Copyright (c) 2007-2017 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+
+#define SENSE 0xAB /* produces 10101011 */
+
+static bool mem_has_bytes(const void *mem, size_t size, uint8_t b)
+{
+       const uint8_t *bytes = mem;
+       unsigned int i;
+
+       for (i = 0; i < size; i++) {
+               if (bytes[i] != b) {
+                       i_debug("bytes[%u] != %u", i, b);
+                       return FALSE;
+               }
+       }
+       return TRUE;
+}
+
+void test_mempool_allocfree(void)
+{
+       pool_t pool;
+       unsigned int i;
+       size_t last_alloc = 0;
+       size_t used = 0;
+       size_t count = 0;
+       void *mem = NULL;
+
+       test_begin("mempool_allocfree");
+       pool = pool_allocfree_create("test");
+
+       for(i = 0; i <= 1000; i++) {
+               /* release previous allocation */
+               if ((i % 3) == 0) {
+                       if (mem != NULL) {
+                               test_assert_idx(mem_has_bytes(mem, last_alloc, SENSE), i);
+                               used -= last_alloc;
+                               count--;
+                       }
+                       last_alloc = 0;
+                       p_free(pool, mem);
+               /* grow previous allocation */
+               } else if ((i % 5) == 0) {
+                       if (mem != NULL)
+                               used -= last_alloc;
+                       else
+                               count++;
+                       mem = p_realloc(pool, mem, last_alloc, i*2);
+                       if (last_alloc > 0)
+                               test_assert_idx(mem_has_bytes(mem, last_alloc, SENSE), i);
+                       memset(mem, SENSE, i*2);
+                       last_alloc = i*2;
+                       used += i*2;
+               /* shrink previous allocation */
+               } else if ((i % 7) == 0) {
+                       if (mem != NULL)
+                               used -= last_alloc;
+                       else
+                               count++;
+                       mem = p_realloc(pool, mem, last_alloc, i-2);
+                       if (last_alloc > 0)
+                               test_assert_idx(mem_has_bytes(mem, i-2, SENSE), i);
+                       memset(mem, SENSE, i-2);
+                       last_alloc = i-2;
+                       used += i-2;
+               /* allocate some memory */
+               } else {
+                       mem = p_malloc(pool, i);
+                       /* fill it with sense marker */
+                       memset(mem, SENSE, i);
+                       used += i;
+                       count++;
+                       last_alloc = i;
+               }
+       }
+
+       test_assert(pool_allocfree_get_total_used_size(pool) == used);
+
+       pool_unref(&pool);
+
+       /* make sure realloc works correctly */
+       pool = pool_allocfree_create("test");
+       mem = NULL;
+
+       for(i = 1; i < 1000; i++) {
+               mem = p_realloc(pool, mem, i-1, i);
+               test_assert_idx(mem_has_bytes(mem, i-1, 0xde), i);
+               memset(mem, 0xde, i);
+       }
+
+       pool_unref(&pool);
+
+       test_end();
+}
+
+enum fatal_test_state fatal_mempool_allocfree(unsigned int stage)
+{
+       static pool_t pool;
+
+       if (pool == NULL && stage != 0)
+               return FATAL_TEST_FAILURE;
+
+       switch(stage) {
+       case 0: /* forbidden size */
+               test_begin("fatal_mempool_allocfree");
+               pool = pool_allocfree_create("fatal");
+               (void)p_malloc(pool, 0);
+               return FATAL_TEST_FAILURE;
+
+       case 1: /* logically impossible size */
+               (void)p_malloc(pool, SSIZE_T_MAX + 1ULL);
+               return FATAL_TEST_FAILURE;
+
+       case 2: /* physically impossible size */
+               (void)p_malloc(pool, SSIZE_T_MAX - (size_t)MEM_ALIGN(1));
+               return FATAL_TEST_FAILURE;
+
+       /* Continue with other tests as follows:
+       case 3:
+               something_fatal();
+               return FATAL_TEST_FAILURE;
+       */
+       }
+
+       /* Either our tests have finished, or the test suite has got confused. */
+       pool_unref(&pool);
+       test_end();
+       return FATAL_TEST_FINISHED;
+}