Merge branch 'jm/mem-pool'

An reusable "memory pool" implementation has been extracted from
fast-import.c, which in turn has become the first user of the
mem-pool API.

* jm/mem-pool:
  mem-pool: move reusable parts of memory pool into its own file
  fast-import: introduce mem_pool type
  fast-import: rename mem_pool type to mp_block

diff --git a/Makefile b/Makefile
index e7191a7e60819465bf90d9fd062fa75e8aee6212..50da82b0169ad7569a900a5b7dbd0b98f147a7f0 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -845,6 +845,7 @@ LIB_OBJS += log-tree.o
 LIB_OBJS += mailinfo.o
 LIB_OBJS += mailmap.o
 LIB_OBJS += match-trees.o
+LIB_OBJS += mem-pool.o
 LIB_OBJS += merge.o
 LIB_OBJS += merge-blobs.o
 LIB_OBJS += merge-recursive.o

diff --git a/fast-import.c b/fast-import.c
index 99f8f56e8c4fbd20b6e6186e26d89eda1baff0d0..05d1079d2380694eb53570ede6eaaed97fc4c8e0 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -170,6 +170,7 @@ Format of STDIN stream:
 #include "run-command.h"
 #include "packfile.h"
 #include "object-store.h"
+#include "mem-pool.h"
 
 #define PACK_ID_BITS 16
 #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
@@ -211,13 +212,6 @@ struct last_object {
        unsigned no_swap : 1;
 };
 
-struct mem_pool {
-       struct mem_pool *next_pool;
-       char *next_free;
-       char *end;
-       uintmax_t space[FLEX_ARRAY]; /* more */
-};
-
 struct atom_str {
        struct atom_str *next_atom;
        unsigned short str_len;
@@ -306,9 +300,8 @@ static int global_argc;
 static const char **global_argv;
 
 /* Memory pools */
-static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
-static size_t total_allocd;
-static struct mem_pool *mem_pool;
+static struct mem_pool fi_mem_pool =  {NULL, 2*1024*1024 -
+                                      sizeof(struct mp_block), 0 };
 
 /* Atom management */
 static unsigned int atom_table_sz = 4451;
@@ -343,6 +336,7 @@ static unsigned int tree_entry_alloc = 1000;
 static void *avail_tree_entry;
 static unsigned int avail_tree_table_sz = 100;
 static struct avail_tree_content **avail_tree_table;
+static size_t tree_entry_allocd;
 static struct strbuf old_tree = STRBUF_INIT;
 static struct strbuf new_tree = STRBUF_INIT;
 
@@ -636,49 +630,10 @@ static unsigned int hc_str(const char *s, size_t len)
        return r;
 }
 
-static void *pool_alloc(size_t len)
-{
-       struct mem_pool *p;
-       void *r;
-
-       /* round up to a 'uintmax_t' alignment */
-       if (len & (sizeof(uintmax_t) - 1))
-               len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1));
-
-       for (p = mem_pool; p; p = p->next_pool)
-               if ((p->end - p->next_free >= len))
-                       break;
-
-       if (!p) {
-               if (len >= (mem_pool_alloc/2)) {
-                       total_allocd += len;
-                       return xmalloc(len);
-               }
-               total_allocd += sizeof(struct mem_pool) + mem_pool_alloc;
-               p = xmalloc(st_add(sizeof(struct mem_pool), mem_pool_alloc));
-               p->next_pool = mem_pool;
-               p->next_free = (char *) p->space;
-               p->end = p->next_free + mem_pool_alloc;
-               mem_pool = p;
-       }
-
-       r = p->next_free;
-       p->next_free += len;
-       return r;
-}
-
-static void *pool_calloc(size_t count, size_t size)
-{
-       size_t len = count * size;
-       void *r = pool_alloc(len);
-       memset(r, 0, len);
-       return r;
-}
-
 static char *pool_strdup(const char *s)
 {
        size_t len = strlen(s) + 1;
-       char *r = pool_alloc(len);
+       char *r = mem_pool_alloc(&fi_mem_pool, len);
        memcpy(r, s, len);
        return r;
 }
@@ -687,7 +642,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)
 {
        struct mark_set *s = marks;
        while ((idnum >> s->shift) >= 1024) {
-               s = pool_calloc(1, sizeof(struct mark_set));
+               s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
                s->shift = marks->shift + 10;
                s->data.sets[0] = marks;
                marks = s;
@@ -696,7 +651,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)
                uintmax_t i = idnum >> s->shift;
                idnum -= i << s->shift;
                if (!s->data.sets[i]) {
-                       s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
+                       s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
                        s->data.sets[i]->shift = s->shift - 10;
                }
                s = s->data.sets[i];
@@ -734,7 +689,7 @@ static struct atom_str *to_atom(const char *s, unsigned short len)
                if (c->str_len == len && !strncmp(s, c->str_dat, len))
                        return c;
 
-       c = pool_alloc(sizeof(struct atom_str) + len + 1);
+       c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1);
        c->str_len = len;
        memcpy(c->str_dat, s, len);
        c->str_dat[len] = 0;
@@ -765,7 +720,7 @@ static struct branch *new_branch(const char *name)
        if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))
                die("Branch name doesn't conform to GIT standards: %s", name);
 
-       b = pool_calloc(1, sizeof(struct branch));
+       b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch));
        b->name = pool_strdup(name);
        b->table_next_branch = branch_table[hc];
        b->branch_tree.versions[0].mode = S_IFDIR;
@@ -801,7 +756,7 @@ static struct tree_content *new_tree_content(unsigned int cnt)
                        avail_tree_table[hc] = f->next_avail;
        } else {
                cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
-               f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
+               f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt);
                f->entry_capacity = cnt;
        }
 
@@ -846,7 +801,7 @@ static struct tree_entry *new_tree_entry(void)
 
        if (!avail_tree_entry) {
                unsigned int n = tree_entry_alloc;
-               total_allocd += n * sizeof(struct tree_entry);
+               tree_entry_allocd += n * sizeof(struct tree_entry);
                ALLOC_ARRAY(e, n);
                avail_tree_entry = e;
                while (n-- > 1) {
@@ -2867,7 +2822,7 @@ static void parse_new_tag(const char *arg)
        enum object_type type;
        const char *v;
 
-       t = pool_alloc(sizeof(struct tag));
+       t = mem_pool_alloc(&fi_mem_pool, sizeof(struct tag));
        memset(t, 0, sizeof(struct tag));
        t->name = pool_strdup(arg);
        if (last_tag)
@@ -3466,12 +3421,12 @@ int cmd_main(int argc, const char **argv)
        atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
        branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
        avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
-       marks = pool_calloc(1, sizeof(struct mark_set));
+       marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
 
        global_argc = argc;
        global_argv = argv;
 
-       rc_free = pool_alloc(cmd_save * sizeof(*rc_free));
+       rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free));
        for (i = 0; i < (cmd_save - 1); i++)
                rc_free[i].next = &rc_free[i + 1];
        rc_free[cmd_save - 1].next = NULL;
@@ -3545,8 +3500,8 @@ int cmd_main(int argc, const char **argv)
                fprintf(stderr, "Total branches:  %10lu (%10lu loads     )\n", branch_count, branch_load_count);
                fprintf(stderr, "      marks:     %10" PRIuMAX " (%10" PRIuMAX " unique    )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
                fprintf(stderr, "      atoms:     %10u\n", atom_cnt);
-               fprintf(stderr, "Memory total:    %10" PRIuMAX " KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
-               fprintf(stderr, "       pools:    %10lu KiB\n", (unsigned long)(total_allocd/1024));
+               fprintf(stderr, "Memory total:    %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024);
+               fprintf(stderr, "       pools:    %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024));
                fprintf(stderr, "     objects:    %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
                fprintf(stderr, "---------------------------------------------------------------------\n");
                pack_report();

diff --git a/mem-pool.c b/mem-pool.c
new file mode 100644 (file)
index 0000000..389d7af
--- /dev/null
+++ b/mem-pool.c
@@ -0,0 +1,55 @@
+/*
+ * Memory Pool implementation logic.
+ */
+
+#include "cache.h"
+#include "mem-pool.h"
+
+static struct mp_block *mem_pool_alloc_block(struct mem_pool *mem_pool, size_t block_alloc)
+{
+       struct mp_block *p;
+
+       mem_pool->pool_alloc += sizeof(struct mp_block) + block_alloc;
+       p = xmalloc(st_add(sizeof(struct mp_block), block_alloc));
+       p->next_block = mem_pool->mp_block;
+       p->next_free = (char *)p->space;
+       p->end = p->next_free + block_alloc;
+       mem_pool->mp_block = p;
+
+       return p;
+}
+
+void *mem_pool_alloc(struct mem_pool *mem_pool, size_t len)
+{
+       struct mp_block *p;
+       void *r;
+
+       /* round up to a 'uintmax_t' alignment */
+       if (len & (sizeof(uintmax_t) - 1))
+               len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1));
+
+       for (p = mem_pool->mp_block; p; p = p->next_block)
+               if (p->end - p->next_free >= len)
+                       break;
+
+       if (!p) {
+               if (len >= (mem_pool->block_alloc / 2)) {
+                       mem_pool->pool_alloc += len;
+                       return xmalloc(len);
+               }
+
+               p = mem_pool_alloc_block(mem_pool, mem_pool->block_alloc);
+       }
+
+       r = p->next_free;
+       p->next_free += len;
+       return r;
+}
+
+void *mem_pool_calloc(struct mem_pool *mem_pool, size_t count, size_t size)
+{
+       size_t len = st_mult(count, size);
+       void *r = mem_pool_alloc(mem_pool, len);
+       memset(r, 0, len);
+       return r;
+}

diff --git a/mem-pool.h b/mem-pool.h
new file mode 100644 (file)
index 0000000..829ad58
--- /dev/null
+++ b/mem-pool.h
@@ -0,0 +1,34 @@
+#ifndef MEM_POOL_H
+#define MEM_POOL_H
+
+struct mp_block {
+       struct mp_block *next_block;
+       char *next_free;
+       char *end;
+       uintmax_t space[FLEX_ARRAY]; /* more */
+};
+
+struct mem_pool {
+       struct mp_block *mp_block;
+
+       /*
+        * The amount of available memory to grow the pool by.
+        * This size does not include the overhead for the mp_block.
+        */
+       size_t block_alloc;
+
+       /* The total amount of memory allocated by the pool. */
+       size_t pool_alloc;
+};
+
+/*
+ * Alloc memory from the mem_pool.
+ */
+void *mem_pool_alloc(struct mem_pool *pool, size_t len);
+
+/*
+ * Allocate and zero memory from the memory pool.
+ */
+void *mem_pool_calloc(struct mem_pool *pool, size_t count, size_t size);
+
+#endif