stringpool: Make them initializable right from the file

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/database.c b/src/database.c
index 41b5ffd3c9687bd635915f4521f82035211b1165..3a4f8b8bb13a81dd4099bdb78b3b89edd6b9b173 100644 (file)
--- a/src/database.c
+++ b/src/database.c
@@ -120,11 +120,8 @@ static int loc_database_read_header_v0(struct loc_database* db) {
        off_t pool_offset  = ntohl(header.pool_offset);
        size_t pool_length = ntohl(header.pool_length);
 
-       int r = loc_stringpool_new(db->ctx, &db->pool, 0);
-       if (r)
-               return r;
-
-       r = loc_stringpool_read(db->pool, db->file, pool_offset, pool_length);
+       int r = loc_stringpool_open(db->ctx, &db->pool,
+               db->file, pool_length, pool_offset);
        if (r)
                return r;
 

diff --git a/src/stringpool.c b/src/stringpool.c
index 4a0a46b06027120b21116428f389ac548b18216c..cbb1fb7af5117f9c22cde038eb6fa14902fee8dd 100644 (file)
--- a/src/stringpool.c
+++ b/src/stringpool.c
@@ -27,78 +27,72 @@
 #include "libloc-private.h"
 #include "stringpool.h"
 
+enum loc_stringpool_mode {
+       STRINGPOOL_DEFAULT,
+       STRINGPOOL_MMAP,
+};
+
 struct loc_stringpool {
        struct loc_ctx* ctx;
-
        int refcount;
+
+       enum loc_stringpool_mode mode;
+
        char* data;
-       char* pos;
+       ssize_t length;
 
-       ssize_t max_length;
+       char* pos;
 };
 
-static int loc_stringpool_deallocate(struct loc_stringpool* pool) {
-       if (pool->data) {
-               int r = munmap(pool->data, pool->max_length);
-               if (r) {
-                       ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
-                               pool->data, strerror(errno));
+static int __loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, enum loc_stringpool_mode mode) {
+       struct loc_stringpool* p = calloc(1, sizeof(*p));
+       if (!p)
+               return -ENOMEM;
+
+       p->ctx = loc_ref(ctx);
+       p->refcount = 1;
 
-                       return r;
-               }
-       }
+       // Save mode
+       p->mode = mode;
+
+       *pool = p;
 
        return 0;
 }
 
-static int loc_stringpool_allocate(struct loc_stringpool* pool, size_t length) {
-       // Drop old data
-       int r = loc_stringpool_deallocate(pool);
-       if (r)
-               return r;
+LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool) {
+       return __loc_stringpool_new(ctx, pool, STRINGPOOL_DEFAULT);
+}
 
-       pool->max_length = length;
+static int loc_stringpool_mmap(struct loc_stringpool* pool, FILE* f, size_t length, off_t offset) {
+       if (pool->mode != STRINGPOOL_MMAP)
+               return -EINVAL;
 
-       // Align to page size
-       while (pool->max_length % sysconf(_SC_PAGE_SIZE) > 0)
-               pool->max_length++;
+       DEBUG(pool->ctx, "Reading string pool starting from %zu (%zu bytes)\n", offset, length);
 
-       DEBUG(pool->ctx, "Allocating pool of %zu bytes\n", pool->max_length);
+       // Map file content into memory
+       pool->data = pool->pos = mmap(NULL, length, PROT_READ,
+               MAP_PRIVATE, fileno(f), offset);
 
-       // Allocate some memory
-       pool->data = pool->pos = mmap(NULL, pool->max_length,
-               PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+       // Store size of section
+       pool->length = length;
 
-       if (pool->data == MAP_FAILED) {
-               DEBUG(pool->ctx, "%s\n", strerror(errno));
+       if (pool->data == MAP_FAILED)
                return -errno;
-       }
-
-       DEBUG(pool->ctx, "Allocated pool at %p\n", pool->data);
 
        return 0;
 }
 
-LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length) {
-       struct loc_stringpool* p = calloc(1, sizeof(*p));
-       if (!p)
-               return -ENOMEM;
-
-       p->ctx = loc_ref(ctx);
-       p->refcount = 1;
-
-       // Allocate the data section
-       if (max_length > 0) {
-               int r = loc_stringpool_allocate(p, max_length);
-               if (r) {
-                       loc_stringpool_unref(p);
-                       return r;
-               }
-       }
+LOC_EXPORT int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
+               FILE* f, size_t length, off_t offset) {
+       int r = __loc_stringpool_new(ctx, pool, STRINGPOOL_MMAP);
+       if (r)
+               return r;
 
-       DEBUG(p->ctx, "String pool allocated at %p\n", p);
-       DEBUG(p->ctx, "  Maximum size: %zu bytes\n", p->max_length);
-       *pool = p;
+       // Map data into memory
+       r = loc_stringpool_mmap(*pool, f, length, offset);
+       if (r)
+               return r;
 
        return 0;
 }
@@ -111,8 +105,24 @@ LOC_EXPORT struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool
 
 static void loc_stringpool_free(struct loc_stringpool* pool) {
        DEBUG(pool->ctx, "Releasing string pool %p\n", pool);
+       int r;
+
+       switch (pool->mode) {
+               case STRINGPOOL_DEFAULT:
+                       if (pool->data)
+                               free(pool->data);
+                       break;
+
+               case STRINGPOOL_MMAP:
+                       if (pool->data) {
+                               r = munmap(pool->data, pool->length);
+                               if (r)
+                                       ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
+                                               pool->data, strerror(errno));
+                       }
+                       break;
+       }
 
-       loc_stringpool_deallocate(pool);
        loc_unref(pool->ctx);
        free(pool);
 }
@@ -130,7 +140,7 @@ static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char*
        if (pos < pool->data)
                return -EFAULT;
 
-       if (pos > (pool->data + pool->max_length))
+       if (pos > (pool->data + pool->length))
                return -EFAULT;
 
        return pos - pool->data;
@@ -142,21 +152,15 @@ static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t o
        return offset + strlen(string) + 1;
 }
 
-static size_t loc_stringpool_space_left(struct loc_stringpool* pool) {
-       return pool->max_length - loc_stringpool_get_size(pool);
-}
-
-LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
-       if (offset >= (ssize_t)pool->max_length)
+static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
+       if (offset < 0 || offset >= pool->length)
                return NULL;
 
-       const char* string = pool->data + offset;
-
-       // If the string is empty, we have reached the end
-       if (!*string)
-               return NULL;
+       return pool->data + offset;
+}
 
-       return string;
+LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
+       return __loc_stringpool_get(pool, offset);
 }
 
 LOC_EXPORT size_t loc_stringpool_get_size(struct loc_stringpool* pool) {
@@ -168,7 +172,7 @@ static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
                return -EINVAL;
 
        off_t offset = 0;
-       while (offset < pool->max_length) {
+       while (offset < pool->length) {
                const char* string = loc_stringpool_get(pool, offset);
                if (!string)
                        break;
@@ -183,26 +187,43 @@ static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
        return -ENOENT;
 }
 
+static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) {
+       DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length);
+
+       // Save pos pointer
+       off_t pos = loc_stringpool_get_offset(pool, pool->pos);
+
+       // Reallocate data section
+       pool->data = realloc(pool->data, length);
+       if (!pool->data)
+               return -ENOMEM;
+
+       pool->length = length;
+
+       // Restore pos
+       pool->pos = __loc_stringpool_get(pool, pos);
+
+       return 0;
+}
+
 static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* string) {
        if (!string || !*string)
                return -EINVAL;
 
        DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool);
 
-       // Check if we have enough space left
-       size_t l = strlen(string) + 1;
-       if (l > loc_stringpool_space_left(pool)) {
-               DEBUG(pool->ctx, "Not enough space to append '%s'\n", string);
-               DEBUG(pool->ctx, "  Need %zu bytes but only have %zu\n", l, loc_stringpool_space_left(pool));
-               return -ENOSPC;
+       // Make sure we have enough space
+       int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1);
+       if (r) {
+               errno = r;
+               return -1;
        }
 
        off_t offset = loc_stringpool_get_offset(pool, pool->pos);
 
        // Copy string byte by byte
-       while (*string && loc_stringpool_space_left(pool) > 1) {
+       while (*string)
                *pool->pos++ = *string++;
-       }
 
        // Terminate the string
        *pool->pos++ = '\0';
@@ -223,7 +244,7 @@ LOC_EXPORT off_t loc_stringpool_add(struct loc_stringpool* pool, const char* str
 LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) {
        off_t offset = 0;
 
-       while (offset < pool->max_length) {
+       while (offset < pool->length) {
                const char* string = loc_stringpool_get(pool, offset);
                if (!string)
                        break;
@@ -234,19 +255,6 @@ LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) {
        }
 }
 
-LOC_EXPORT int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length) {
-       DEBUG(pool->ctx, "Reading string pool from %zu (%zu bytes)\n", offset, length);
-
-       pool->data = pool->pos = mmap(NULL, length, PROT_READ,
-               MAP_PRIVATE, fileno(f), offset);
-       pool->max_length = length;
-
-       if (pool->data == MAP_FAILED)
-               return -errno;
-
-       return 0;
-}
-
 LOC_EXPORT size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f) {
        size_t size = loc_stringpool_get_size(pool);
 

diff --git a/src/stringpool.h b/src/stringpool.h
index c3dffdd564e8ab89eedcc373de3b5fa8aa3caed0..29a0eb38d66389c52c32f32aed1d8848e8b16892 100644 (file)
--- a/src/stringpool.h
+++ b/src/stringpool.h
@@ -23,7 +23,10 @@
 #include <loc/libloc.h>
 
 struct loc_stringpool;
-int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length);
+int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool);
+int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
+       FILE* f, size_t length, off_t offset);
+
 struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool);
 struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool);
 
@@ -33,7 +36,6 @@ size_t loc_stringpool_get_size(struct loc_stringpool* pool);
 off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string);
 void loc_stringpool_dump(struct loc_stringpool* pool);
 
-int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length);
 size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f);
 
 #endif

diff --git a/src/test-as.c b/src/test-as.c
index e0e1b3f6949f9a9580e14951ec64fd8b23b26cb4..48c5ab727582c564271de9c8a0bba22d29e7ac01 100644 (file)
--- a/src/test-as.c
+++ b/src/test-as.c
@@ -41,7 +41,7 @@ int main(int argc, char** argv) {
        char name[256];
        for (unsigned int i = 1; i <= TEST_AS_COUNT; i++) {
                struct loc_as* as;
-               int r = loc_writer_add_as(writer, &as, i);
+               loc_writer_add_as(writer, &as, i);
 
                sprintf(name, "Test AS%u", i);
                loc_as_set_name(as, name);

diff --git a/src/test-stringpool.c b/src/test-stringpool.c
index 169554a0aa5d98e734a3d9fd8c05012105f7200c..1b219f38a99a002aa176a5cdfaa84dfe574b2d44 100644 (file)
--- a/src/test-stringpool.c
+++ b/src/test-stringpool.c
@@ -55,7 +55,7 @@ int main(int argc, char** argv) {
 
        // Create the stringpool
        struct loc_stringpool* pool;
-       err = loc_stringpool_new(ctx, &pool, 10002 * 4);
+       err = loc_stringpool_new(ctx, &pool);
        if (err < 0)
                exit(EXIT_FAILURE);
 

diff --git a/src/writer.c b/src/writer.c
index a76c3f1ed6cb15dedc6447fab5c925320b93a988..177464634d5d9fcff6cdbb37e8efa88f5017dd99 100644 (file)
--- a/src/writer.c
+++ b/src/writer.c
@@ -48,7 +48,7 @@ LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer) {
        w->ctx = loc_ref(ctx);
        w->refcount = 1;
 
-       int r = loc_stringpool_new(ctx, &w->pool, 1024 * 1024);
+       int r = loc_stringpool_new(ctx, &w->pool);
        if (r) {
                loc_writer_unref(w);
                return r;