stringpool: Use mmap to make reading the pool more efficient

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/configure.ac b/configure.ac
index ace67d39eb04b2d1ba78a3feef8804190669ba95..f8bc41e689c3717d0da588d2e089df476b350f6b 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -40,6 +40,8 @@ AS_IF([test "x$enable_debug" = "xyes"], [
 ])
 
 AC_CHECK_FUNCS([ \
+        mmap \
+        munmap \
        __secure_getenv \
        secure_getenv \
         qsort \

diff --git a/src/database.c b/src/database.c
index 9c07590f6cd3e10146e98848bfdc0375c51d8482..494e2f762f6bb3e01dce04753c38ab98bdad58d5 100644 (file)
--- a/src/database.c
+++ b/src/database.c
@@ -387,6 +387,10 @@ LOC_EXPORT int loc_database_write(struct loc_database* db, FILE* f) {
        }
        header.as_length = htonl(db->as_count * sizeof(dbas));
 
+       // Move to next page boundary
+       while (offset % LOC_DATABASE_PAGE_SIZE > 0)
+               offset += fwrite("", 1, 1, f);
+
        // Save the offset of the pool section
        DEBUG(db->ctx, "Pool starts at %jd bytes\n", offset);
        header.pool_offset = htonl(offset);

diff --git a/src/loc/format.h b/src/loc/format.h
index 130d0ff6b732416ad7cb9cda2bda356b0073209a..e6f6dbac7d14b0d8b913fd10c2550fe4c2cadc98 100644 (file)
--- a/src/loc/format.h
+++ b/src/loc/format.h
@@ -22,6 +22,8 @@
 #define LOC_DATABASE_MAGIC      "LOCDBXX"
 #define LOC_DATABASE_VERSION    0
 
+#define LOC_DATABASE_PAGE_SIZE  4096
+
 struct loc_database_magic {
        char magic[7];
 

diff --git a/src/stringpool.c b/src/stringpool.c
index 2b27dd7a2e89d85d8f149260e8eaf5aa9b653712..259cd00173620a1436931fb9112e8e0838b9e56d 100644 (file)
--- a/src/stringpool.c
+++ b/src/stringpool.c
@@ -19,6 +19,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
 
 #include <loc/libloc.h>
 #include "libloc-private.h"
@@ -34,21 +36,44 @@ struct loc_stringpool {
        ssize_t max_length;
 };
 
+static int loc_stringpool_deallocate(struct loc_stringpool* pool) {
+       if (pool->data) {
+               int r = munmap(pool->data, pool->max_length);
+               if (r) {
+                       ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
+                               pool->data, strerror(errno));
+
+                       return r;
+               }
+       }
+
+       return 0;
+}
+
 static int loc_stringpool_allocate(struct loc_stringpool* pool, size_t length) {
        // Drop old data
-       if (pool->data)
-               free(pool->data);
+       int r = loc_stringpool_deallocate(pool);
+       if (r)
+               return r;
 
        pool->max_length = length;
+
+       // Align to page size
+       while (pool->max_length % sysconf(_SC_PAGE_SIZE) > 0)
+               pool->max_length++;
+
        DEBUG(pool->ctx, "Allocating pool of %zu bytes\n", pool->max_length);
 
-       if (pool->max_length > 0) {
-               pool->data = malloc(pool->max_length);
-               if (!pool->data)
-                       return -ENOMEM;
+       // Allocate some memory
+       pool->data = pool->pos = mmap(NULL, pool->max_length,
+               PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+
+       if (pool->data == MAP_FAILED) {
+               DEBUG(pool->ctx, "%s\n", strerror(errno));
+               return -errno;
        }
 
-       pool->pos = pool->data;
+       DEBUG(pool->ctx, "Allocated pool at %p\n", pool->data);
 
        return 0;
 }
@@ -62,10 +87,12 @@ LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** p
        p->refcount = 1;
 
        // Allocate the data section
-       int r = loc_stringpool_allocate(p, max_length);
-       if (r) {
-               loc_stringpool_unref(p);
-               return r;
+       if (max_length > 0) {
+               int r = loc_stringpool_allocate(p, max_length);
+               if (r) {
+                       loc_stringpool_unref(p);
+                       return r;
+               }
        }
 
        DEBUG(p->ctx, "String pool allocated at %p\n", p);
@@ -84,11 +111,8 @@ LOC_EXPORT struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool
 static void loc_stringpool_free(struct loc_stringpool* pool) {
        DEBUG(pool->ctx, "Releasing string pool %p\n", pool);
 
+       loc_stringpool_deallocate(pool);
        loc_unref(pool->ctx);
-
-       if (pool->data)
-               free(pool->data);
-
        free(pool);
 }
 
@@ -209,26 +233,17 @@ LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) {
        }
 }
 
-LOC_EXPORT int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length) {
-       // Allocate enough space
-       int r = loc_stringpool_allocate(pool, length);
-       if (r)
-               return r;
-
-       DEBUG(pool->ctx, "Reading string pool from %zu\n", offset);
+#include <assert.h>
 
-       // Seek to the right offset
-       r = fseek(f, offset, SEEK_SET);
-       if (r)
-               return r;
+LOC_EXPORT int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length) {
+       DEBUG(pool->ctx, "Reading string pool from %zu (%zu bytes)\n", offset, length);
 
-       size_t bytes_read = fread(pool->data, sizeof(*pool->data), length, f);
-       if (bytes_read < length) {
-               ERROR(pool->ctx, "Could not read pool. Only read %zu bytes\n", bytes_read);
-               return 1;
-       }
+       pool->data = pool->pos = mmap(NULL, length, PROT_READ,
+               MAP_PRIVATE, fileno(f), offset);
+       pool->max_length = length;
 
-       DEBUG(pool->ctx, "Read pool of %zu bytes\n", length);
+       if (pool->data == MAP_FAILED)
+               return -errno;
 
        return 0;
 }