From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 12 Dec 2017 16:45:22 +0000 (+0000)
Subject: stringpool: Make them initializable right from the file
X-Git-Tag: 0.9.0~167
X-Git-Url: http://git.ipfire.org/?p=people%2Fms%2Flibloc.git;a=commitdiff_plain;h=0e974d4b392023e2f38e8ab8cfe5977b0fb439b4

stringpool: Make them initializable right from the file

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/database.c b/src/database.c
index 41b5ffd..3a4f8b8 100644
--- a/src/database.c
+++ b/src/database.c
@@ -120,11 +120,8 @@ static int loc_database_read_header_v0(struct loc_database* db) {
 	off_t pool_offset  = ntohl(header.pool_offset);
 	size_t pool_length = ntohl(header.pool_length);
 
-	int r = loc_stringpool_new(db->ctx, &db->pool, 0);
-	if (r)
-		return r;
-
-	r = loc_stringpool_read(db->pool, db->file, pool_offset, pool_length);
+	int r = loc_stringpool_open(db->ctx, &db->pool,
+		db->file, pool_length, pool_offset);
 	if (r)
 		return r;
 
diff --git a/src/stringpool.c b/src/stringpool.c
index 4a0a46b..cbb1fb7 100644
--- a/src/stringpool.c
+++ b/src/stringpool.c
@@ -27,78 +27,72 @@
 #include "libloc-private.h"
 #include "stringpool.h"
 
+enum loc_stringpool_mode {
+	STRINGPOOL_DEFAULT,
+	STRINGPOOL_MMAP,
+};
+
 struct loc_stringpool {
 	struct loc_ctx* ctx;
-
 	int refcount;
+
+	enum loc_stringpool_mode mode;
+
 	char* data;
-	char* pos;
+	ssize_t length;
 
-	ssize_t max_length;
+	char* pos;
 };
 
-static int loc_stringpool_deallocate(struct loc_stringpool* pool) {
-	if (pool->data) {
-		int r = munmap(pool->data, pool->max_length);
-		if (r) {
-			ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
-				pool->data, strerror(errno));
+static int __loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, enum loc_stringpool_mode mode) {
+	struct loc_stringpool* p = calloc(1, sizeof(*p));
+	if (!p)
+		return -ENOMEM;
+
+	p->ctx = loc_ref(ctx);
+	p->refcount = 1;
 
-			return r;
-		}
-	}
+	// Save mode
+	p->mode = mode;
+
+	*pool = p;
 
 	return 0;
 }
 
-static int loc_stringpool_allocate(struct loc_stringpool* pool, size_t length) {
-	// Drop old data
-	int r = loc_stringpool_deallocate(pool);
-	if (r)
-		return r;
+LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool) {
+	return __loc_stringpool_new(ctx, pool, STRINGPOOL_DEFAULT);
+}
 
-	pool->max_length = length;
+static int loc_stringpool_mmap(struct loc_stringpool* pool, FILE* f, size_t length, off_t offset) {
+	if (pool->mode != STRINGPOOL_MMAP)
+		return -EINVAL;
 
-	// Align to page size
-	while (pool->max_length % sysconf(_SC_PAGE_SIZE) > 0)
-		pool->max_length++;
+	DEBUG(pool->ctx, "Reading string pool starting from %zu (%zu bytes)\n", offset, length);
 
-	DEBUG(pool->ctx, "Allocating pool of %zu bytes\n", pool->max_length);
+	// Map file content into memory
+	pool->data = pool->pos = mmap(NULL, length, PROT_READ,
+		MAP_PRIVATE, fileno(f), offset);
 
-	// Allocate some memory
-	pool->data = pool->pos = mmap(NULL, pool->max_length,
-		PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+	// Store size of section
+	pool->length = length;
 
-	if (pool->data == MAP_FAILED) {
-		DEBUG(pool->ctx, "%s\n", strerror(errno));
+	if (pool->data == MAP_FAILED)
 		return -errno;
-	}
-
-	DEBUG(pool->ctx, "Allocated pool at %p\n", pool->data);
 
 	return 0;
 }
 
-LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length) {
-	struct loc_stringpool* p = calloc(1, sizeof(*p));
-	if (!p)
-		return -ENOMEM;
-
-	p->ctx = loc_ref(ctx);
-	p->refcount = 1;
-
-	// Allocate the data section
-	if (max_length > 0) {
-		int r = loc_stringpool_allocate(p, max_length);
-		if (r) {
-			loc_stringpool_unref(p);
-			return r;
-		}
-	}
+LOC_EXPORT int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
+		FILE* f, size_t length, off_t offset) {
+	int r = __loc_stringpool_new(ctx, pool, STRINGPOOL_MMAP);
+	if (r)
+		return r;
 
-	DEBUG(p->ctx, "String pool allocated at %p\n", p);
-	DEBUG(p->ctx, "  Maximum size: %zu bytes\n", p->max_length);
-	*pool = p;
+	// Map data into memory
+	r = loc_stringpool_mmap(*pool, f, length, offset);
+	if (r)
+		return r;
 
 	return 0;
 }
@@ -111,8 +105,24 @@ LOC_EXPORT struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool
 
 static void loc_stringpool_free(struct loc_stringpool* pool) {
 	DEBUG(pool->ctx, "Releasing string pool %p\n", pool);
+	int r;
+
+	switch (pool->mode) {
+		case STRINGPOOL_DEFAULT:
+			if (pool->data)
+				free(pool->data);
+			break;
+
+		case STRINGPOOL_MMAP:
+			if (pool->data) {
+				r = munmap(pool->data, pool->length);
+				if (r)
+					ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
+						pool->data, strerror(errno));
+			}
+			break;
+	}
 
-	loc_stringpool_deallocate(pool);
 	loc_unref(pool->ctx);
 	free(pool);
 }
@@ -130,7 +140,7 @@ static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char*
 	if (pos < pool->data)
 		return -EFAULT;
 
-	if (pos > (pool->data + pool->max_length))
+	if (pos > (pool->data + pool->length))
 		return -EFAULT;
 
 	return pos - pool->data;
@@ -142,21 +152,15 @@ static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t o
 	return offset + strlen(string) + 1;
 }
 
-static size_t loc_stringpool_space_left(struct loc_stringpool* pool) {
-	return pool->max_length - loc_stringpool_get_size(pool);
-}
-
-LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
-	if (offset >= (ssize_t)pool->max_length)
+static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
+	if (offset < 0 || offset >= pool->length)
 		return NULL;
 
-	const char* string = pool->data + offset;
-
-	// If the string is empty, we have reached the end
-	if (!*string)
-		return NULL;
+	return pool->data + offset;
+}
 
-	return string;
+LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
+	return __loc_stringpool_get(pool, offset);
 }
 
 LOC_EXPORT size_t loc_stringpool_get_size(struct loc_stringpool* pool) {
@@ -168,7 +172,7 @@ static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
 		return -EINVAL;
 
 	off_t offset = 0;
-	while (offset < pool->max_length) {
+	while (offset < pool->length) {
 		const char* string = loc_stringpool_get(pool, offset);
 		if (!string)
 			break;
@@ -183,26 +187,43 @@ static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
 	return -ENOENT;
 }
 
+static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) {
+	DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length);
+
+	// Save pos pointer
+	off_t pos = loc_stringpool_get_offset(pool, pool->pos);
+
+	// Reallocate data section
+	pool->data = realloc(pool->data, length);
+	if (!pool->data)
+		return -ENOMEM;
+
+	pool->length = length;
+
+	// Restore pos
+	pool->pos = __loc_stringpool_get(pool, pos);
+
+	return 0;
+}
+
 static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* string) {
 	if (!string || !*string)
 		return -EINVAL;
 
 	DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool);
 
-	// Check if we have enough space left
-	size_t l = strlen(string) + 1;
-	if (l > loc_stringpool_space_left(pool)) {
-		DEBUG(pool->ctx, "Not enough space to append '%s'\n", string);
-		DEBUG(pool->ctx, "  Need %zu bytes but only have %zu\n", l, loc_stringpool_space_left(pool));
-		return -ENOSPC;
+	// Make sure we have enough space
+	int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1);
+	if (r) {
+		errno = r;
+		return -1;
 	}
 
 	off_t offset = loc_stringpool_get_offset(pool, pool->pos);
 
 	// Copy string byte by byte
-	while (*string && loc_stringpool_space_left(pool) > 1) {
+	while (*string)
 		*pool->pos++ = *string++;
-	}
 
 	// Terminate the string
 	*pool->pos++ = '\0';
@@ -223,7 +244,7 @@ LOC_EXPORT off_t loc_stringpool_add(struct loc_stringpool* pool, const char* str
 LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) {
 	off_t offset = 0;
 
-	while (offset < pool->max_length) {
+	while (offset < pool->length) {
 		const char* string = loc_stringpool_get(pool, offset);
 		if (!string)
 			break;
@@ -234,19 +255,6 @@ LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) {
 	}
 }
 
-LOC_EXPORT int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length) {
-	DEBUG(pool->ctx, "Reading string pool from %zu (%zu bytes)\n", offset, length);
-
-	pool->data = pool->pos = mmap(NULL, length, PROT_READ,
-		MAP_PRIVATE, fileno(f), offset);
-	pool->max_length = length;
-
-	if (pool->data == MAP_FAILED)
-		return -errno;
-
-	return 0;
-}
-
 LOC_EXPORT size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f) {
 	size_t size = loc_stringpool_get_size(pool);
 
diff --git a/src/stringpool.h b/src/stringpool.h
index c3dffdd..29a0eb3 100644
--- a/src/stringpool.h
+++ b/src/stringpool.h
@@ -23,7 +23,10 @@
 #include <loc/libloc.h>
 
 struct loc_stringpool;
-int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length);
+int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool);
+int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
+	FILE* f, size_t length, off_t offset);
+
 struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool);
 struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool);
 
@@ -33,7 +36,6 @@ size_t loc_stringpool_get_size(struct loc_stringpool* pool);
 off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string);
 void loc_stringpool_dump(struct loc_stringpool* pool);
 
-int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length);
 size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f);
 
 #endif
diff --git a/src/test-as.c b/src/test-as.c
index e0e1b3f..48c5ab7 100644
--- a/src/test-as.c
+++ b/src/test-as.c
@@ -41,7 +41,7 @@ int main(int argc, char** argv) {
 	char name[256];
 	for (unsigned int i = 1; i <= TEST_AS_COUNT; i++) {
 		struct loc_as* as;
-		int r = loc_writer_add_as(writer, &as, i);
+		loc_writer_add_as(writer, &as, i);
 
 		sprintf(name, "Test AS%u", i);
 		loc_as_set_name(as, name);
diff --git a/src/test-stringpool.c b/src/test-stringpool.c
index 169554a..1b219f3 100644
--- a/src/test-stringpool.c
+++ b/src/test-stringpool.c
@@ -55,7 +55,7 @@ int main(int argc, char** argv) {
 
 	// Create the stringpool
 	struct loc_stringpool* pool;
-	err = loc_stringpool_new(ctx, &pool, 10002 * 4);
+	err = loc_stringpool_new(ctx, &pool);
 	if (err < 0)
 		exit(EXIT_FAILURE);
 
diff --git a/src/writer.c b/src/writer.c
index a76c3f1..1774646 100644
--- a/src/writer.c
+++ b/src/writer.c
@@ -48,7 +48,7 @@ LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer) {
 	w->ctx = loc_ref(ctx);
 	w->refcount = 1;
 
-	int r = loc_stringpool_new(ctx, &w->pool, 1024 * 1024);
+	int r = loc_stringpool_new(ctx, &w->pool);
 	if (r) {
 		loc_writer_unref(w);
 		return r;