From 2601e83eca9f5d8447186256c642aef25441f07e Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Fri, 8 Dec 2017 14:49:17 +0000 Subject: [PATCH] Draft initial database format This patch adds a writer and reader for the database and already has some metadata and the ability to write and read the string pool from disk. Signed-off-by: Michael Tremer --- .gitignore | 1 + Makefile.am | 14 ++- src/.gitignore | 1 + src/database.c | 286 ++++++++++++++++++++++++++++++++++++++++++++ src/database.h | 38 ++++++ src/libloc.c | 29 +++++ src/libloc.sym | 13 ++ src/loc/libloc.h | 2 + src/stringpool.c | 68 +++++++++-- src/stringpool.h | 5 + src/test-database.c | 131 ++++++++++++++++++++ 11 files changed, 578 insertions(+), 10 deletions(-) create mode 100644 src/database.c create mode 100644 src/database.h create mode 100644 src/test-database.c diff --git a/.gitignore b/.gitignore index 6976980..acb026f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ Makefile.in /configure /libtool /stamp-h1 +/test.db diff --git a/Makefile.am b/Makefile.am index d516901..a47cdae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -44,6 +44,8 @@ lib_LTLIBRARIES = \ src_libloc_la_SOURCES =\ src/libloc-private.h \ src/libloc.c \ + src/database.c \ + src/database.h \ src/stringpool.c \ src/stringpool.h @@ -68,11 +70,13 @@ CLEANFILES += \ TESTS = \ src/test-libloc \ - src/test-stringpool + src/test-stringpool \ + src/test-database check_PROGRAMS = \ src/test-libloc \ - src/test-stringpool + src/test-stringpool \ + src/test-database src_test_libloc_SOURCES = \ src/test-libloc.c @@ -85,3 +89,9 @@ src_test_stringpool_SOURCES = \ src_test_stringpool_LDADD = \ src/libloc.la + +src_test_database_SOURCES = \ + src/test-database.c + +src_test_database_LDADD = \ + src/libloc.la diff --git a/src/.gitignore b/src/.gitignore index 9df8d54..15e9181 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -6,4 +6,5 @@ *.trs libloc.pc test-libloc +test-database test-stringpool diff --git a/src/database.c b/src/database.c new file mode 100644 index 0000000..85e3912 --- /dev/null +++ b/src/database.c @@ -0,0 +1,286 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libloc-private.h" +#include "database.h" +#include "stringpool.h" + +struct loc_database { + struct loc_ctx* ctx; + int refcount; + + unsigned int version; + off_t vendor; + off_t description; + + struct loc_stringpool* pool; +}; + +const char* LOC_DATABASE_MAGIC = "LOCDBXX"; +unsigned int LOC_DATABASE_VERSION = 0; + +struct loc_database_magic { + char magic[7]; + + // Database version information + uint8_t version; +}; + +struct loc_database_header_v0 { + // Vendor who created the database + uint32_t vendor; + + // Description of the database + uint32_t description; + + // Tells us where the pool starts + uint32_t pool_offset; + uint32_t pool_length; +}; + +LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, size_t pool_size) { + struct loc_database* db = calloc(1, sizeof(*db)); + if (!db) + return -ENOMEM; + + // Reference context + db->ctx = loc_ref(ctx); + db->refcount = 1; + + DEBUG(db->ctx, "Database allocated at %p\n", db); + + // Create string pool + int r = loc_stringpool_new(db->ctx, &db->pool, pool_size); + if (r) { + loc_database_unref(db); + return r; + } + + *database = db; + + return 0; +} + +LOC_EXPORT int loc_database_open(struct loc_ctx* ctx, struct loc_database** database, FILE* f) { + int r = loc_database_new(ctx, database, 0); + if (r) + return r; + + return loc_database_read(*database, f); +} + +LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) { + db->refcount++; + + return db; +} + +static void loc_database_free(struct loc_database* db) { + DEBUG(db->ctx, "Releasing database %p\n", db); + + loc_stringpool_unref(db->pool); + + loc_unref(db->ctx); + free(db); +} + +LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) { + if (--db->refcount > 0) + return NULL; + + loc_database_free(db); + return NULL; +} + +LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) { + return loc_stringpool_get(db->pool, db->vendor); +} + +LOC_EXPORT int loc_database_set_vendor(struct loc_database* db, const char* vendor) { + // Add the string to the string pool + off_t offset = loc_stringpool_add(db->pool, vendor); + if (offset < 0) + return offset; + + db->vendor = offset; + return 0; +} + +LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) { + return loc_stringpool_get(db->pool, db->description); +} + +LOC_EXPORT int loc_database_set_description(struct loc_database* db, const char* description) { + // Add the string to the string pool + off_t offset = loc_stringpool_add(db->pool, description); + if (offset < 0) + return offset; + + db->description = offset; + return 0; +} + +static int loc_database_read_magic(struct loc_database* db, FILE* f) { + struct loc_database_magic magic; + + // Read from file + size_t bytes_read = fread(&magic, 1, sizeof(magic), f); + + // Check if we have been able to read enough data + if (bytes_read < sizeof(magic)) { + ERROR(db->ctx, "Could not read enough data to validate magic bytes\n"); + DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic)); + return -ENOMSG; + } + + // Compare magic bytes + if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) { + DEBUG(db->ctx, "Magic value matches\n"); + + // Parse version + db->version = ntohs(magic.version); + DEBUG(db->ctx, "Database version is %u\n", db->version); + + return 0; + } + + ERROR(db->ctx, "Database format is not compatible\n"); + + // Return an error + return 1; +} + +static int loc_database_read_header_v0(struct loc_database* db, FILE* f) { + struct loc_database_header_v0 header; + + // Read from file + size_t size = fread(&header, 1, sizeof(header), f); + + if (size < sizeof(header)) { + ERROR(db->ctx, "Could not read enough data for header\n"); + return -ENOMSG; + } + + // Copy over data + db->vendor = ntohl(header.vendor); + db->description = ntohl(header.description); + + // Open pool + off_t pool_offset = ntohl(header.pool_offset); + size_t pool_length = ntohl(header.pool_length); + + int r = loc_stringpool_read(db->pool, f, pool_offset, pool_length); + if (r) + return r; + + return 0; +} + +static int loc_database_read_header(struct loc_database* db, FILE* f) { + switch (db->version) { + case 0: + return loc_database_read_header_v0(db, f); + + default: + ERROR(db->ctx, "Incompatible database version: %u\n", db->version); + return 1; + } +} + +LOC_EXPORT int loc_database_read(struct loc_database* db, FILE* f) { + int r = fseek(f, 0, SEEK_SET); + if (r) + return r; + + // Read magic bytes + r = loc_database_read_magic(db, f); + if (r) + return r; + + // Read the header + r = loc_database_read_header(db, f); + if (r) + return r; + + return 0; +} + +static void loc_database_make_magic(struct loc_database* db, struct loc_database_magic* magic) { + // Copy magic bytes + for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++) + magic->magic[i] = LOC_DATABASE_MAGIC[i]; + + // Set version + magic->version = htons(LOC_DATABASE_VERSION); +} + +LOC_EXPORT int loc_database_write(struct loc_database* db, FILE* f) { + struct loc_database_magic magic; + loc_database_make_magic(db, &magic); + + // Make the header + struct loc_database_header_v0 header; + header.vendor = htonl(db->vendor); + header.description = htonl(db->description); + + int r; + off_t offset = 0; + + // Start writing at the beginning of the file + r = fseek(f, 0, SEEK_SET); + if (r) + return r; + + // Write the magic + offset += fwrite(&magic, 1, sizeof(magic), f); + + // Skip the space we need to write the header later + r = fseek(f, sizeof(header), SEEK_CUR); + if (r) { + DEBUG(db->ctx, "Could not seek to position after header\n"); + return r; + } + offset += sizeof(header); + + // Save the offset of the pool section + DEBUG(db->ctx, "Pool starts at %jd bytes\n", offset); + header.pool_offset = htonl(offset); + + // Size of the pool + size_t pool_length = loc_stringpool_write(db->pool, f); + DEBUG(db->ctx, "Pool has a length of %zu bytes\n", pool_length); + header.pool_length = htonl(pool_length); + + // Write the header + r = fseek(f, sizeof(magic), SEEK_SET); + if (r) + return r; + + offset += fwrite(&header, 1, sizeof(header), f); + + return 0; +} diff --git a/src/database.h b/src/database.h new file mode 100644 index 0000000..72cda5a --- /dev/null +++ b/src/database.h @@ -0,0 +1,38 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. +*/ + +#ifndef LIBLOC_DATABASE_H +#define LIBLOC_DATABASE_H + +#include + +#include + +struct loc_database; +int loc_database_new(struct loc_ctx* ctx, struct loc_database** db, size_t pool_size); +int loc_database_open(struct loc_ctx* ctx, struct loc_database** database, FILE* f); +struct loc_database* loc_database_ref(struct loc_database* db); +struct loc_database* loc_database_unref(struct loc_database* db); + +const char* loc_database_get_vendor(struct loc_database* db); +int loc_database_set_vendor(struct loc_database* db, const char* vendor); +const char* loc_database_get_description(struct loc_database* db); +int loc_database_set_description(struct loc_database* db, const char* description); + +int loc_database_read(struct loc_database* db, FILE* f); +int loc_database_write(struct loc_database* db, FILE* f); + +#endif diff --git a/src/libloc.c b/src/libloc.c index fe44b3c..58e13cc 100644 --- a/src/libloc.c +++ b/src/libloc.c @@ -25,6 +25,7 @@ #include #include "libloc-private.h" +#include "database.h" struct loc_ctx { int refcount; @@ -32,6 +33,8 @@ struct loc_ctx { int priority, const char *file, int line, const char *fn, const char *format, va_list args); int log_priority; + + struct loc_database* db; }; void loc_log(struct loc_ctx* ctx, @@ -80,6 +83,8 @@ LOC_EXPORT int loc_new(struct loc_ctx** ctx) { c->log_fn = log_stderr; c->log_priority = LOG_ERR; + c->db = NULL; + const char* env = secure_getenv("LOC_LOG"); if (env) loc_set_log_priority(c, log_priority(env)); @@ -107,6 +112,10 @@ LOC_EXPORT struct loc_ctx* loc_unref(struct loc_ctx* ctx) { if (--ctx->refcount > 0) return NULL; + // Release any loaded databases + if (ctx->db) + loc_database_unref(ctx->db); + INFO(ctx, "context %p released\n", ctx); free(ctx); @@ -127,3 +136,23 @@ LOC_EXPORT int loc_get_log_priority(struct loc_ctx* ctx) { LOC_EXPORT void loc_set_log_priority(struct loc_ctx* ctx, int priority) { ctx->log_priority = priority; } + +LOC_EXPORT int loc_load(struct loc_ctx* ctx, const char* path) { + FILE* f = fopen(path, "r"); + if (!f) + return -errno; + + // Release any previously openend database + if (ctx->db) + loc_database_unref(ctx->db); + + // Open the new database + int r = loc_database_open(ctx, &ctx->db, f); + if (r) + return r; + + // Close the file + fclose(f); + + return 0; +} diff --git a/src/libloc.sym b/src/libloc.sym index faf3edd..2a527f1 100644 --- a/src/libloc.sym +++ b/src/libloc.sym @@ -1,9 +1,21 @@ LIBLOC_PRIVATE { global: + # Database + loc_database_get_description; + loc_database_get_vendor; + loc_database_new; + loc_database_open; + loc_database_ref; + loc_database_set_description; + loc_database_set_vendor; + loc_database_unref; + loc_database_write; + # String Pool loc_stringpool_add; loc_stringpool_dump; loc_stringpool_get; + loc_stringpool_get_size; loc_stringpool_new; loc_stringpool_ref; loc_stringpool_unref; @@ -17,6 +29,7 @@ global: loc_unref; loc_set_log_priority; loc_new; + loc_load; local: *; }; diff --git a/src/loc/libloc.h b/src/loc/libloc.h index 6d9f2ac..5cf31c5 100644 --- a/src/loc/libloc.h +++ b/src/loc/libloc.h @@ -35,6 +35,8 @@ void loc_set_log_fn(struct loc_ctx* ctx, int loc_get_log_priority(struct loc_ctx* ctx); void loc_set_log_priority(struct loc_ctx* ctx, int priority); +int loc_load(struct loc_ctx* ctx, const char* path); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/stringpool.c b/src/stringpool.c index 299816c..abd3831 100644 --- a/src/stringpool.c +++ b/src/stringpool.c @@ -34,21 +34,39 @@ struct loc_stringpool { ssize_t max_length; }; -LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length) { - if (max_length <= 0) - return -EINVAL; +static int loc_stringpool_allocate(struct loc_stringpool* pool, size_t length) { + // Drop old data + if (pool->data) + free(pool->data); + + pool->max_length = length; + DEBUG(pool->ctx, "Allocating pool of %zu bytes\n", pool->max_length); + + if (pool->max_length > 0) { + pool->data = malloc(pool->max_length); + if (!pool->data) + return -ENOMEM; + } + pool->pos = pool->data; + + return 0; +} + +LOC_EXPORT int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool, size_t max_length) { struct loc_stringpool* p = calloc(1, sizeof(*p)); if (!p) return -ENOMEM; p->ctx = loc_ref(ctx); p->refcount = 1; - p->max_length = max_length; // Allocate the data section - p->data = malloc(p->max_length); - p->pos = p->data; + int r = loc_stringpool_allocate(p, max_length); + if (r) { + loc_stringpool_unref(p); + return r; + } DEBUG(p->ctx, "String pool allocated at %p\n", p); DEBUG(p->ctx, " Maximum size: %zu bytes\n", p->max_length); @@ -68,7 +86,9 @@ static void loc_stringpool_free(struct loc_stringpool* pool) { loc_unref(pool->ctx); - free(pool->data); + if (pool->data) + free(pool->data); + free(pool); } @@ -98,7 +118,7 @@ static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t o } static size_t loc_stringpool_space_left(struct loc_stringpool* pool) { - return pool->max_length - loc_stringpool_get_offset(pool, pool->pos); + return pool->max_length - loc_stringpool_get_size(pool); } LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) { @@ -114,6 +134,10 @@ LOC_EXPORT const char* loc_stringpool_get(struct loc_stringpool* pool, off_t off return string; } +LOC_EXPORT size_t loc_stringpool_get_size(struct loc_stringpool* pool) { + return loc_stringpool_get_offset(pool, pool->pos); +} + static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) { if (!s || !*s) return -EINVAL; @@ -184,3 +208,31 @@ LOC_EXPORT void loc_stringpool_dump(struct loc_stringpool* pool) { offset = loc_stringpool_get_next_offset(pool, offset); } } + +LOC_EXPORT int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length) { + // Allocate enough space + int r = loc_stringpool_allocate(pool, length); + if (r) + return r; + + // Seek to the right offset + r = fseek(f, offset, SEEK_SET); + if (r) + return r; + + size_t bytes_read = fread(pool->data, sizeof(*pool->data), length, f); + if (bytes_read < length) { + ERROR(pool->ctx, "Could not read pool. Only read %zu bytes\n", bytes_read); + return 1; + } + + DEBUG(pool->ctx, "Read pool of %zu bytes\n", length); + + return 0; +} + +LOC_EXPORT size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f) { + size_t size = loc_stringpool_get_size(pool); + + return fwrite(pool->data, sizeof(*pool->data), size, f); +} diff --git a/src/stringpool.h b/src/stringpool.h index a07cd5c..dbe82c8 100644 --- a/src/stringpool.h +++ b/src/stringpool.h @@ -27,7 +27,12 @@ struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool); struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool); const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset); +size_t loc_stringpool_get_size(struct loc_stringpool* pool); + off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string); void loc_stringpool_dump(struct loc_stringpool* pool); +int loc_stringpool_read(struct loc_stringpool* pool, FILE* f, off_t offset, size_t length); +size_t loc_stringpool_write(struct loc_stringpool* pool, FILE* f); + #endif diff --git a/src/test-database.c b/src/test-database.c new file mode 100644 index 0000000..1137326 --- /dev/null +++ b/src/test-database.c @@ -0,0 +1,131 @@ +/* + libloc - A library to determine the location of someone on the Internet + + Copyright (C) 2017 IPFire Development Team + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "database.h" + +const char* DESCRIPTION = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. " + "Proin ultrices pulvinar dolor, et sollicitudin eros ultricies " + "vitae. Nam in volutpat libero. Nulla facilisi. Pellentesque " + "tempor felis enim. Integer congue nisi in maximus pretium. " + "Pellentesque et turpis elementum, luctus mi at, interdum erat. " + "Maecenas ut venenatis nunc."; + +int main(int argc, char** argv) { + int err; + + struct loc_ctx* ctx; + err = loc_new(&ctx); + if (err < 0) + exit(EXIT_FAILURE); + + // Create a database + struct loc_database* db; + err = loc_database_new(ctx, &db, 1024); + if (err < 0) + exit(EXIT_FAILURE); + + // Set the vendor + err = loc_database_set_vendor(db, "Test Vendor"); + if (err) { + fprintf(stderr, "Could not set vendor\n"); + exit(EXIT_FAILURE); + } + + // Retrieve vendor + const char* vendor1 = loc_database_get_vendor(db); + if (vendor1) { + printf("Vendor is: %s\n", vendor1); + } else { + fprintf(stderr, "Could not retrieve vendor\n"); + exit(EXIT_FAILURE); + } + + // Set a description + err = loc_database_set_description(db, DESCRIPTION); + if (err) { + fprintf(stderr, "Could not set description\n"); + exit(EXIT_FAILURE); + } + + // Retrieve description + const char* description = loc_database_get_description(db); + if (description) { + printf("Description is: %s\n", description); + } else { + fprintf(stderr, "Could not retrieve description\n"); + exit(EXIT_FAILURE); + } + + FILE* f = fopen("test.db", "w"); + if (!f) { + fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + err = loc_database_write(db, f); + if (err) { + fprintf(stderr, "Could not write database: %s\n", strerror(err)); + exit(EXIT_FAILURE); + } + + // Close the file + fclose(f); + + // Close the database + //loc_database_unref(db); + + // And open it again from disk + f = fopen("test.db", "r"); + if (!f) { + fprintf(stderr, "Could not open file for reading: %s\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + struct loc_database* db2; + err = loc_database_open(ctx, &db2, f); + if (err) { + fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + exit(EXIT_FAILURE); + } + + const char* vendor2 = loc_database_get_vendor(db2); + if (!vendor2) { + fprintf(stderr, "Could not retrieve vendor\n"); + exit(EXIT_FAILURE); + } else if (strcmp(vendor1, vendor2) != 0) { + fprintf(stderr, "Vendor doesn't match: %s != %s\n", vendor1, vendor2); + exit(EXIT_FAILURE); + } + + // Close the database + loc_database_unref(db2); + fclose(f); + + loc_unref(ctx); + + return EXIT_SUCCESS; +} -- 2.39.2