From: Jochen Sprickerhof Date: Mon, 26 Sep 2022 16:01:35 +0000 (+0200) Subject: New upstream version 0.9.15 X-Git-Tag: upstream/0.9.15^0 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=aa9346d804d6d10dbf4627f2aa0b076366ed9f31;p=location%2Fdebian%2Flibloc.git New upstream version 0.9.15 --- diff --git a/.gitignore b/.gitignore index 20bc895..41e3075 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ Makefile.in /*.db.xz /libtool /stamp-h1 +/src/cron/location-update /src/scripts/location /src/scripts/location-importer /src/systemd/location-update.service diff --git a/Makefile.am b/Makefile.am index eef75c2..81bd992 100644 --- a/Makefile.am +++ b/Makefile.am @@ -51,6 +51,7 @@ SED_PROCESS = \ -e 's,@databasedir\@,$(databasedir),g' \ < $< > $@ || rm $@ +cron_dailydir = $(sysconfdir)/cron.daily databasedir = $(localstatedir)/lib/location pkgconfigdir = $(libdir)/pkgconfig @@ -281,6 +282,7 @@ CLEANFILES += \ # ------------------------------------------------------------------------------ +# Use systemd timers if available if HAVE_SYSTEMD systemdsystemunit_DATA = \ src/systemd/location-update.service \ @@ -291,9 +293,18 @@ CLEANFILES += \ INSTALL_DIRS += \ $(systemdsystemunitdir) + +# Otherwise fall back to cron +else +cron_daily_SCRIPTS = \ + src/cron/location-update + +CLEANFILES += \ + $(cron_daily_DATA) endif EXTRA_DIST += \ + src/cron/location-update.in \ src/systemd/location-update.service.in \ src/systemd/location-update.timer.in @@ -322,7 +333,8 @@ TESTS = \ $(dist_check_SCRIPTS) EXTRA_DIST += \ - tests/data/location-2022-03-30.db + tests/data/location-2022-03-30.db \ + tests/data/signing-key.pem CLEANFILES += \ testdata.db @@ -333,6 +345,7 @@ testdata.db: examples/python/create-database.py $(PYTHON) $< $@ dist_check_SCRIPTS = \ + tests/python/test-database.py \ tests/python/test-export.py check_PROGRAMS = \ diff --git a/configure.ac b/configure.ac index b2db205..9fad1b9 100644 --- a/configure.ac +++ b/configure.ac @@ -1,6 +1,6 @@ AC_PREREQ(2.60) AC_INIT([libloc], - [0.9.14], + [0.9.15], [location@lists.ipfire.org], [libloc], [https://location.ipfire.org/]) @@ -18,7 +18,7 @@ AM_INIT_AUTOMAKE([ dist-xz subdir-objects ]) -AC_PROG_CC_STDC +AC_PROG_CC AC_USE_SYSTEM_EXTENSIONS AC_SYS_LARGEFILE AC_CONFIG_MACRO_DIR([m4]) @@ -81,6 +81,7 @@ AC_CHECK_FUNCS([ \ htobe16 \ htobe32 \ htobe64 \ + madvise \ mmap \ munmap \ res_query \ @@ -119,6 +120,19 @@ CC_CHECK_FLAGS_APPEND([my_LDFLAGS], [LDFLAGS], [-fno-semantic-interposition]) # ------------------------------------------------------------------------------ +AC_ARG_WITH([database-path], + AS_HELP_STRING([--with-database-path], [The default database path]), + [], [with_database_path=/var/lib/${PACKAGE_NAME}/database.db] +) + +if test -z "${with_database_path}"; then + AC_MSG_ERROR([The default database path is empty]) +fi + +AC_DEFINE_UNQUOTED([LIBLOC_DEFAULT_DATABASE_PATH], ["${with_database_path}"], + [The default path for the database]) +AC_SUBST([DEFAULT_DATABASE_PATH], [${with_database_path}]) + AC_ARG_WITH([systemd], AS_HELP_STRING([--with-systemd], [Enable systemd support.]) ) @@ -204,6 +218,7 @@ AC_MSG_RESULT([ cflags: ${CFLAGS} ldflags: ${LDFLAGS} + database path: ${with_database_path} debug: ${enable_debug} systemd support: ${have_systemd} diff --git a/debian/changelog b/debian/changelog index 248958e..87699a3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,59 @@ +libloc (0.9.15-1) unstable; urgency=medium + + [ Peter Müller ] + * Non-maintainer upload. + * location-importer.in: Fix dangling variable + + [ Michael Tremer ] + * Replace strerror(errno) with %m in format string throughout + * Don't abuse errno as return code + * country: Refactor storing country code and continent code + * *_unref: Always expect a valid pointer + * cron: Add a cronjob if systemd is not available + * Check return value of fread() when reading header + * configure: Replace obsolete AC_PROG_CC_STDC macro + * writer: Check if stringpool has been initialized before free + * database: Use MAP_PRIVATE with mmap() + * database: Do not try to unmap failed mappings + * database: Log any errors when mmap() fails + * database: Increase page size to 64k + * python: Correctly raise any errors when opening the database + * database: Improve error reporting when the magic cannot be read + * python: Fix errors for Database.lookup() + * stringpool: Implement mmap as optional + * database: Fall back when mmap() isn't available + * tests: Add some simple database tests + * stringpool: Drop function to find next offset + * database: country: Return better error codes + * python: Export DatabaseEnumerator type + * tests: database: Expand test coverage + * database: Refactor error handling on create + * database: Break opening procedure into smaller parts + * database: Refactor checking magic + * database: Check if this version of libloc supports the database + format + * database: Map the entire database into memory as a whole + * database: Read header from mapped data + * hexdump: Don't try to dump any empty memory + * database: Read all data from the large mmap() + * database: Call madvise() to tell the kernel that we will randomly + access the data + * database: Encourage the compiler to inline some functions + * database: Drop unused offset variable in objects + * database: Drop debug line + * database: Initialize r on create + * tests: Add signing key to verify signatures + * configure: Check for madvise + * Fix compilation on MacOS X + * country: Drop unused CC_LEN + * tests: country: Don't crash when a country could not be found + * Revert "database: Increase page size to 64k" + * writer: Flush everything to disk after writing finishes + * configure: Make the default database path configurable + * python: Add new open() interface to easily open a database + + -- Michael Tremer Mon, 26 Sep 2022 15:36:44 +0000 + libloc (0.9.14-1) unstable; urgency=medium [ Michael Tremer ] diff --git a/src/address.c b/src/address.c index dd32dbc..d338243 100644 --- a/src/address.c +++ b/src/address.c @@ -22,6 +22,7 @@ #include #include +#include #define LOC_ADDRESS_BUFFERS 6 #define LOC_ADDRESS_BUFFER_LENGTH INET6_ADDRSTRLEN diff --git a/src/as-list.c b/src/as-list.c index 7154fd2..50805e7 100644 --- a/src/as-list.c +++ b/src/as-list.c @@ -14,11 +14,11 @@ Lesser General Public License for more details. */ -#include #include #include #include +#include #include struct loc_as_list { @@ -42,7 +42,7 @@ static int loc_as_list_grow(struct loc_as_list* list) { struct loc_as** elements = reallocarray(list->elements, list->elements_size + size, sizeof(*list->elements)); if (!elements) - return -errno; + return 1; list->elements = elements; list->elements_size += size; @@ -54,7 +54,7 @@ LOC_EXPORT int loc_as_list_new(struct loc_ctx* ctx, struct loc_as_list** list) { struct loc_as_list* l = calloc(1, sizeof(*l)); if (!l) - return -ENOMEM; + return 1; l->ctx = loc_ref(ctx); l->refcount = 1; diff --git a/src/as.c b/src/as.c index 2cbd5c8..97c0a56 100644 --- a/src/as.c +++ b/src/as.c @@ -42,7 +42,7 @@ struct loc_as { LOC_EXPORT int loc_as_new(struct loc_ctx* ctx, struct loc_as** as, uint32_t number) { struct loc_as* a = calloc(1, sizeof(*a)); if (!a) - return -ENOMEM; + return 1; a->ctx = loc_ref(ctx); a->refcount = 1; diff --git a/src/country-list.c b/src/country-list.c index b2aea8b..536fd9e 100644 --- a/src/country-list.c +++ b/src/country-list.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -42,7 +43,7 @@ static int loc_country_list_grow(struct loc_country_list* list) { struct loc_country** elements = reallocarray(list->elements, list->elements_size + size, sizeof(*list->elements)); if (!elements) - return -errno; + return 1; list->elements = elements; list->elements_size += size; @@ -81,9 +82,6 @@ static void loc_country_list_free(struct loc_country_list* list) { } LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) { - if (!list) - return NULL; - if (--list->refcount > 0) return list; @@ -159,8 +157,8 @@ LOC_EXPORT int loc_country_list_contains_code( // Ignore invalid country codes which would never match if (errno == EINVAL) return 0; - else - return r; + + return r; } r = loc_country_list_contains(list, country); diff --git a/src/country.c b/src/country.c index 0f13319..309cee1 100644 --- a/src/country.c +++ b/src/country.c @@ -39,8 +39,10 @@ struct loc_country { struct loc_ctx* ctx; int refcount; - char* code; - char* continent_code; + // Store the country code in a 3 byte buffer. Two bytes for the code, and NULL so + // that we can use strcmp() and return a pointer. + char code[3]; + char continent_code[3]; char* name; }; @@ -54,12 +56,13 @@ LOC_EXPORT int loc_country_new(struct loc_ctx* ctx, struct loc_country** country struct loc_country* c = calloc(1, sizeof(*c)); if (!c) - return -ENOMEM; + return 1; c->ctx = loc_ref(ctx); c->refcount = 1; - c->code = strdup(country_code); + // Set the country code + loc_country_code_copy(c->code, country_code); DEBUG(c->ctx, "Country %s allocated at %p\n", c->code, c); *country = c; @@ -76,12 +79,6 @@ LOC_EXPORT struct loc_country* loc_country_ref(struct loc_country* country) { static void loc_country_free(struct loc_country* country) { DEBUG(country->ctx, "Releasing country %s %p\n", country->code, country); - if (country->code) - free(country->code); - - if (country->continent_code) - free(country->continent_code); - if (country->name) free(country->name); @@ -94,7 +91,6 @@ LOC_EXPORT struct loc_country* loc_country_unref(struct loc_country* country) { return NULL; loc_country_free(country); - return NULL; } @@ -107,13 +103,14 @@ LOC_EXPORT const char* loc_country_get_continent_code(struct loc_country* countr } LOC_EXPORT int loc_country_set_continent_code(struct loc_country* country, const char* continent_code) { - // XXX validate input - - // Free previous value - if (country->continent_code) - free(country->continent_code); + // Check for valid input + if (!continent_code || strlen(continent_code) != 2) { + errno = EINVAL; + return 1; + } - country->continent_code = strdup(continent_code); + // Store the code + loc_country_code_copy(country->continent_code, continent_code); return 0; } @@ -126,37 +123,36 @@ LOC_EXPORT int loc_country_set_name(struct loc_country* country, const char* nam if (country->name) free(country->name); - if (name) + if (name) { country->name = strdup(name); + // Report error if we could not copy the string + if (!country->name) + return 1; + } + return 0; } LOC_EXPORT int loc_country_cmp(struct loc_country* country1, struct loc_country* country2) { - return strcmp(country1->code, country2->code); + return strncmp(country1->code, country2->code, 2); } int loc_country_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool, struct loc_country** country, const struct loc_database_country_v1* dbobj) { - char buffer[3]; + char buffer[3] = "XX"; // Read country code loc_country_code_copy(buffer, dbobj->code); - // Terminate buffer - buffer[2] = '\0'; - // Create a new country object int r = loc_country_new(ctx, country, buffer); if (r) return r; - // Continent Code - loc_country_code_copy(buffer, dbobj->continent_code); - - r = loc_country_set_continent_code(*country, buffer); - if (r) - goto FAIL; + // Copy continent code + if (*dbobj->continent_code) + loc_country_code_copy((*country)->continent_code, dbobj->continent_code); // Set name const char* name = loc_stringpool_get(pool, be32toh(dbobj->name)); @@ -175,20 +171,20 @@ FAIL: int loc_country_to_database_v1(struct loc_country* country, struct loc_stringpool* pool, struct loc_database_country_v1* dbobj) { + off_t name = 0; + // Add country code - for (unsigned int i = 0; i < 2; i++) { - dbobj->code[i] = country->code[i] ? country->code[i] : '\0'; - } + if (*country->code) + loc_country_code_copy(dbobj->code, country->code); // Add continent code - if (country->continent_code) { - for (unsigned int i = 0; i < 2; i++) { - dbobj->continent_code[i] = country->continent_code[i] ? country->continent_code[i] : '\0'; - } - } + if (*country->continent_code) + loc_country_code_copy(dbobj->continent_code, country->continent_code); // Save the name string in the string pool - off_t name = loc_stringpool_add(pool, country->name ? country->name : ""); + if (country->name) + name = loc_stringpool_add(pool, country->name); + dbobj->name = htobe32(name); return 0; @@ -227,7 +223,7 @@ LOC_EXPORT int loc_country_special_code_to_flag(const char* cc) { // Return flags for any known special country for (const struct loc_special_country* country = loc_special_countries; country->flags; country++) { - if (strcmp(country->code, cc) == 0) + if (strncmp(country->code, cc, 2) == 0) return country->flags; } diff --git a/src/cron/location-update.in b/src/cron/location-update.in new file mode 100644 index 0000000..232de10 --- /dev/null +++ b/src/cron/location-update.in @@ -0,0 +1,21 @@ +#!/bin/bash +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2022 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +# Call the location database updater +exec @bindir@/location update diff --git a/src/database.c b/src/database.c index b57407e..be60aa4 100644 --- a/src/database.c +++ b/src/database.c @@ -50,6 +50,17 @@ #include #include +struct loc_database_objects { + char* data; + size_t length; + size_t count; +}; + +struct loc_database_signature { + const char* data; + size_t length; +}; + struct loc_database { struct loc_ctx* ctx; int refcount; @@ -63,28 +74,26 @@ struct loc_database { off_t license; // Signatures - char* signature1; - size_t signature1_length; - char* signature2; - size_t signature2_length; + struct loc_database_signature signature1; + struct loc_database_signature signature2; + + // Data mapped into memory + char* data; + off_t length; + + struct loc_stringpool* pool; // ASes in the database - struct loc_database_as_v1* as_v1; - size_t as_count; + struct loc_database_objects as_objects; // Network tree - struct loc_database_network_node_v1* network_nodes_v1; - size_t network_nodes_count; + struct loc_database_objects network_node_objects; // Networks - struct loc_database_network_v1* networks_v1; - size_t networks_count; + struct loc_database_objects network_objects; // Countries - struct loc_database_country_v1* countries_v1; - size_t countries_count; - - struct loc_stringpool* pool; + struct loc_database_objects country_objects; }; #define MAX_STACK_DEPTH 256 @@ -132,7 +141,63 @@ struct loc_database_enumerator { struct in6_addr gap4_start; }; -static int loc_database_read_magic(struct loc_database* db) { +/* + Checks if it is safe to read the buffer of size length starting at p. +*/ +#define loc_database_check_boundaries(db, p) \ + __loc_database_check_boundaries(db, (const char*)p, sizeof(*p)) + +static inline int __loc_database_check_boundaries(struct loc_database* db, + const char* p, const size_t length) { + size_t offset = p - db->data; + + // Return if everything is within the boundary + if (offset <= db->length - length) + return 1; + + DEBUG(db->ctx, "Database read check failed at %p for %zu byte(s)\n", p, length); + DEBUG(db->ctx, " p = %p (offset = %jd, length = %zu)\n", p, offset, length); + DEBUG(db->ctx, " data = %p (length = %zu)\n", db->data, db->length); + DEBUG(db->ctx, " end = %p\n", db->data + db->length); + DEBUG(db->ctx, " overflow of %zu byte(s)\n", offset + length - db->length); + + // Otherwise raise EFAULT + errno = EFAULT; + return 0; +} + +/* + Returns a pointer to the n-th object +*/ +static inline char* loc_database_object(struct loc_database* db, + const struct loc_database_objects* objects, const size_t length, const off_t n) { + // Calculate offset + const off_t offset = n * length; + + // Return a pointer to where the object lies + char* object = objects->data + offset; + + // Check if the object is part of the memory + if (!__loc_database_check_boundaries(db, object, length)) + return NULL; + + return object; +} + +static int loc_database_version_supported(struct loc_database* db, uint8_t version) { + switch (version) { + // Supported versions + case LOC_DATABASE_VERSION_1: + return 1; + + default: + ERROR(db->ctx, "Database version %d is not supported\n", version); + errno = ENOTSUP; + return 0; + } +} + +static int loc_database_check_magic(struct loc_database* db) { struct loc_database_magic magic; // Read from file @@ -142,200 +207,177 @@ static int loc_database_read_magic(struct loc_database* db) { if (bytes_read < sizeof(magic)) { ERROR(db->ctx, "Could not read enough data to validate magic bytes\n"); DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic)); - return -ENOMSG; + goto ERROR; } // Compare magic bytes - if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) { + if (memcmp(magic.magic, LOC_DATABASE_MAGIC, sizeof(magic.magic)) == 0) { DEBUG(db->ctx, "Magic value matches\n"); + // Do we support this version? + if (!loc_database_version_supported(db, magic.version)) + return 1; + // Parse version db->version = magic.version; return 0; } +ERROR: ERROR(db->ctx, "Unrecognized file type\n"); + errno = ENOMSG; // Return an error return 1; } -static int loc_database_read_as_section_v1(struct loc_database* db, - const struct loc_database_header_v1* header) { - off_t as_offset = be32toh(header->as_offset); - size_t as_length = be32toh(header->as_length); - - DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", (intmax_t)as_offset, as_length); +/* + Maps the entire database into memory +*/ +static int loc_database_mmap(struct loc_database* db) { + int r; - if (as_length > 0) { - db->as_v1 = mmap(NULL, as_length, PROT_READ, - MAP_SHARED, fileno(db->f), as_offset); + // Get file descriptor + int fd = fileno(db->f); - if (db->as_v1 == MAP_FAILED) - return -errno; + // Determine the length of the database + db->length = lseek(fd, 0, SEEK_END); + if (db->length < 0) { + ERROR(db->ctx, "Could not determine the length of the database: %m\n"); + return 1; } - db->as_count = as_length / sizeof(*db->as_v1); - - INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count); - - return 0; -} - -static int loc_database_read_network_nodes_section_v1(struct loc_database* db, - const struct loc_database_header_v1* header) { - off_t network_nodes_offset = be32toh(header->network_tree_offset); - size_t network_nodes_length = be32toh(header->network_tree_length); - - DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n", - (intmax_t)network_nodes_offset, network_nodes_length); - - if (network_nodes_length > 0) { - db->network_nodes_v1 = mmap(NULL, network_nodes_length, PROT_READ, - MAP_SHARED, fileno(db->f), network_nodes_offset); + rewind(db->f); - if (db->network_nodes_v1 == MAP_FAILED) - return -errno; + // Map all data + db->data = mmap(NULL, db->length, PROT_READ, MAP_SHARED, fd, 0); + if (db->data == MAP_FAILED) { + ERROR(db->ctx, "Could not map the database: %m\n"); + db->data = NULL; + return 1; } - db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v1); - - INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count); + DEBUG(db->ctx, "Mapped database of %zu byte(s) at %p\n", db->length, db->data); - return 0; -} - -static int loc_database_read_networks_section_v1(struct loc_database* db, - const struct loc_database_header_v1* header) { - off_t networks_offset = be32toh(header->network_data_offset); - size_t networks_length = be32toh(header->network_data_length); - - DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n", - (intmax_t)networks_offset, networks_length); - - if (networks_length > 0) { - db->networks_v1 = mmap(NULL, networks_length, PROT_READ, - MAP_SHARED, fileno(db->f), networks_offset); - - if (db->networks_v1 == MAP_FAILED) - return -errno; + // Tell the system that we expect to read data randomly + r = madvise(db->data, db->length, MADV_RANDOM); + if (r) { + ERROR(db->ctx, "madvise() failed: %m\n"); + return r; } - db->networks_count = networks_length / sizeof(*db->networks_v1); - - INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count); - return 0; } -static int loc_database_read_countries_section_v1(struct loc_database* db, - const struct loc_database_header_v1* header) { - off_t countries_offset = be32toh(header->countries_offset); - size_t countries_length = be32toh(header->countries_length); - - DEBUG(db->ctx, "Reading countries section from %jd (%zu bytes)\n", - (intmax_t)countries_offset, countries_length); - - if (countries_length > 0) { - db->countries_v1 = mmap(NULL, countries_length, PROT_READ, - MAP_SHARED, fileno(db->f), countries_offset); - - if (db->countries_v1 == MAP_FAILED) - return -errno; - } - - db->countries_count = countries_length / sizeof(*db->countries_v1); - - INFO(db->ctx, "Read %zu countries from the database\n", - db->countries_count); +/* + Maps arbitrary objects from the database into memory. +*/ +static int loc_database_map_objects(struct loc_database* db, struct loc_database_objects* objects, + const size_t size, const off_t offset, const size_t length) { + // Store parameters + objects->data = db->data + offset; + objects->length = length; + objects->count = objects->length / size; return 0; } static int loc_database_read_signature(struct loc_database* db, - char** dst, char* src, size_t length) { + struct loc_database_signature* signature, const char* data, const size_t length) { // Check for a plausible signature length if (length > LOC_SIGNATURE_MAX_LENGTH) { ERROR(db->ctx, "Signature too long: %zu\n", length); - return -EINVAL; + errno = EINVAL; + return 1; } - DEBUG(db->ctx, "Reading signature of %zu bytes\n", length); + // Store data & length + signature->data = data; + signature->length = length; - // Allocate space - *dst = malloc(length); - if (!*dst) - return -ENOMEM; + DEBUG(db->ctx, "Read signature of %zu byte(s) at %p\n", + signature->length, signature->data); - // Copy payload - memcpy(*dst, src, length); + hexdump(db->ctx, signature->data, signature->length); return 0; } static int loc_database_read_header_v1(struct loc_database* db) { - struct loc_database_header_v1 header; + const struct loc_database_header_v1* header = + (const struct loc_database_header_v1*)(db->data + LOC_DATABASE_MAGIC_SIZE); int r; - // Read from file - size_t size = fread(&header, 1, sizeof(header), db->f); + DEBUG(db->ctx, "Reading header at %p\n", header); - if (size < sizeof(header)) { + // Check if we can read the header + if (!loc_database_check_boundaries(db, header)) { ERROR(db->ctx, "Could not read enough data for header\n"); - return -ENOMSG; + return 1; } - // Copy over data - db->created_at = be64toh(header.created_at); - db->vendor = be32toh(header.vendor); - db->description = be32toh(header.description); - db->license = be32toh(header.license); + // Dump the entire header + hexdump(db->ctx, header, sizeof(*header)); - db->signature1_length = be16toh(header.signature1_length); - db->signature2_length = be16toh(header.signature2_length); + // Copy over data + db->created_at = be64toh(header->created_at); + db->vendor = be32toh(header->vendor); + db->description = be32toh(header->description); + db->license = be32toh(header->license); // Read signatures - if (db->signature1_length) { - r = loc_database_read_signature(db, &db->signature1, - header.signature1, db->signature1_length); - if (r) - return r; - } + r = loc_database_read_signature(db, &db->signature1, + header->signature1, be16toh(header->signature1_length)); + if (r) + return r; - if (db->signature2_length) { - r = loc_database_read_signature(db, &db->signature2, - header.signature2, db->signature2_length); - if (r) - return r; - } + r = loc_database_read_signature(db, &db->signature2, + header->signature2, be16toh(header->signature2_length)); + if (r) + return r; - // Open pool - off_t pool_offset = be32toh(header.pool_offset); - size_t pool_length = be32toh(header.pool_length); + const char* stringpool_start = db->data + be32toh(header->pool_offset); + size_t stringpool_length = be32toh(header->pool_length); - r = loc_stringpool_open(db->ctx, &db->pool, - db->f, pool_length, pool_offset); + // Check if the stringpool is part of the mapped area + if (!__loc_database_check_boundaries(db, stringpool_start, stringpool_length)) + return 1; + + // Open the stringpool + r = loc_stringpool_open(db->ctx, &db->pool, stringpool_start, stringpool_length); if (r) return r; - // AS section - r = loc_database_read_as_section_v1(db, &header); + // Map AS objects + r = loc_database_map_objects(db, &db->as_objects, + sizeof(struct loc_database_as_v1), + be32toh(header->as_offset), + be32toh(header->as_length)); if (r) return r; - // Network Nodes - r = loc_database_read_network_nodes_section_v1(db, &header); + // Map Network Nodes + r = loc_database_map_objects(db, &db->network_node_objects, + sizeof(struct loc_database_network_node_v1), + be32toh(header->network_tree_offset), + be32toh(header->network_tree_length)); if (r) return r; - // Networks - r = loc_database_read_networks_section_v1(db, &header); + // Map Networks + r = loc_database_map_objects(db, &db->network_objects, + sizeof(struct loc_database_network_v1), + be32toh(header->network_data_offset), + be32toh(header->network_data_length)); if (r) return r; - // countries - r = loc_database_read_countries_section_v1(db, &header); + // Map countries + r = loc_database_map_objects(db, &db->country_objects, + sizeof(struct loc_database_country_v1), + be32toh(header->countries_offset), + be32toh(header->countries_length)); if (r) return r; @@ -355,30 +397,47 @@ static int loc_database_read_header(struct loc_database* db) { } } -static int loc_database_read(struct loc_database* db, FILE* f) { - clock_t start = clock(); - +static int loc_database_clone_handle(struct loc_database* db, FILE* f) { + // Fetch the FD of the original handle int fd = fileno(f); // Clone file descriptor fd = dup(fd); if (!fd) { ERROR(db->ctx, "Could not duplicate file descriptor\n"); - return -1; + return 1; } // Reopen the file so that we can keep our own file handle db->f = fdopen(fd, "r"); if (!db->f) { ERROR(db->ctx, "Could not re-open database file\n"); - return -1; + return 1; } // Rewind to the start of the file rewind(db->f); + return 0; +} + +static int loc_database_open(struct loc_database* db, FILE* f) { + int r; + + clock_t start = clock(); + + // Clone the file handle + r = loc_database_clone_handle(db, f); + if (r) + return r; + // Read magic bytes - int r = loc_database_read_magic(db); + r = loc_database_check_magic(db); + if (r) + return r; + + // Map the database into memory + r = loc_database_mmap(db); if (r) return r; @@ -395,14 +454,44 @@ static int loc_database_read(struct loc_database* db, FILE* f) { return 0; } +static void loc_database_free(struct loc_database* db) { + int r; + + DEBUG(db->ctx, "Releasing database %p\n", db); + + // Unmap the entire database + if (db->data) { + r = munmap(db->data, db->length); + if (r) + ERROR(db->ctx, "Could not unmap the database: %m\n"); + } + + // Free the stringpool + if (db->pool) + loc_stringpool_unref(db->pool); + + // Close database file + if (db->f) + fclose(db->f); + + loc_unref(db->ctx); + free(db); +} + LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) { + struct loc_database* db = NULL; + int r = 1; + // Fail on invalid file handle - if (!f) - return -EINVAL; + if (!f) { + errno = EINVAL; + return 1; + } - struct loc_database* db = calloc(1, sizeof(*db)); + // Allocate the database object + db = calloc(1, sizeof(*db)); if (!db) - return -ENOMEM; + goto ERROR; // Reference context db->ctx = loc_ref(ctx); @@ -410,15 +499,19 @@ LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** datab DEBUG(db->ctx, "Database object allocated at %p\n", db); - int r = loc_database_read(db, f); - if (r) { - loc_database_unref(db); - return r; - } + // Try to open the database + r = loc_database_open(db, f); + if (r) + goto ERROR; *database = db; - return 0; + +ERROR: + if (db) + loc_database_free(db); + + return r; } LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) { @@ -427,56 +520,6 @@ LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) { return db; } -static void loc_database_free(struct loc_database* db) { - int r; - - DEBUG(db->ctx, "Releasing database %p\n", db); - - // Removing all ASes - if (db->as_v1) { - r = munmap(db->as_v1, db->as_count * sizeof(*db->as_v1)); - if (r) - ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno)); - } - - // Remove mapped network sections - if (db->networks_v1) { - r = munmap(db->networks_v1, db->networks_count * sizeof(*db->networks_v1)); - if (r) - ERROR(db->ctx, "Could not unmap networks section: %s\n", strerror(errno)); - } - - // Remove mapped network nodes section - if (db->network_nodes_v1) { - r = munmap(db->network_nodes_v1, db->network_nodes_count * sizeof(*db->network_nodes_v1)); - if (r) - ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno)); - } - - // Remove mapped countries section - if (db->countries_v1) { - r = munmap(db->countries_v1, db->countries_count * sizeof(*db->countries_v1)); - if (r) - ERROR(db->ctx, "Could not unmap countries section: %s\n", strerror(errno)); - } - - if (db->pool) - loc_stringpool_unref(db->pool); - - // Free signature - if (db->signature1) - free(db->signature1); - if (db->signature2) - free(db->signature2); - - // Close database file - if (db->f) - fclose(db->f); - - loc_unref(db->ctx); - free(db); -} - LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) { if (--db->refcount > 0) return NULL; @@ -486,8 +529,10 @@ LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) { } LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { + size_t bytes_read = 0; + // Cannot do this when no signature is available - if (!db->signature1 && !db->signature2) { + if (!db->signature1.data && !db->signature2.data) { DEBUG(db->ctx, "No signature available to verify\n"); return 1; } @@ -498,8 +543,8 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { // Load public key EVP_PKEY* pkey = PEM_read_PUBKEY(f, NULL, NULL, NULL); if (!pkey) { - char* error = ERR_error_string(ERR_get_error(), NULL); - ERROR(db->ctx, "Could not parse public key: %s\n", error); + ERROR(db->ctx, "Could not parse public key: %s\n", + ERR_error_string(ERR_get_error(), NULL)); return -1; } @@ -523,7 +568,12 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { // Read magic struct loc_database_magic magic; - fread(&magic, 1, sizeof(magic), db->f); + bytes_read = fread(&magic, 1, sizeof(magic), db->f); + if (bytes_read < sizeof(magic)) { + ERROR(db->ctx, "Could not read header: %m\n"); + r = 1; + goto CLEANUP; + } hexdump(db->ctx, &magic, sizeof(magic)); @@ -538,7 +588,6 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { // Read the header struct loc_database_header_v1 header_v1; - size_t bytes_read; switch (db->version) { case LOC_DATABASE_VERSION_1: @@ -593,11 +642,11 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { } // Check first signature - if (db->signature1) { - hexdump(db->ctx, db->signature1, db->signature1_length); + if (db->signature1.data) { + hexdump(db->ctx, db->signature1.data, db->signature1.length); r = EVP_DigestVerifyFinal(mdctx, - (unsigned char*)db->signature1, db->signature1_length); + (unsigned char*)db->signature1.data, db->signature1.length); if (r == 0) { DEBUG(db->ctx, "The first signature is invalid\n"); @@ -613,11 +662,11 @@ LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) { } // Check second signature only when the first one was invalid - if (r && db->signature2) { - hexdump(db->ctx, db->signature2, db->signature2_length); + if (r && db->signature2.data) { + hexdump(db->ctx, db->signature2.data, db->signature2.length); r = EVP_DigestVerifyFinal(mdctx, - (unsigned char*)db->signature2, db->signature2_length); + (unsigned char*)db->signature2.data, db->signature2.length); if (r == 0) { DEBUG(db->ctx, "The second signature is invalid\n"); @@ -661,29 +710,39 @@ LOC_EXPORT const char* loc_database_get_license(struct loc_database* db) { } LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) { - return db->as_count; + return db->as_objects.count; } // Returns the AS at position pos static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) { - if ((size_t)pos >= db->as_count) - return -EINVAL; + struct loc_database_as_v1* as_v1 = NULL; + int r; + + if ((size_t)pos >= db->as_objects.count) { + errno = ERANGE; + return 1; + } DEBUG(db->ctx, "Fetching AS at position %jd\n", (intmax_t)pos); - int r; switch (db->version) { case LOC_DATABASE_VERSION_1: - r = loc_as_new_from_database_v1(db->ctx, db->pool, as, db->as_v1 + pos); + // Find the object + as_v1 = (struct loc_database_as_v1*)loc_database_object(db, + &db->as_objects, sizeof(*as_v1), pos); + if (!as_v1) + return 1; + + r = loc_as_new_from_database_v1(db->ctx, db->pool, as, as_v1); break; default: - return -1; + errno = ENOTSUP; + return 1; } - if (r == 0) { + if (r == 0) DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as)); - } return r; } @@ -691,7 +750,7 @@ static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, of // Performs a binary search to find the AS in the list LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) { off_t lo = 0; - off_t hi = db->as_count - 1; + off_t hi = db->as_objects.count - 1; #ifdef ENABLE_DEBUG // Save start time @@ -739,24 +798,32 @@ LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, // Returns the network at position pos static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network, struct in6_addr* address, unsigned int prefix, off_t pos) { - if ((size_t)pos >= db->networks_count) { + struct loc_database_network_v1* network_v1 = NULL; + int r; + + if ((size_t)pos >= db->network_objects.count) { DEBUG(db->ctx, "Network ID out of range: %jd/%jd\n", - (intmax_t)pos, (intmax_t)db->networks_count); - return -EINVAL; + (intmax_t)pos, (intmax_t)db->network_objects.count); + errno = ERANGE; + return 1; } - DEBUG(db->ctx, "Fetching network at position %jd\n", (intmax_t)pos); - int r; switch (db->version) { case LOC_DATABASE_VERSION_1: - r = loc_network_new_from_database_v1(db->ctx, network, - address, prefix, db->networks_v1 + pos); + // Read the object + network_v1 = (struct loc_database_network_v1*)loc_database_object(db, + &db->network_objects, sizeof(*network_v1), pos); + if (!network_v1) + return 1; + + r = loc_network_new_from_database_v1(db->ctx, network, address, prefix, network_v1); break; default: - return -1; + errno = ENOTSUP; + return 1; } if (r == 0) @@ -774,13 +841,13 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru const struct loc_database_network_node_v1* node) { off_t network_index = be32toh(node->network); - DEBUG(db->ctx, "Handling leaf node at %jd (%jd)\n", (intmax_t)(node - db->network_nodes_v1), (intmax_t)network_index); + DEBUG(db->ctx, "Handling leaf node at %jd\n", (intmax_t)network_index); // Fetch the network - int r = loc_database_fetch_network(db, network, - network_address, prefix, network_index); + int r = loc_database_fetch_network(db, network, network_address, prefix, network_index); if (r) { - ERROR(db->ctx, "Could not fetch network %jd from database\n", (intmax_t)network_index); + ERROR(db->ctx, "Could not fetch network %jd from database: %m\n", + (intmax_t)network_index); return r; } @@ -800,29 +867,37 @@ static int __loc_database_lookup_handle_leaf(struct loc_database* db, const stru // Searches for an exact match along the path static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* address, struct loc_network** network, struct in6_addr* network_address, - const struct loc_database_network_node_v1* node, unsigned int level) { + off_t node_index, unsigned int level) { + struct loc_database_network_node_v1* node_v1 = NULL; + int r; - off_t node_index; + + // Fetch the next node + node_v1 = (struct loc_database_network_node_v1*)loc_database_object(db, + &db->network_node_objects, sizeof(*node_v1), node_index); + if (!node_v1) + return 1; // Follow the path int bit = loc_address_get_bit(address, level); loc_address_set_bit(network_address, level, bit); if (bit == 0) - node_index = be32toh(node->zero); + node_index = be32toh(node_v1->zero); else - node_index = be32toh(node->one); + node_index = be32toh(node_v1->one); // If the node index is zero, the tree ends here // and we cannot descend any further if (node_index > 0) { // Check boundaries - if ((size_t)node_index >= db->network_nodes_count) - return -EINVAL; + if ((size_t)node_index >= db->network_node_objects.count) { + errno = ERANGE; + return 1; + } // Move on to the next node - r = __loc_database_lookup(db, address, network, network_address, - db->network_nodes_v1 + node_index, level + 1); + r = __loc_database_lookup(db, address, network, network_address, node_index, level + 1); // End here if a result was found if (r == 0) @@ -838,8 +913,8 @@ static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* } // If this node has a leaf, we will check if it matches - if (__loc_database_node_is_leaf(node)) { - r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node); + if (__loc_database_node_is_leaf(node_v1)) { + r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node_v1); if (r <= 0) return r; } @@ -859,8 +934,7 @@ LOC_EXPORT int loc_database_lookup(struct loc_database* db, clock_t start = clock(); #endif - int r = __loc_database_lookup(db, address, network, &network_address, - db->network_nodes_v1, 0); + int r = __loc_database_lookup(db, address, network, &network_address, 0, 0); #ifdef ENABLE_DEBUG clock_t end = clock(); @@ -887,24 +961,35 @@ LOC_EXPORT int loc_database_lookup_from_string(struct loc_database* db, // Returns the country at position pos static int loc_database_fetch_country(struct loc_database* db, struct loc_country** country, off_t pos) { - if ((size_t)pos >= db->countries_count) - return -EINVAL; + struct loc_database_country_v1* country_v1 = NULL; + int r; + + // Check if the country is within range + if ((size_t)pos >= db->country_objects.count) { + errno = ERANGE; + return 1; + } DEBUG(db->ctx, "Fetching country at position %jd\n", (intmax_t)pos); - int r; switch (db->version) { case LOC_DATABASE_VERSION_1: - r = loc_country_new_from_database_v1(db->ctx, db->pool, country, db->countries_v1 + pos); + // Read the object + country_v1 = (struct loc_database_country_v1*)loc_database_object(db, + &db->country_objects, sizeof(*country_v1), pos); + if (!country_v1) + return 1; + + r = loc_country_new_from_database_v1(db->ctx, db->pool, country, country_v1); break; default: - return -1; + errno = ENOTSUP; + return 1; } - if (r == 0) { + if (r == 0) DEBUG(db->ctx, "Got country %s\n", loc_country_get_code(*country)); - } return r; } @@ -913,7 +998,13 @@ static int loc_database_fetch_country(struct loc_database* db, LOC_EXPORT int loc_database_get_country(struct loc_database* db, struct loc_country** country, const char* code) { off_t lo = 0; - off_t hi = db->countries_count - 1; + off_t hi = db->country_objects.count - 1; + + // Check if the country code is valid + if (!loc_country_code_is_valid(code)) { + errno = EINVAL; + return 1; + } #ifdef ENABLE_DEBUG // Save start time @@ -957,7 +1048,7 @@ LOC_EXPORT int loc_database_get_country(struct loc_database* db, // Nothing found *country = NULL; - return 1; + return 0; } // Enumerator @@ -979,7 +1070,8 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera loc_as_list_unref(enumerator->asns); // Free network search - free(enumerator->networks_visited); + if (enumerator->networks_visited) + free(enumerator->networks_visited); // Free subnet/bogons stack if (enumerator->stack) @@ -993,9 +1085,12 @@ static void loc_database_enumerator_free(struct loc_database_enumerator* enumera LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator, struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) { + int r; + struct loc_database_enumerator* e = calloc(1, sizeof(*e)); - if (!e) + if (!e) { return -ENOMEM; + } // Reference context e->ctx = loc_ref(db->ctx); @@ -1008,14 +1103,17 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum // Initialise graph search e->network_stack_depth = 1; - e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited)); + e->networks_visited = calloc(db->network_node_objects.count, sizeof(*e->networks_visited)); + if (!e->networks_visited) { + ERROR(db->ctx, "Could not allocated visited networks: %m\n"); + r = 1; + goto ERROR; + } // Allocate stack - int r = loc_network_list_new(e->ctx, &e->stack); - if (r) { - loc_database_enumerator_free(e); - return r; - } + r = loc_network_list_new(e->ctx, &e->stack); + if (r) + goto ERROR; // Initialize bogon search loc_address_reset(&e->gap6_start, AF_INET6); @@ -1025,6 +1123,12 @@ LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enum *enumerator = e; return 0; + +ERROR: + if (e) + loc_database_enumerator_free(e); + + return r; } LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator) { @@ -1114,7 +1218,7 @@ LOC_EXPORT int loc_database_enumerator_next_as( struct loc_database* db = enumerator->db; - while (enumerator->as_index < db->as_count) { + while (enumerator->as_index < db->as_objects.count) { // Fetch the next AS int r = loc_database_fetch_as(db, as, enumerator->as_index++); if (r) @@ -1149,7 +1253,15 @@ static int loc_database_enumerator_stack_push_node( // Check if there is any space left on the stack if (e->network_stack_depth >= MAX_STACK_DEPTH) { ERROR(e->ctx, "Maximum stack size reached: %d\n", e->network_stack_depth); - return -1; + return 1; + } + + // Check if the node is in range + if (offset >= (off_t)e->db->network_node_objects.count) { + ERROR(e->ctx, "Trying to add invalid node with offset %jd/%zu\n", + offset, e->db->network_node_objects.count); + errno = ERANGE; + return 1; } // Increase stack size @@ -1235,6 +1347,8 @@ static int __loc_database_enumerator_next_network( // Get object from top of the stack struct loc_node_stack* node = &enumerator->network_stack[enumerator->network_stack_depth]; + DEBUG(enumerator->ctx, " Got node: %jd\n", node->offset); + // Remove the node from the stack if we have already visited it if (enumerator->networks_visited[node->offset]) { enumerator->network_stack_depth--; @@ -1250,18 +1364,19 @@ static int __loc_database_enumerator_next_network( // Pop node from top of the stack struct loc_database_network_node_v1* n = - enumerator->db->network_nodes_v1 + node->offset; + (struct loc_database_network_node_v1*)loc_database_object(enumerator->db, + &enumerator->db->network_node_objects, sizeof(*n), node->offset); + if (!n) + return 1; // Add edges to stack int r = loc_database_enumerator_stack_push_node(enumerator, be32toh(n->one), 1, node->depth + 1); - if (r) return r; r = loc_database_enumerator_stack_push_node(enumerator, be32toh(n->zero), 0, node->depth + 1); - if (r) return r; @@ -1568,7 +1683,7 @@ LOC_EXPORT int loc_database_enumerator_next_country( struct loc_database* db = enumerator->db; - while (enumerator->country_index < db->countries_count) { + while (enumerator->country_index < db->country_objects.count) { // Fetch the next country int r = loc_database_fetch_country(db, country, enumerator->country_index++); if (r) diff --git a/src/libloc.c b/src/libloc.c index cf2d740..450c5e6 100644 --- a/src/libloc.c +++ b/src/libloc.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include @@ -77,7 +76,7 @@ static int log_priority(const char* priority) { LOC_EXPORT int loc_new(struct loc_ctx** ctx) { struct loc_ctx* c = calloc(1, sizeof(*c)); if (!c) - return -ENOMEM; + return 1; c->refcount = 1; c->log_fn = log_stderr; @@ -104,9 +103,6 @@ LOC_EXPORT struct loc_ctx* loc_ref(struct loc_ctx* ctx) { } LOC_EXPORT struct loc_ctx* loc_unref(struct loc_ctx* ctx) { - if (!ctx) - return NULL; - if (--ctx->refcount > 0) return NULL; diff --git a/src/libloc/address.h b/src/libloc/address.h index f7fe333..f4c0ee3 100644 --- a/src/libloc/address.h +++ b/src/libloc/address.h @@ -22,6 +22,8 @@ #include #include +#include + /* All of these functions are private and for internal use only */ diff --git a/src/libloc/compat.h b/src/libloc/compat.h index 8750976..aecfeb4 100644 --- a/src/libloc/compat.h +++ b/src/libloc/compat.h @@ -35,6 +35,10 @@ # define s6_addr32 __u6_addr.__u6_addr32 #endif +#ifndef reallocarray +# define reallocarray(ptr, nmemb, size) realloc(ptr, nmemb * size) +#endif + #endif #endif diff --git a/src/libloc/format.h b/src/libloc/format.h index a04c089..030394b 100644 --- a/src/libloc/format.h +++ b/src/libloc/format.h @@ -20,6 +20,7 @@ #include #define LOC_DATABASE_MAGIC "LOCDBXX" +#define LOC_DATABASE_MAGIC_SIZE sizeof(struct loc_database_magic) enum loc_database_version { LOC_DATABASE_VERSION_UNSET = 0, @@ -33,7 +34,7 @@ enum loc_database_version { #define LOC_DATABASE_DOMAIN "_v%u._db.location.ipfire.org" #define LOC_DATABASE_PAGE_SIZE 4096 -#define LOC_SIGNATURE_MAX_LENGTH (LOC_DATABASE_PAGE_SIZE / 2) +#define LOC_SIGNATURE_MAX_LENGTH 2048 struct loc_database_magic { char magic[7]; diff --git a/src/libloc/private.h b/src/libloc/private.h index 2ee97d1..89ce5bc 100644 --- a/src/libloc/private.h +++ b/src/libloc/private.h @@ -66,6 +66,9 @@ static inline void hexdump(struct loc_ctx* ctx, const void* addr, size_t len) { DEBUG(ctx, "Dumping %zu byte(s)\n", len); + if (!len) + return; + // Process every byte in the data for (i = 0; i < len; i++) { // Multiple of 16 means new line (with line offset) diff --git a/src/libloc/stringpool.h b/src/libloc/stringpool.h index 932aad7..c6bd216 100644 --- a/src/libloc/stringpool.h +++ b/src/libloc/stringpool.h @@ -27,7 +27,7 @@ struct loc_stringpool; int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool); int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool, - FILE* f, size_t length, off_t offset); + const char* data, const size_t length); struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool); struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool); diff --git a/src/network-list.c b/src/network-list.c index e2bbf05..1b64a7e 100644 --- a/src/network-list.c +++ b/src/network-list.c @@ -44,7 +44,7 @@ static int loc_network_list_grow(struct loc_network_list* list) { struct loc_network** elements = reallocarray(list->elements, list->elements_size + size, sizeof(*list->elements)); if (!elements) - return -errno; + return 1; list->elements = elements; list->elements_size += size; @@ -83,9 +83,6 @@ static void loc_network_list_free(struct loc_network_list* list) { } LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) { - if (!list) - return NULL; - if (--list->refcount > 0) return list; diff --git a/src/network.c b/src/network.c index 37a483e..10af51a 100644 --- a/src/network.c +++ b/src/network.c @@ -59,10 +59,8 @@ LOC_EXPORT int loc_network_new(struct loc_ctx* ctx, struct loc_network** network } struct loc_network* n = calloc(1, sizeof(*n)); - if (!n) { - errno = ENOMEM; + if (!n) return 1; - } n->ctx = loc_ref(ctx); n->refcount = 1; @@ -118,9 +116,6 @@ static void loc_network_free(struct loc_network* network) { } LOC_EXPORT struct loc_network* loc_network_unref(struct loc_network* network) { - if (!network) - return NULL; - if (--network->refcount > 0) return network; @@ -580,7 +575,7 @@ int loc_network_new_from_database_v1(struct loc_ctx* ctx, struct loc_network** n int r = loc_network_new(ctx, network, address, prefix); if (r) { - ERROR(ctx, "Could not allocate a new network: %s", strerror(-r)); + ERROR(ctx, "Could not allocate a new network: %m\n"); return r; } @@ -632,7 +627,7 @@ struct loc_network_tree_node { int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) { struct loc_network_tree* t = calloc(1, sizeof(*t)); if (!t) - return -ENOMEM; + return 1; t->ctx = loc_ref(ctx); t->refcount = 1; @@ -858,9 +853,6 @@ static void loc_network_tree_node_free(struct loc_network_tree_node* node) { } struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) { - if (!node) - return NULL; - if (--node->refcount > 0) return node; diff --git a/src/python/database.c b/src/python/database.c index a33c31f..d6ee4d0 100644 --- a/src/python/database.c +++ b/src/python/database.c @@ -45,28 +45,36 @@ static void Database_dealloc(DatabaseObject* self) { static int Database_init(DatabaseObject* self, PyObject* args, PyObject* kwargs) { const char* path = NULL; + FILE* f = NULL; + // Parse arguments if (!PyArg_ParseTuple(args, "s", &path)) return -1; + // Copy path self->path = strdup(path); + if (!self->path) + goto ERROR; // Open the file for reading - FILE* f = fopen(self->path, "r"); - if (!f) { - PyErr_SetFromErrno(PyExc_IOError); - return -1; - } + f = fopen(self->path, "r"); + if (!f) + goto ERROR; // Load the database int r = loc_database_new(loc_ctx, &self->db, f); - fclose(f); - - // Return on any errors if (r) - return -1; + goto ERROR; + fclose(f); return 0; + +ERROR: + if (f) + fclose(f); + + PyErr_SetFromErrno(PyExc_OSError); + return -1; } static PyObject* Database_repr(DatabaseObject* self) { @@ -158,17 +166,32 @@ static PyObject* Database_get_as(DatabaseObject* self, PyObject* args) { } static PyObject* Database_get_country(DatabaseObject* self, PyObject* args) { + struct loc_country* country = NULL; const char* country_code = NULL; if (!PyArg_ParseTuple(args, "s", &country_code)) return NULL; - struct loc_country* country; + // Fetch the country int r = loc_database_get_country(self->db, &country, country_code); if (r) { - Py_RETURN_NONE; + switch (errno) { + case EINVAL: + PyErr_SetString(PyExc_ValueError, "Invalid country code"); + break; + + default: + PyErr_SetFromErrno(PyExc_OSError); + break; + } + + return NULL; } + // No result + if (!country) + Py_RETURN_NONE; + PyObject* obj = new_country(&CountryType, country); loc_country_unref(country); @@ -191,18 +214,21 @@ static PyObject* Database_lookup(DatabaseObject* self, PyObject* args) { loc_network_unref(network); return obj; + } // Nothing found - } else if (r == 1) { + if (!errno) Py_RETURN_NONE; - // Invalid input - } else if (r == -EINVAL) { - PyErr_Format(PyExc_ValueError, "Invalid IP address: %s", address); - return NULL; + // Handle any errors + switch (errno) { + case EINVAL: + PyErr_Format(PyExc_ValueError, "Invalid IP address: %s", address); + + default: + PyErr_SetFromErrno(PyExc_OSError); } - // Unexpected error return NULL; } @@ -353,7 +379,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, struct loc_as_list* asns; r = loc_as_list_new(loc_ctx, &asns); if (r) { - PyErr_SetString(PyExc_SystemError, "Could not create AS list"); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -372,7 +398,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, struct loc_as* as; r = loc_as_new(loc_ctx, &as, number); if (r) { - PyErr_SetString(PyExc_SystemError, "Could not create AS"); + PyErr_SetFromErrno(PyExc_OSError); loc_as_list_unref(asns); loc_as_unref(as); @@ -381,7 +407,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, r = loc_as_list_append(asns, as); if (r) { - PyErr_SetString(PyExc_SystemError, "Could not append AS to the list"); + PyErr_SetFromErrno(PyExc_OSError); loc_as_list_unref(asns); loc_as_unref(as); @@ -393,7 +419,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, r = loc_database_enumerator_set_asns(enumerator, asns); if (r) { - PyErr_SetFromErrno(PyExc_SystemError); + PyErr_SetFromErrno(PyExc_OSError); loc_as_list_unref(asns); return NULL; @@ -407,7 +433,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, r = loc_database_enumerator_set_flag(enumerator, flags); if (r) { - PyErr_SetFromErrno(PyExc_SystemError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } } @@ -417,7 +443,7 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args, r = loc_database_enumerator_set_family(enumerator, family); if (r) { - PyErr_SetFromErrno(PyExc_SystemError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } } diff --git a/src/python/location/__init__.py b/src/python/location/__init__.py index 9b570c7..f635737 100644 --- a/src/python/location/__init__.py +++ b/src/python/location/__init__.py @@ -22,3 +22,13 @@ from _location import * # Initialise logging from . import logger + +def open(path=None): + """ + Opens the database at path, or opens the default database. + """ + if not path: + path = DATABASE_PATH + + # Open the database + return Database(path) diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c index 09cd5dd..45c7bd8 100644 --- a/src/python/locationmodule.c +++ b/src/python/locationmodule.c @@ -121,6 +121,10 @@ PyMODINIT_FUNC PyInit__location(void) { if (PyModule_AddStringConstant(m, "__version__", PACKAGE_VERSION)) return NULL; + // Default Database Path + if (PyModule_AddStringConstant(m, "DATABASE_PATH", LIBLOC_DEFAULT_DATABASE_PATH)) + return NULL; + // AS if (PyType_Ready(&ASType) < 0) return NULL; @@ -147,7 +151,7 @@ PyMODINIT_FUNC PyInit__location(void) { return NULL; Py_INCREF(&DatabaseEnumeratorType); - //PyModule_AddObject(m, "DatabaseEnumerator", (PyObject *)&DatabaseEnumeratorType); + PyModule_AddObject(m, "DatabaseEnumerator", (PyObject *)&DatabaseEnumeratorType); // Network if (PyType_Ready(&NetworkType) < 0) diff --git a/src/python/network.c b/src/python/network.c index 474b6de..c14174e 100644 --- a/src/python/network.c +++ b/src/python/network.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include diff --git a/src/python/writer.c b/src/python/writer.c index c54596a..5d8027c 100644 --- a/src/python/writer.c +++ b/src/python/writer.c @@ -227,7 +227,7 @@ static PyObject* Writer_write(WriterObject* self, PyObject* args) { FILE* f = fopen(path, "w+"); if (!f) { - PyErr_Format(PyExc_IOError, strerror(errno)); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -236,7 +236,7 @@ static PyObject* Writer_write(WriterObject* self, PyObject* args) { // Raise any errors if (r) { - PyErr_Format(PyExc_IOError, strerror(errno)); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in index 1785791..8d47497 100644 --- a/src/scripts/location-importer.in +++ b/src/scripts/location-importer.in @@ -1499,7 +1499,7 @@ class CLI(object): # Iterate through every line, filter comments and add remaining ASNs to # the override table in case they are valid... with self.db.transaction(): - for sline in t.readlines(): + for sline in f.readlines(): # The response is assumed to be encoded in UTF-8... sline = sline.decode("utf-8") diff --git a/src/scripts/location.in b/src/scripts/location.in index 25119a8..be13156 100644 --- a/src/scripts/location.in +++ b/src/scripts/location.in @@ -59,7 +59,7 @@ class CLI(object): # database parser.add_argument("--database", "-d", - default="@databasedir@/database.db", help=_("Path to database"), + default=location.DATABASE_PATH, help=_("Path to database"), ) # public key diff --git a/src/stringpool.c b/src/stringpool.c index 187c9d3..9986a61 100644 --- a/src/stringpool.c +++ b/src/stringpool.c @@ -27,103 +27,75 @@ #include #include -enum loc_stringpool_mode { - STRINGPOOL_DEFAULT, - STRINGPOOL_MMAP, -}; +#define LOC_STRINGPOOL_BLOCK_SIZE (512 * 1024) struct loc_stringpool { struct loc_ctx* ctx; int refcount; - enum loc_stringpool_mode mode; - - char* data; + // Reference to any mapped data + const char* data; ssize_t length; - char* pos; + // Reference to own storage + char* blocks; + size_t size; }; -static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char* pos) { - if (pos < pool->data) - return -EFAULT; - - if (pos > (pool->data + pool->length)) - return -EFAULT; - - return pos - pool->data; -} - -static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) { - if (offset < 0 || offset >= pool->length) - return NULL; - - return pool->data + offset; -} - -static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) { - DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length); - - // Save pos pointer - off_t pos = loc_stringpool_get_offset(pool, pool->pos); +static int loc_stringpool_grow(struct loc_stringpool* pool, const size_t size) { + DEBUG(pool->ctx, "Growing string pool by %zu byte(s)\n", size); - // Reallocate data section - pool->data = realloc(pool->data, length); - if (!pool->data) - return -ENOMEM; + // Increment size + pool->size += size; - pool->length = length; + // Reallocate blocks + pool->blocks = realloc(pool->blocks, pool->size); + if (!pool->blocks) { + ERROR(pool->ctx, "Could not grow string pool: %m\n"); + return 1; + } - // Restore pos - pool->pos = __loc_stringpool_get(pool, pos); + // Update data pointer + pool->data = pool->blocks; return 0; } static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* string) { - if (!string) - return -EINVAL; + if (!string) { + errno = EINVAL; + return -1; + } DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool); + // How much space to we need? + const size_t length = strlen(string) + 1; + // Make sure we have enough space - int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1); - if (r) { - errno = r; - return -1; + if (pool->length + length > pool->size) { + int r = loc_stringpool_grow(pool, LOC_STRINGPOOL_BLOCK_SIZE); + if (r) + return r; } - off_t offset = loc_stringpool_get_offset(pool, pool->pos); + off_t offset = pool->length; - // Copy string byte by byte - while (*string) - *pool->pos++ = *string++; + // Copy the string + memcpy(pool->blocks + offset, string, length); - // Terminate the string - *pool->pos++ = '\0'; + // Update the length of the pool + pool->length += length; return offset; } static void loc_stringpool_free(struct loc_stringpool* pool) { DEBUG(pool->ctx, "Releasing string pool %p\n", pool); - int r; - - switch (pool->mode) { - case STRINGPOOL_DEFAULT: - if (pool->data) - free(pool->data); - break; - - case STRINGPOOL_MMAP: - if (pool->data) { - r = munmap(pool->data, pool->length); - if (r) - ERROR(pool->ctx, "Could not unmap data at %p: %s\n", - pool->data, strerror(errno)); - } - break; - } + + // Free any data + if (pool->blocks) + free(pool->blocks); loc_unref(pool->ctx); free(pool); @@ -137,56 +109,34 @@ int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool) { p->ctx = loc_ref(ctx); p->refcount = 1; - // Save mode - p->mode = STRINGPOOL_DEFAULT; - *pool = p; return 0; } -static int loc_stringpool_mmap(struct loc_stringpool* pool, FILE* f, size_t length, off_t offset) { - if (pool->mode != STRINGPOOL_MMAP) - return -EINVAL; - - DEBUG(pool->ctx, "Reading string pool starting from %jd (%zu bytes)\n", (intmax_t)offset, length); - - // Map file content into memory - pool->data = pool->pos = mmap(NULL, length, PROT_READ, - MAP_PRIVATE, fileno(f), offset); - - // Store size of section - pool->length = length; - - if (pool->data == MAP_FAILED) - return 1; - - return 0; -} - int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool, - FILE* f, size_t length, off_t offset) { + const char* data, const size_t length) { struct loc_stringpool* p = NULL; // Allocate a new stringpool int r = loc_stringpool_new(ctx, &p); if (r) - return r; + goto ERROR; - // Change mode to mmap - p->mode = STRINGPOOL_MMAP; + // Store data and length + p->data = data; + p->length = length; - // Map data into memory - if (length > 0) { - r = loc_stringpool_mmap(p, f, length, offset); - if (r) { - loc_stringpool_free(p); - return r; - } - } + DEBUG(p->ctx, "Opened string pool at %p (%zu bytes)\n", p->data, p->length); *pool = p; return 0; + +ERROR: + if (p) + loc_stringpool_free(p); + + return r; } struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool) { @@ -204,40 +154,46 @@ struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool) { return NULL; } -static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t offset) { - const char* string = loc_stringpool_get(pool, offset); - if (!string) - return offset; - - return offset + strlen(string) + 1; -} - const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) { - return __loc_stringpool_get(pool, offset); + // Check boundaries + if (offset < 0 || offset >= pool->length) { + errno = ERANGE; + return NULL; + } + + // Return any data that we have in memory + return pool->data + offset; } size_t loc_stringpool_get_size(struct loc_stringpool* pool) { - return loc_stringpool_get_offset(pool, pool->pos); + return pool->length; } static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) { - if (!s || !*s) - return -EINVAL; + if (!s || !*s) { + errno = EINVAL; + return -1; + } off_t offset = 0; while (offset < pool->length) { const char* string = loc_stringpool_get(pool, offset); + + // Error! if (!string) - break; + return 1; - int r = strcmp(s, string); - if (r == 0) + // Is this a match? + if (strcmp(s, string) == 0) return offset; - offset = loc_stringpool_get_next_offset(pool, offset); + // Shift offset + offset += strlen(string) + 1; } - return -ENOENT; + // Nothing found + errno = ENOENT; + return -1; } off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string) { @@ -256,11 +212,12 @@ void loc_stringpool_dump(struct loc_stringpool* pool) { while (offset < pool->length) { const char* string = loc_stringpool_get(pool, offset); if (!string) - break; + return; printf("%jd (%zu): %s\n", (intmax_t)offset, strlen(string), string); - offset = loc_stringpool_get_next_offset(pool, offset); + // Shift offset + offset += strlen(string) + 1; } } diff --git a/src/test-as.c b/src/test-as.c index 91b62fd..b135c6b 100644 --- a/src/test-as.c +++ b/src/test-as.c @@ -55,13 +55,13 @@ int main(int argc, char** argv) { FILE* f = tmpfile(); if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + fprintf(stderr, "Could not open file for writing: %m\n"); exit(EXIT_FAILURE); } err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); if (err) { - fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + fprintf(stderr, "Could not write database: %m\n"); exit(EXIT_FAILURE); } @@ -71,7 +71,7 @@ int main(int argc, char** argv) { struct loc_database* db; err = loc_database_new(ctx, &db, f); if (err) { - fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + fprintf(stderr, "Could not open database: %m\n"); exit(EXIT_FAILURE); } diff --git a/src/test-country.c b/src/test-country.c index c6aff49..f9db204 100644 --- a/src/test-country.c +++ b/src/test-country.c @@ -124,13 +124,13 @@ int main(int argc, char** argv) { FILE* f = tmpfile(); if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + fprintf(stderr, "Could not open file for writing: %m\n"); exit(EXIT_FAILURE); } err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); if (err) { - fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + fprintf(stderr, "Could not write database: %m\n"); exit(EXIT_FAILURE); } loc_writer_unref(writer); @@ -139,13 +139,13 @@ int main(int argc, char** argv) { struct loc_database* db; err = loc_database_new(ctx, &db, f); if (err) { - fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + fprintf(stderr, "Could not open database: %m\n"); exit(EXIT_FAILURE); } // Lookup an address in the subnet err = loc_database_get_country(db, &country, "YY"); - if (err) { + if (err || !country) { fprintf(stderr, "Could not find country: YY\n"); exit(EXIT_FAILURE); } diff --git a/src/test-database.c b/src/test-database.c index 7037f6a..8ba558a 100644 --- a/src/test-database.c +++ b/src/test-database.c @@ -167,13 +167,13 @@ int main(int argc, char** argv) { FILE* f = tmpfile(); if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + fprintf(stderr, "Could not open file for writing: %m\n"); exit(EXIT_FAILURE); } err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); if (err) { - fprintf(stderr, "Could not write database: %s\n", strerror(err)); + fprintf(stderr, "Could not write database: %m\n"); exit(EXIT_FAILURE); } loc_writer_unref(writer); @@ -182,7 +182,7 @@ int main(int argc, char** argv) { struct loc_database* db; err = loc_database_new(ctx, &db, f); if (err) { - fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + fprintf(stderr, "Could not open database: %m\n"); exit(EXIT_FAILURE); } diff --git a/src/test-network.c b/src/test-network.c index 1c49d60..717ad3a 100644 --- a/src/test-network.c +++ b/src/test-network.c @@ -260,13 +260,13 @@ int main(int argc, char** argv) { FILE* f = tmpfile(); if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + fprintf(stderr, "Could not open file for writing: %m\n"); exit(EXIT_FAILURE); } err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); if (err) { - fprintf(stderr, "Could not write database: %s\n", strerror(-err)); + fprintf(stderr, "Could not write database: %m\n"); exit(EXIT_FAILURE); } loc_writer_unref(writer); @@ -284,7 +284,7 @@ int main(int argc, char** argv) { struct loc_database* db; err = loc_database_new(ctx, &db, f); if (err) { - fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + fprintf(stderr, "Could not open database: %m\n"); exit(EXIT_FAILURE); } @@ -302,7 +302,6 @@ int main(int argc, char** argv) { fprintf(stderr, "Could look up 2001:db8:fffe:1::, but I shouldn't\n"); exit(EXIT_FAILURE); } - loc_network_unref(network1); const struct bit_length_test { const char* network; diff --git a/src/test-signature.c b/src/test-signature.c index bc66c96..9af9236 100644 --- a/src/test-signature.c +++ b/src/test-signature.c @@ -34,20 +34,20 @@ int main(int argc, char** argv) { // Open public key FILE* public_key = fopen(ABS_SRCDIR "/examples/public-key.pem", "r"); if (!public_key) { - fprintf(stderr, "Could not open public key file: %s\n", strerror(errno)); + fprintf(stderr, "Could not open public key file: %m\n"); exit(EXIT_FAILURE); } // Open private key FILE* private_key1 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r"); if (!private_key1) { - fprintf(stderr, "Could not open private key file: %s\n", strerror(errno)); + fprintf(stderr, "Could not open private key file: %m\n"); exit(EXIT_FAILURE); } FILE* private_key2 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r"); if (!private_key2) { - fprintf(stderr, "Could not open private key file: %s\n", strerror(errno)); + fprintf(stderr, "Could not open private key file: %m\n"); exit(EXIT_FAILURE); } @@ -67,13 +67,13 @@ int main(int argc, char** argv) { FILE* f = tmpfile(); if (!f) { - fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno)); + fprintf(stderr, "Could not open file for writing: %m\n"); exit(EXIT_FAILURE); } err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET); if (err) { - fprintf(stderr, "Could not write database: %s\n", strerror(err)); + fprintf(stderr, "Could not write database: %m\n"); exit(EXIT_FAILURE); } loc_writer_unref(writer); @@ -82,7 +82,7 @@ int main(int argc, char** argv) { struct loc_database* db; err = loc_database_new(ctx, &db, f); if (err) { - fprintf(stderr, "Could not open database: %s\n", strerror(-err)); + fprintf(stderr, "Could not open database: %m\n"); exit(EXIT_FAILURE); } @@ -96,7 +96,7 @@ int main(int argc, char** argv) { // Open another public key public_key = freopen(ABS_SRCDIR "/src/signing-key.pem", "r", public_key); if (!public_key) { - fprintf(stderr, "Could not open public key file: %s\n", strerror(errno)); + fprintf(stderr, "Could not open public key file: %m\n"); exit(EXIT_FAILURE); } diff --git a/src/test-stringpool.c b/src/test-stringpool.c index 392aa29..a94d8f8 100644 --- a/src/test-stringpool.c +++ b/src/test-stringpool.c @@ -74,7 +74,7 @@ int main(int argc, char** argv) { // Append a string off_t pos = loc_stringpool_add(pool, "ABC"); if (pos < 0) { - fprintf(stderr, "Could not add string: %s\n", strerror(-pos)); + fprintf(stderr, "Could not add string: %m\n"); exit(EXIT_FAILURE); } @@ -108,7 +108,7 @@ int main(int argc, char** argv) { free(string); if (pos < 0) { - fprintf(stderr, "Could not add string %d: %s\n", i, strerror(-pos)); + fprintf(stderr, "Could not add string %d: %m\n", i); exit(EXIT_FAILURE); } } diff --git a/src/writer.c b/src/writer.c index 4420972..51e9a8e 100644 --- a/src/writer.c +++ b/src/writer.c @@ -91,7 +91,7 @@ LOC_EXPORT int loc_writer_new(struct loc_ctx* ctx, struct loc_writer** writer, FILE* fkey1, FILE* fkey2) { struct loc_writer* w = calloc(1, sizeof(*w)); if (!w) - return -ENOMEM; + return 1; w->ctx = loc_ref(ctx); w->refcount = 1; @@ -172,7 +172,8 @@ static void loc_writer_free(struct loc_writer* writer) { loc_network_tree_unref(writer->networks); // Unref the string pool - loc_stringpool_unref(writer->pool); + if (writer->pool) + loc_stringpool_unref(writer->pool); loc_unref(writer->ctx); free(writer); @@ -537,6 +538,8 @@ static int loc_database_write_countries(struct loc_writer* writer, static int loc_writer_create_signature(struct loc_writer* writer, struct loc_database_header_v1* header, FILE* f, EVP_PKEY* private_key, char* signature, size_t* length) { + size_t bytes_read = 0; + DEBUG(writer->ctx, "Creating signature...\n"); // Read file from the beginning @@ -554,7 +557,12 @@ static int loc_writer_create_signature(struct loc_writer* writer, // Read magic struct loc_database_magic magic; - fread(&magic, 1, sizeof(magic), f); + bytes_read = fread(&magic, 1, sizeof(magic), f); + if (bytes_read < sizeof(magic)) { + ERROR(writer->ctx, "Could not read header: %m\n"); + r = 1; + goto END; + } hexdump(writer->ctx, &magic, sizeof(magic)); @@ -580,11 +588,11 @@ static int loc_writer_create_signature(struct loc_writer* writer, // Walk through the file in chunks of 64kB char buffer[64 * 1024]; while (!feof(f)) { - size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); + bytes_read = fread(buffer, 1, sizeof(buffer), f); if (ferror(f)) { - ERROR(writer->ctx, "Error reading from file: %s\n", strerror(errno)); - r = errno; + ERROR(writer->ctx, "Error reading from file: %m\n"); + r = 1; goto END; } @@ -745,5 +753,8 @@ LOC_EXPORT int loc_writer_write(struct loc_writer* writer, FILE* f, enum loc_dat fwrite(&header, 1, sizeof(header), f); + // Flush everything + fflush(f); + return r; } diff --git a/tests/data/signing-key.pem b/tests/data/signing-key.pem new file mode 120000 index 0000000..b1da823 --- /dev/null +++ b/tests/data/signing-key.pem @@ -0,0 +1 @@ +../../src/signing-key.pem \ No newline at end of file diff --git a/tests/python/test-database.py b/tests/python/test-database.py new file mode 100755 index 0000000..4846e7a --- /dev/null +++ b/tests/python/test-database.py @@ -0,0 +1,144 @@ +#!/usr/bin/python3 +############################################################################### +# # +# libloc - A library to determine the location of someone on the Internet # +# # +# Copyright (C) 2022 IPFire Development Team # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +############################################################################### + +import location +import os +import unittest + +TEST_DATA_DIR = os.environ["TEST_DATA_DIR"] + +class Test(unittest.TestCase): + def setUp(self): + path = os.path.join(TEST_DATA_DIR, "location-2022-03-30.db") + + # Load the database + self.db = location.Database(path) + + def test_metadata(self): + """ + Check if any metadata matches what we expected + """ + # Vendor + self.assertEqual(self.db.vendor, "IPFire Project") + + # Description + self.assertEqual(self.db.description, + "This database has been obtained from https://location.ipfire.org/\n\nFind the full license terms at https://creativecommons.org/licenses/by-sa/4.0/") + + # License + self.assertEqual(self.db.license, "CC BY-SA 4.0") + + # Created At + self.assertEqual(self.db.created_at, 1648619023) + + def test_fetch_network(self): + """ + Try fetching some results that should exist + """ + n = self.db.lookup("81.3.27.38") + self.assertIsInstance(n, location.Network) + + n = self.db.lookup("1.1.1.1") + self.assertIsInstance(n, location.Network) + + n = self.db.lookup("8.8.8.8") + self.assertIsInstance(n, location.Network) + + def test_fetch_network_nonexistant(self): + """ + Try to fetch something that should not exist + """ + n = self.db.lookup("255.255.255.255") + self.assertIsNone(n) + + def test_fetch_network_invalid(self): + """ + Feed some invalid inputs into the lookup function + """ + with self.assertRaises(ValueError): + self.db.lookup("XXX") + + with self.assertRaises(ValueError): + self.db.lookup("455.455.455.455") + + def test_verify(self): + """ + Verify the database + """ + # Path to the signature file + path = os.path.join(TEST_DATA_DIR, "signing-key.pem") + + # Try to verify with an invalid signature + with self.assertRaises(TypeError): + self.db.verify(None) + + # Perform verification with the correct key + with open(path, "r") as f: + self.assertTrue(self.db.verify(f)) + + # Perform verification with invalid keys + with open("/dev/null", "r") as f: + self.assertFalse(self.db.verify(f)) + + with open("/dev/urandom", "r") as f: + self.assertFalse(self.db.verify(f)) + + def test_search_as(self): + """ + Try to fetch an AS + """ + # Fetch an existing AS + self.assertIsInstance(self.db.get_as(204867), location.AS) + + # Fetch a non-existing AS + self.assertIsNone(self.db.get_as(0)) + + # Fetch an AS with a number that is out of range + with self.assertRaises(OverflowError): + self.db.get_as(2**32 + 1) + + def test_get_country(self): + """ + Try fetching a country + """ + # Fetch an existing country + self.assertIsInstance(self.db.get_country("DE"), location.Country) + + # Fetch a non-existing country + self.assertIsNone(self.db.get_country("AA")) + + # Fetch a country with an invalid country code + with self.assertRaises(ValueError): + self.db.get_country("XXX") + + def test_list_bogons(self): + """ + Generate a list of bogons + """ + # Fetch all bogons + bogons = self.db.list_bogons() + + # We should have received an enumerator full of networks + self.assertIsInstance(bogons, location.DatabaseEnumerator) + for bogon in bogons: + self.assertIsInstance(bogon, location.Network) + + +if __name__ == "__main__": + unittest.main()