/*.db.xz
/libtool
/stamp-h1
+/src/cron/location-update
/src/scripts/location
/src/scripts/location-importer
/src/systemd/location-update.service
-e 's,@databasedir\@,$(databasedir),g' \
< $< > $@ || rm $@
+cron_dailydir = $(sysconfdir)/cron.daily
databasedir = $(localstatedir)/lib/location
pkgconfigdir = $(libdir)/pkgconfig
# ------------------------------------------------------------------------------
+# Use systemd timers if available
if HAVE_SYSTEMD
systemdsystemunit_DATA = \
src/systemd/location-update.service \
INSTALL_DIRS += \
$(systemdsystemunitdir)
+
+# Otherwise fall back to cron
+else
+cron_daily_SCRIPTS = \
+ src/cron/location-update
+
+CLEANFILES += \
+ $(cron_daily_DATA)
endif
EXTRA_DIST += \
+ src/cron/location-update.in \
src/systemd/location-update.service.in \
src/systemd/location-update.timer.in
$(dist_check_SCRIPTS)
EXTRA_DIST += \
- tests/data/location-2022-03-30.db
+ tests/data/location-2022-03-30.db \
+ tests/data/signing-key.pem
CLEANFILES += \
testdata.db
$(PYTHON) $< $@
dist_check_SCRIPTS = \
+ tests/python/test-database.py \
tests/python/test-export.py
check_PROGRAMS = \
AC_PREREQ(2.60)
AC_INIT([libloc],
- [0.9.14],
+ [0.9.15],
[location@lists.ipfire.org],
[libloc],
[https://location.ipfire.org/])
dist-xz
subdir-objects
])
-AC_PROG_CC_STDC
+AC_PROG_CC
AC_USE_SYSTEM_EXTENSIONS
AC_SYS_LARGEFILE
AC_CONFIG_MACRO_DIR([m4])
htobe16 \
htobe32 \
htobe64 \
+ madvise \
mmap \
munmap \
res_query \
# ------------------------------------------------------------------------------
+AC_ARG_WITH([database-path],
+ AS_HELP_STRING([--with-database-path], [The default database path]),
+ [], [with_database_path=/var/lib/${PACKAGE_NAME}/database.db]
+)
+
+if test -z "${with_database_path}"; then
+ AC_MSG_ERROR([The default database path is empty])
+fi
+
+AC_DEFINE_UNQUOTED([LIBLOC_DEFAULT_DATABASE_PATH], ["${with_database_path}"],
+ [The default path for the database])
+AC_SUBST([DEFAULT_DATABASE_PATH], [${with_database_path}])
+
AC_ARG_WITH([systemd],
AS_HELP_STRING([--with-systemd], [Enable systemd support.])
)
cflags: ${CFLAGS}
ldflags: ${LDFLAGS}
+ database path: ${with_database_path}
debug: ${enable_debug}
systemd support: ${have_systemd}
+libloc (0.9.15-1) unstable; urgency=medium
+
+ [ Peter Müller ]
+ * Non-maintainer upload.
+ * location-importer.in: Fix dangling variable
+
+ [ Michael Tremer ]
+ * Replace strerror(errno) with %m in format string throughout
+ * Don't abuse errno as return code
+ * country: Refactor storing country code and continent code
+ * *_unref: Always expect a valid pointer
+ * cron: Add a cronjob if systemd is not available
+ * Check return value of fread() when reading header
+ * configure: Replace obsolete AC_PROG_CC_STDC macro
+ * writer: Check if stringpool has been initialized before free
+ * database: Use MAP_PRIVATE with mmap()
+ * database: Do not try to unmap failed mappings
+ * database: Log any errors when mmap() fails
+ * database: Increase page size to 64k
+ * python: Correctly raise any errors when opening the database
+ * database: Improve error reporting when the magic cannot be read
+ * python: Fix errors for Database.lookup()
+ * stringpool: Implement mmap as optional
+ * database: Fall back when mmap() isn't available
+ * tests: Add some simple database tests
+ * stringpool: Drop function to find next offset
+ * database: country: Return better error codes
+ * python: Export DatabaseEnumerator type
+ * tests: database: Expand test coverage
+ * database: Refactor error handling on create
+ * database: Break opening procedure into smaller parts
+ * database: Refactor checking magic
+ * database: Check if this version of libloc supports the database
+ format
+ * database: Map the entire database into memory as a whole
+ * database: Read header from mapped data
+ * hexdump: Don't try to dump any empty memory
+ * database: Read all data from the large mmap()
+ * database: Call madvise() to tell the kernel that we will randomly
+ access the data
+ * database: Encourage the compiler to inline some functions
+ * database: Drop unused offset variable in objects
+ * database: Drop debug line
+ * database: Initialize r on create
+ * tests: Add signing key to verify signatures
+ * configure: Check for madvise
+ * Fix compilation on MacOS X
+ * country: Drop unused CC_LEN
+ * tests: country: Don't crash when a country could not be found
+ * Revert "database: Increase page size to 64k"
+ * writer: Flush everything to disk after writing finishes
+ * configure: Make the default database path configurable
+ * python: Add new open() interface to easily open a database
+
+ -- Michael Tremer <michael.tremer@ipfire.org> Mon, 26 Sep 2022 15:36:44 +0000
+
libloc (0.9.14-1) unstable; urgency=medium
[ Michael Tremer ]
#include <string.h>
#include <libloc/address.h>
+#include <libloc/compat.h>
#define LOC_ADDRESS_BUFFERS 6
#define LOC_ADDRESS_BUFFER_LENGTH INET6_ADDRSTRLEN
Lesser General Public License for more details.
*/
-#include <errno.h>
#include <stdlib.h>
#include <libloc/as.h>
#include <libloc/as-list.h>
+#include <libloc/compat.h>
#include <libloc/private.h>
struct loc_as_list {
struct loc_as** elements = reallocarray(list->elements,
list->elements_size + size, sizeof(*list->elements));
if (!elements)
- return -errno;
+ return 1;
list->elements = elements;
list->elements_size += size;
struct loc_as_list** list) {
struct loc_as_list* l = calloc(1, sizeof(*l));
if (!l)
- return -ENOMEM;
+ return 1;
l->ctx = loc_ref(ctx);
l->refcount = 1;
LOC_EXPORT int loc_as_new(struct loc_ctx* ctx, struct loc_as** as, uint32_t number) {
struct loc_as* a = calloc(1, sizeof(*a));
if (!a)
- return -ENOMEM;
+ return 1;
a->ctx = loc_ref(ctx);
a->refcount = 1;
#include <errno.h>
#include <stdlib.h>
+#include <libloc/compat.h>
#include <libloc/country.h>
#include <libloc/country-list.h>
#include <libloc/private.h>
struct loc_country** elements = reallocarray(list->elements,
list->elements_size + size, sizeof(*list->elements));
if (!elements)
- return -errno;
+ return 1;
list->elements = elements;
list->elements_size += size;
}
LOC_EXPORT struct loc_country_list* loc_country_list_unref(struct loc_country_list* list) {
- if (!list)
- return NULL;
-
if (--list->refcount > 0)
return list;
// Ignore invalid country codes which would never match
if (errno == EINVAL)
return 0;
- else
- return r;
+
+ return r;
}
r = loc_country_list_contains(list, country);
struct loc_ctx* ctx;
int refcount;
- char* code;
- char* continent_code;
+ // Store the country code in a 3 byte buffer. Two bytes for the code, and NULL so
+ // that we can use strcmp() and return a pointer.
+ char code[3];
+ char continent_code[3];
char* name;
};
struct loc_country* c = calloc(1, sizeof(*c));
if (!c)
- return -ENOMEM;
+ return 1;
c->ctx = loc_ref(ctx);
c->refcount = 1;
- c->code = strdup(country_code);
+ // Set the country code
+ loc_country_code_copy(c->code, country_code);
DEBUG(c->ctx, "Country %s allocated at %p\n", c->code, c);
*country = c;
static void loc_country_free(struct loc_country* country) {
DEBUG(country->ctx, "Releasing country %s %p\n", country->code, country);
- if (country->code)
- free(country->code);
-
- if (country->continent_code)
- free(country->continent_code);
-
if (country->name)
free(country->name);
return NULL;
loc_country_free(country);
-
return NULL;
}
}
LOC_EXPORT int loc_country_set_continent_code(struct loc_country* country, const char* continent_code) {
- // XXX validate input
-
- // Free previous value
- if (country->continent_code)
- free(country->continent_code);
+ // Check for valid input
+ if (!continent_code || strlen(continent_code) != 2) {
+ errno = EINVAL;
+ return 1;
+ }
- country->continent_code = strdup(continent_code);
+ // Store the code
+ loc_country_code_copy(country->continent_code, continent_code);
return 0;
}
if (country->name)
free(country->name);
- if (name)
+ if (name) {
country->name = strdup(name);
+ // Report error if we could not copy the string
+ if (!country->name)
+ return 1;
+ }
+
return 0;
}
LOC_EXPORT int loc_country_cmp(struct loc_country* country1, struct loc_country* country2) {
- return strcmp(country1->code, country2->code);
+ return strncmp(country1->code, country2->code, 2);
}
int loc_country_new_from_database_v1(struct loc_ctx* ctx, struct loc_stringpool* pool,
struct loc_country** country, const struct loc_database_country_v1* dbobj) {
- char buffer[3];
+ char buffer[3] = "XX";
// Read country code
loc_country_code_copy(buffer, dbobj->code);
- // Terminate buffer
- buffer[2] = '\0';
-
// Create a new country object
int r = loc_country_new(ctx, country, buffer);
if (r)
return r;
- // Continent Code
- loc_country_code_copy(buffer, dbobj->continent_code);
-
- r = loc_country_set_continent_code(*country, buffer);
- if (r)
- goto FAIL;
+ // Copy continent code
+ if (*dbobj->continent_code)
+ loc_country_code_copy((*country)->continent_code, dbobj->continent_code);
// Set name
const char* name = loc_stringpool_get(pool, be32toh(dbobj->name));
int loc_country_to_database_v1(struct loc_country* country,
struct loc_stringpool* pool, struct loc_database_country_v1* dbobj) {
+ off_t name = 0;
+
// Add country code
- for (unsigned int i = 0; i < 2; i++) {
- dbobj->code[i] = country->code[i] ? country->code[i] : '\0';
- }
+ if (*country->code)
+ loc_country_code_copy(dbobj->code, country->code);
// Add continent code
- if (country->continent_code) {
- for (unsigned int i = 0; i < 2; i++) {
- dbobj->continent_code[i] = country->continent_code[i] ? country->continent_code[i] : '\0';
- }
- }
+ if (*country->continent_code)
+ loc_country_code_copy(dbobj->continent_code, country->continent_code);
// Save the name string in the string pool
- off_t name = loc_stringpool_add(pool, country->name ? country->name : "");
+ if (country->name)
+ name = loc_stringpool_add(pool, country->name);
+
dbobj->name = htobe32(name);
return 0;
// Return flags for any known special country
for (const struct loc_special_country* country = loc_special_countries;
country->flags; country++) {
- if (strcmp(country->code, cc) == 0)
+ if (strncmp(country->code, cc, 2) == 0)
return country->flags;
}
--- /dev/null
+#!/bin/bash
+###############################################################################
+# #
+# libloc - A library to determine the location of someone on the Internet #
+# #
+# Copyright (C) 2022 IPFire Development Team <info@ipfire.org> #
+# #
+# This library is free software; you can redistribute it and/or #
+# modify it under the terms of the GNU Lesser General Public #
+# License as published by the Free Software Foundation; either #
+# version 2.1 of the License, or (at your option) any later version. #
+# #
+# This library is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
+# Lesser General Public License for more details. #
+# #
+###############################################################################
+
+# Call the location database updater
+exec @bindir@/location update
#include <libloc/private.h>
#include <libloc/stringpool.h>
+struct loc_database_objects {
+ char* data;
+ size_t length;
+ size_t count;
+};
+
+struct loc_database_signature {
+ const char* data;
+ size_t length;
+};
+
struct loc_database {
struct loc_ctx* ctx;
int refcount;
off_t license;
// Signatures
- char* signature1;
- size_t signature1_length;
- char* signature2;
- size_t signature2_length;
+ struct loc_database_signature signature1;
+ struct loc_database_signature signature2;
+
+ // Data mapped into memory
+ char* data;
+ off_t length;
+
+ struct loc_stringpool* pool;
// ASes in the database
- struct loc_database_as_v1* as_v1;
- size_t as_count;
+ struct loc_database_objects as_objects;
// Network tree
- struct loc_database_network_node_v1* network_nodes_v1;
- size_t network_nodes_count;
+ struct loc_database_objects network_node_objects;
// Networks
- struct loc_database_network_v1* networks_v1;
- size_t networks_count;
+ struct loc_database_objects network_objects;
// Countries
- struct loc_database_country_v1* countries_v1;
- size_t countries_count;
-
- struct loc_stringpool* pool;
+ struct loc_database_objects country_objects;
};
#define MAX_STACK_DEPTH 256
struct in6_addr gap4_start;
};
-static int loc_database_read_magic(struct loc_database* db) {
+/*
+ Checks if it is safe to read the buffer of size length starting at p.
+*/
+#define loc_database_check_boundaries(db, p) \
+ __loc_database_check_boundaries(db, (const char*)p, sizeof(*p))
+
+static inline int __loc_database_check_boundaries(struct loc_database* db,
+ const char* p, const size_t length) {
+ size_t offset = p - db->data;
+
+ // Return if everything is within the boundary
+ if (offset <= db->length - length)
+ return 1;
+
+ DEBUG(db->ctx, "Database read check failed at %p for %zu byte(s)\n", p, length);
+ DEBUG(db->ctx, " p = %p (offset = %jd, length = %zu)\n", p, offset, length);
+ DEBUG(db->ctx, " data = %p (length = %zu)\n", db->data, db->length);
+ DEBUG(db->ctx, " end = %p\n", db->data + db->length);
+ DEBUG(db->ctx, " overflow of %zu byte(s)\n", offset + length - db->length);
+
+ // Otherwise raise EFAULT
+ errno = EFAULT;
+ return 0;
+}
+
+/*
+ Returns a pointer to the n-th object
+*/
+static inline char* loc_database_object(struct loc_database* db,
+ const struct loc_database_objects* objects, const size_t length, const off_t n) {
+ // Calculate offset
+ const off_t offset = n * length;
+
+ // Return a pointer to where the object lies
+ char* object = objects->data + offset;
+
+ // Check if the object is part of the memory
+ if (!__loc_database_check_boundaries(db, object, length))
+ return NULL;
+
+ return object;
+}
+
+static int loc_database_version_supported(struct loc_database* db, uint8_t version) {
+ switch (version) {
+ // Supported versions
+ case LOC_DATABASE_VERSION_1:
+ return 1;
+
+ default:
+ ERROR(db->ctx, "Database version %d is not supported\n", version);
+ errno = ENOTSUP;
+ return 0;
+ }
+}
+
+static int loc_database_check_magic(struct loc_database* db) {
struct loc_database_magic magic;
// Read from file
if (bytes_read < sizeof(magic)) {
ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
- return -ENOMSG;
+ goto ERROR;
}
// Compare magic bytes
- if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
+ if (memcmp(magic.magic, LOC_DATABASE_MAGIC, sizeof(magic.magic)) == 0) {
DEBUG(db->ctx, "Magic value matches\n");
+ // Do we support this version?
+ if (!loc_database_version_supported(db, magic.version))
+ return 1;
+
// Parse version
db->version = magic.version;
return 0;
}
+ERROR:
ERROR(db->ctx, "Unrecognized file type\n");
+ errno = ENOMSG;
// Return an error
return 1;
}
-static int loc_database_read_as_section_v1(struct loc_database* db,
- const struct loc_database_header_v1* header) {
- off_t as_offset = be32toh(header->as_offset);
- size_t as_length = be32toh(header->as_length);
-
- DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", (intmax_t)as_offset, as_length);
+/*
+ Maps the entire database into memory
+*/
+static int loc_database_mmap(struct loc_database* db) {
+ int r;
- if (as_length > 0) {
- db->as_v1 = mmap(NULL, as_length, PROT_READ,
- MAP_SHARED, fileno(db->f), as_offset);
+ // Get file descriptor
+ int fd = fileno(db->f);
- if (db->as_v1 == MAP_FAILED)
- return -errno;
+ // Determine the length of the database
+ db->length = lseek(fd, 0, SEEK_END);
+ if (db->length < 0) {
+ ERROR(db->ctx, "Could not determine the length of the database: %m\n");
+ return 1;
}
- db->as_count = as_length / sizeof(*db->as_v1);
-
- INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
-
- return 0;
-}
-
-static int loc_database_read_network_nodes_section_v1(struct loc_database* db,
- const struct loc_database_header_v1* header) {
- off_t network_nodes_offset = be32toh(header->network_tree_offset);
- size_t network_nodes_length = be32toh(header->network_tree_length);
-
- DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n",
- (intmax_t)network_nodes_offset, network_nodes_length);
-
- if (network_nodes_length > 0) {
- db->network_nodes_v1 = mmap(NULL, network_nodes_length, PROT_READ,
- MAP_SHARED, fileno(db->f), network_nodes_offset);
+ rewind(db->f);
- if (db->network_nodes_v1 == MAP_FAILED)
- return -errno;
+ // Map all data
+ db->data = mmap(NULL, db->length, PROT_READ, MAP_SHARED, fd, 0);
+ if (db->data == MAP_FAILED) {
+ ERROR(db->ctx, "Could not map the database: %m\n");
+ db->data = NULL;
+ return 1;
}
- db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v1);
-
- INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count);
+ DEBUG(db->ctx, "Mapped database of %zu byte(s) at %p\n", db->length, db->data);
- return 0;
-}
-
-static int loc_database_read_networks_section_v1(struct loc_database* db,
- const struct loc_database_header_v1* header) {
- off_t networks_offset = be32toh(header->network_data_offset);
- size_t networks_length = be32toh(header->network_data_length);
-
- DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n",
- (intmax_t)networks_offset, networks_length);
-
- if (networks_length > 0) {
- db->networks_v1 = mmap(NULL, networks_length, PROT_READ,
- MAP_SHARED, fileno(db->f), networks_offset);
-
- if (db->networks_v1 == MAP_FAILED)
- return -errno;
+ // Tell the system that we expect to read data randomly
+ r = madvise(db->data, db->length, MADV_RANDOM);
+ if (r) {
+ ERROR(db->ctx, "madvise() failed: %m\n");
+ return r;
}
- db->networks_count = networks_length / sizeof(*db->networks_v1);
-
- INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count);
-
return 0;
}
-static int loc_database_read_countries_section_v1(struct loc_database* db,
- const struct loc_database_header_v1* header) {
- off_t countries_offset = be32toh(header->countries_offset);
- size_t countries_length = be32toh(header->countries_length);
-
- DEBUG(db->ctx, "Reading countries section from %jd (%zu bytes)\n",
- (intmax_t)countries_offset, countries_length);
-
- if (countries_length > 0) {
- db->countries_v1 = mmap(NULL, countries_length, PROT_READ,
- MAP_SHARED, fileno(db->f), countries_offset);
-
- if (db->countries_v1 == MAP_FAILED)
- return -errno;
- }
-
- db->countries_count = countries_length / sizeof(*db->countries_v1);
-
- INFO(db->ctx, "Read %zu countries from the database\n",
- db->countries_count);
+/*
+ Maps arbitrary objects from the database into memory.
+*/
+static int loc_database_map_objects(struct loc_database* db, struct loc_database_objects* objects,
+ const size_t size, const off_t offset, const size_t length) {
+ // Store parameters
+ objects->data = db->data + offset;
+ objects->length = length;
+ objects->count = objects->length / size;
return 0;
}
static int loc_database_read_signature(struct loc_database* db,
- char** dst, char* src, size_t length) {
+ struct loc_database_signature* signature, const char* data, const size_t length) {
// Check for a plausible signature length
if (length > LOC_SIGNATURE_MAX_LENGTH) {
ERROR(db->ctx, "Signature too long: %zu\n", length);
- return -EINVAL;
+ errno = EINVAL;
+ return 1;
}
- DEBUG(db->ctx, "Reading signature of %zu bytes\n", length);
+ // Store data & length
+ signature->data = data;
+ signature->length = length;
- // Allocate space
- *dst = malloc(length);
- if (!*dst)
- return -ENOMEM;
+ DEBUG(db->ctx, "Read signature of %zu byte(s) at %p\n",
+ signature->length, signature->data);
- // Copy payload
- memcpy(*dst, src, length);
+ hexdump(db->ctx, signature->data, signature->length);
return 0;
}
static int loc_database_read_header_v1(struct loc_database* db) {
- struct loc_database_header_v1 header;
+ const struct loc_database_header_v1* header =
+ (const struct loc_database_header_v1*)(db->data + LOC_DATABASE_MAGIC_SIZE);
int r;
- // Read from file
- size_t size = fread(&header, 1, sizeof(header), db->f);
+ DEBUG(db->ctx, "Reading header at %p\n", header);
- if (size < sizeof(header)) {
+ // Check if we can read the header
+ if (!loc_database_check_boundaries(db, header)) {
ERROR(db->ctx, "Could not read enough data for header\n");
- return -ENOMSG;
+ return 1;
}
- // Copy over data
- db->created_at = be64toh(header.created_at);
- db->vendor = be32toh(header.vendor);
- db->description = be32toh(header.description);
- db->license = be32toh(header.license);
+ // Dump the entire header
+ hexdump(db->ctx, header, sizeof(*header));
- db->signature1_length = be16toh(header.signature1_length);
- db->signature2_length = be16toh(header.signature2_length);
+ // Copy over data
+ db->created_at = be64toh(header->created_at);
+ db->vendor = be32toh(header->vendor);
+ db->description = be32toh(header->description);
+ db->license = be32toh(header->license);
// Read signatures
- if (db->signature1_length) {
- r = loc_database_read_signature(db, &db->signature1,
- header.signature1, db->signature1_length);
- if (r)
- return r;
- }
+ r = loc_database_read_signature(db, &db->signature1,
+ header->signature1, be16toh(header->signature1_length));
+ if (r)
+ return r;
- if (db->signature2_length) {
- r = loc_database_read_signature(db, &db->signature2,
- header.signature2, db->signature2_length);
- if (r)
- return r;
- }
+ r = loc_database_read_signature(db, &db->signature2,
+ header->signature2, be16toh(header->signature2_length));
+ if (r)
+ return r;
- // Open pool
- off_t pool_offset = be32toh(header.pool_offset);
- size_t pool_length = be32toh(header.pool_length);
+ const char* stringpool_start = db->data + be32toh(header->pool_offset);
+ size_t stringpool_length = be32toh(header->pool_length);
- r = loc_stringpool_open(db->ctx, &db->pool,
- db->f, pool_length, pool_offset);
+ // Check if the stringpool is part of the mapped area
+ if (!__loc_database_check_boundaries(db, stringpool_start, stringpool_length))
+ return 1;
+
+ // Open the stringpool
+ r = loc_stringpool_open(db->ctx, &db->pool, stringpool_start, stringpool_length);
if (r)
return r;
- // AS section
- r = loc_database_read_as_section_v1(db, &header);
+ // Map AS objects
+ r = loc_database_map_objects(db, &db->as_objects,
+ sizeof(struct loc_database_as_v1),
+ be32toh(header->as_offset),
+ be32toh(header->as_length));
if (r)
return r;
- // Network Nodes
- r = loc_database_read_network_nodes_section_v1(db, &header);
+ // Map Network Nodes
+ r = loc_database_map_objects(db, &db->network_node_objects,
+ sizeof(struct loc_database_network_node_v1),
+ be32toh(header->network_tree_offset),
+ be32toh(header->network_tree_length));
if (r)
return r;
- // Networks
- r = loc_database_read_networks_section_v1(db, &header);
+ // Map Networks
+ r = loc_database_map_objects(db, &db->network_objects,
+ sizeof(struct loc_database_network_v1),
+ be32toh(header->network_data_offset),
+ be32toh(header->network_data_length));
if (r)
return r;
- // countries
- r = loc_database_read_countries_section_v1(db, &header);
+ // Map countries
+ r = loc_database_map_objects(db, &db->country_objects,
+ sizeof(struct loc_database_country_v1),
+ be32toh(header->countries_offset),
+ be32toh(header->countries_length));
if (r)
return r;
}
}
-static int loc_database_read(struct loc_database* db, FILE* f) {
- clock_t start = clock();
-
+static int loc_database_clone_handle(struct loc_database* db, FILE* f) {
+ // Fetch the FD of the original handle
int fd = fileno(f);
// Clone file descriptor
fd = dup(fd);
if (!fd) {
ERROR(db->ctx, "Could not duplicate file descriptor\n");
- return -1;
+ return 1;
}
// Reopen the file so that we can keep our own file handle
db->f = fdopen(fd, "r");
if (!db->f) {
ERROR(db->ctx, "Could not re-open database file\n");
- return -1;
+ return 1;
}
// Rewind to the start of the file
rewind(db->f);
+ return 0;
+}
+
+static int loc_database_open(struct loc_database* db, FILE* f) {
+ int r;
+
+ clock_t start = clock();
+
+ // Clone the file handle
+ r = loc_database_clone_handle(db, f);
+ if (r)
+ return r;
+
// Read magic bytes
- int r = loc_database_read_magic(db);
+ r = loc_database_check_magic(db);
+ if (r)
+ return r;
+
+ // Map the database into memory
+ r = loc_database_mmap(db);
if (r)
return r;
return 0;
}
+static void loc_database_free(struct loc_database* db) {
+ int r;
+
+ DEBUG(db->ctx, "Releasing database %p\n", db);
+
+ // Unmap the entire database
+ if (db->data) {
+ r = munmap(db->data, db->length);
+ if (r)
+ ERROR(db->ctx, "Could not unmap the database: %m\n");
+ }
+
+ // Free the stringpool
+ if (db->pool)
+ loc_stringpool_unref(db->pool);
+
+ // Close database file
+ if (db->f)
+ fclose(db->f);
+
+ loc_unref(db->ctx);
+ free(db);
+}
+
LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
+ struct loc_database* db = NULL;
+ int r = 1;
+
// Fail on invalid file handle
- if (!f)
- return -EINVAL;
+ if (!f) {
+ errno = EINVAL;
+ return 1;
+ }
- struct loc_database* db = calloc(1, sizeof(*db));
+ // Allocate the database object
+ db = calloc(1, sizeof(*db));
if (!db)
- return -ENOMEM;
+ goto ERROR;
// Reference context
db->ctx = loc_ref(ctx);
DEBUG(db->ctx, "Database object allocated at %p\n", db);
- int r = loc_database_read(db, f);
- if (r) {
- loc_database_unref(db);
- return r;
- }
+ // Try to open the database
+ r = loc_database_open(db, f);
+ if (r)
+ goto ERROR;
*database = db;
-
return 0;
+
+ERROR:
+ if (db)
+ loc_database_free(db);
+
+ return r;
}
LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
return db;
}
-static void loc_database_free(struct loc_database* db) {
- int r;
-
- DEBUG(db->ctx, "Releasing database %p\n", db);
-
- // Removing all ASes
- if (db->as_v1) {
- r = munmap(db->as_v1, db->as_count * sizeof(*db->as_v1));
- if (r)
- ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno));
- }
-
- // Remove mapped network sections
- if (db->networks_v1) {
- r = munmap(db->networks_v1, db->networks_count * sizeof(*db->networks_v1));
- if (r)
- ERROR(db->ctx, "Could not unmap networks section: %s\n", strerror(errno));
- }
-
- // Remove mapped network nodes section
- if (db->network_nodes_v1) {
- r = munmap(db->network_nodes_v1, db->network_nodes_count * sizeof(*db->network_nodes_v1));
- if (r)
- ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno));
- }
-
- // Remove mapped countries section
- if (db->countries_v1) {
- r = munmap(db->countries_v1, db->countries_count * sizeof(*db->countries_v1));
- if (r)
- ERROR(db->ctx, "Could not unmap countries section: %s\n", strerror(errno));
- }
-
- if (db->pool)
- loc_stringpool_unref(db->pool);
-
- // Free signature
- if (db->signature1)
- free(db->signature1);
- if (db->signature2)
- free(db->signature2);
-
- // Close database file
- if (db->f)
- fclose(db->f);
-
- loc_unref(db->ctx);
- free(db);
-}
-
LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
if (--db->refcount > 0)
return NULL;
}
LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
+ size_t bytes_read = 0;
+
// Cannot do this when no signature is available
- if (!db->signature1 && !db->signature2) {
+ if (!db->signature1.data && !db->signature2.data) {
DEBUG(db->ctx, "No signature available to verify\n");
return 1;
}
// Load public key
EVP_PKEY* pkey = PEM_read_PUBKEY(f, NULL, NULL, NULL);
if (!pkey) {
- char* error = ERR_error_string(ERR_get_error(), NULL);
- ERROR(db->ctx, "Could not parse public key: %s\n", error);
+ ERROR(db->ctx, "Could not parse public key: %s\n",
+ ERR_error_string(ERR_get_error(), NULL));
return -1;
}
// Read magic
struct loc_database_magic magic;
- fread(&magic, 1, sizeof(magic), db->f);
+ bytes_read = fread(&magic, 1, sizeof(magic), db->f);
+ if (bytes_read < sizeof(magic)) {
+ ERROR(db->ctx, "Could not read header: %m\n");
+ r = 1;
+ goto CLEANUP;
+ }
hexdump(db->ctx, &magic, sizeof(magic));
// Read the header
struct loc_database_header_v1 header_v1;
- size_t bytes_read;
switch (db->version) {
case LOC_DATABASE_VERSION_1:
}
// Check first signature
- if (db->signature1) {
- hexdump(db->ctx, db->signature1, db->signature1_length);
+ if (db->signature1.data) {
+ hexdump(db->ctx, db->signature1.data, db->signature1.length);
r = EVP_DigestVerifyFinal(mdctx,
- (unsigned char*)db->signature1, db->signature1_length);
+ (unsigned char*)db->signature1.data, db->signature1.length);
if (r == 0) {
DEBUG(db->ctx, "The first signature is invalid\n");
}
// Check second signature only when the first one was invalid
- if (r && db->signature2) {
- hexdump(db->ctx, db->signature2, db->signature2_length);
+ if (r && db->signature2.data) {
+ hexdump(db->ctx, db->signature2.data, db->signature2.length);
r = EVP_DigestVerifyFinal(mdctx,
- (unsigned char*)db->signature2, db->signature2_length);
+ (unsigned char*)db->signature2.data, db->signature2.length);
if (r == 0) {
DEBUG(db->ctx, "The second signature is invalid\n");
}
LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
- return db->as_count;
+ return db->as_objects.count;
}
// Returns the AS at position pos
static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
- if ((size_t)pos >= db->as_count)
- return -EINVAL;
+ struct loc_database_as_v1* as_v1 = NULL;
+ int r;
+
+ if ((size_t)pos >= db->as_objects.count) {
+ errno = ERANGE;
+ return 1;
+ }
DEBUG(db->ctx, "Fetching AS at position %jd\n", (intmax_t)pos);
- int r;
switch (db->version) {
case LOC_DATABASE_VERSION_1:
- r = loc_as_new_from_database_v1(db->ctx, db->pool, as, db->as_v1 + pos);
+ // Find the object
+ as_v1 = (struct loc_database_as_v1*)loc_database_object(db,
+ &db->as_objects, sizeof(*as_v1), pos);
+ if (!as_v1)
+ return 1;
+
+ r = loc_as_new_from_database_v1(db->ctx, db->pool, as, as_v1);
break;
default:
- return -1;
+ errno = ENOTSUP;
+ return 1;
}
- if (r == 0) {
+ if (r == 0)
DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as));
- }
return r;
}
// Performs a binary search to find the AS in the list
LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) {
off_t lo = 0;
- off_t hi = db->as_count - 1;
+ off_t hi = db->as_objects.count - 1;
#ifdef ENABLE_DEBUG
// Save start time
// Returns the network at position pos
static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network,
struct in6_addr* address, unsigned int prefix, off_t pos) {
- if ((size_t)pos >= db->networks_count) {
+ struct loc_database_network_v1* network_v1 = NULL;
+ int r;
+
+ if ((size_t)pos >= db->network_objects.count) {
DEBUG(db->ctx, "Network ID out of range: %jd/%jd\n",
- (intmax_t)pos, (intmax_t)db->networks_count);
- return -EINVAL;
+ (intmax_t)pos, (intmax_t)db->network_objects.count);
+ errno = ERANGE;
+ return 1;
}
-
DEBUG(db->ctx, "Fetching network at position %jd\n", (intmax_t)pos);
- int r;
switch (db->version) {
case LOC_DATABASE_VERSION_1:
- r = loc_network_new_from_database_v1(db->ctx, network,
- address, prefix, db->networks_v1 + pos);
+ // Read the object
+ network_v1 = (struct loc_database_network_v1*)loc_database_object(db,
+ &db->network_objects, sizeof(*network_v1), pos);
+ if (!network_v1)
+ return 1;
+
+ r = loc_network_new_from_database_v1(db->ctx, network, address, prefix, network_v1);
break;
default:
- return -1;
+ errno = ENOTSUP;
+ return 1;
}
if (r == 0)
const struct loc_database_network_node_v1* node) {
off_t network_index = be32toh(node->network);
- DEBUG(db->ctx, "Handling leaf node at %jd (%jd)\n", (intmax_t)(node - db->network_nodes_v1), (intmax_t)network_index);
+ DEBUG(db->ctx, "Handling leaf node at %jd\n", (intmax_t)network_index);
// Fetch the network
- int r = loc_database_fetch_network(db, network,
- network_address, prefix, network_index);
+ int r = loc_database_fetch_network(db, network, network_address, prefix, network_index);
if (r) {
- ERROR(db->ctx, "Could not fetch network %jd from database\n", (intmax_t)network_index);
+ ERROR(db->ctx, "Could not fetch network %jd from database: %m\n",
+ (intmax_t)network_index);
return r;
}
// Searches for an exact match along the path
static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* address,
struct loc_network** network, struct in6_addr* network_address,
- const struct loc_database_network_node_v1* node, unsigned int level) {
+ off_t node_index, unsigned int level) {
+ struct loc_database_network_node_v1* node_v1 = NULL;
+
int r;
- off_t node_index;
+
+ // Fetch the next node
+ node_v1 = (struct loc_database_network_node_v1*)loc_database_object(db,
+ &db->network_node_objects, sizeof(*node_v1), node_index);
+ if (!node_v1)
+ return 1;
// Follow the path
int bit = loc_address_get_bit(address, level);
loc_address_set_bit(network_address, level, bit);
if (bit == 0)
- node_index = be32toh(node->zero);
+ node_index = be32toh(node_v1->zero);
else
- node_index = be32toh(node->one);
+ node_index = be32toh(node_v1->one);
// If the node index is zero, the tree ends here
// and we cannot descend any further
if (node_index > 0) {
// Check boundaries
- if ((size_t)node_index >= db->network_nodes_count)
- return -EINVAL;
+ if ((size_t)node_index >= db->network_node_objects.count) {
+ errno = ERANGE;
+ return 1;
+ }
// Move on to the next node
- r = __loc_database_lookup(db, address, network, network_address,
- db->network_nodes_v1 + node_index, level + 1);
+ r = __loc_database_lookup(db, address, network, network_address, node_index, level + 1);
// End here if a result was found
if (r == 0)
}
// If this node has a leaf, we will check if it matches
- if (__loc_database_node_is_leaf(node)) {
- r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node);
+ if (__loc_database_node_is_leaf(node_v1)) {
+ r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node_v1);
if (r <= 0)
return r;
}
clock_t start = clock();
#endif
- int r = __loc_database_lookup(db, address, network, &network_address,
- db->network_nodes_v1, 0);
+ int r = __loc_database_lookup(db, address, network, &network_address, 0, 0);
#ifdef ENABLE_DEBUG
clock_t end = clock();
// Returns the country at position pos
static int loc_database_fetch_country(struct loc_database* db,
struct loc_country** country, off_t pos) {
- if ((size_t)pos >= db->countries_count)
- return -EINVAL;
+ struct loc_database_country_v1* country_v1 = NULL;
+ int r;
+
+ // Check if the country is within range
+ if ((size_t)pos >= db->country_objects.count) {
+ errno = ERANGE;
+ return 1;
+ }
DEBUG(db->ctx, "Fetching country at position %jd\n", (intmax_t)pos);
- int r;
switch (db->version) {
case LOC_DATABASE_VERSION_1:
- r = loc_country_new_from_database_v1(db->ctx, db->pool, country, db->countries_v1 + pos);
+ // Read the object
+ country_v1 = (struct loc_database_country_v1*)loc_database_object(db,
+ &db->country_objects, sizeof(*country_v1), pos);
+ if (!country_v1)
+ return 1;
+
+ r = loc_country_new_from_database_v1(db->ctx, db->pool, country, country_v1);
break;
default:
- return -1;
+ errno = ENOTSUP;
+ return 1;
}
- if (r == 0) {
+ if (r == 0)
DEBUG(db->ctx, "Got country %s\n", loc_country_get_code(*country));
- }
return r;
}
LOC_EXPORT int loc_database_get_country(struct loc_database* db,
struct loc_country** country, const char* code) {
off_t lo = 0;
- off_t hi = db->countries_count - 1;
+ off_t hi = db->country_objects.count - 1;
+
+ // Check if the country code is valid
+ if (!loc_country_code_is_valid(code)) {
+ errno = EINVAL;
+ return 1;
+ }
#ifdef ENABLE_DEBUG
// Save start time
// Nothing found
*country = NULL;
- return 1;
+ return 0;
}
// Enumerator
loc_as_list_unref(enumerator->asns);
// Free network search
- free(enumerator->networks_visited);
+ if (enumerator->networks_visited)
+ free(enumerator->networks_visited);
// Free subnet/bogons stack
if (enumerator->stack)
LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
struct loc_database* db, enum loc_database_enumerator_mode mode, int flags) {
+ int r;
+
struct loc_database_enumerator* e = calloc(1, sizeof(*e));
- if (!e)
+ if (!e) {
return -ENOMEM;
+ }
// Reference context
e->ctx = loc_ref(db->ctx);
// Initialise graph search
e->network_stack_depth = 1;
- e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
+ e->networks_visited = calloc(db->network_node_objects.count, sizeof(*e->networks_visited));
+ if (!e->networks_visited) {
+ ERROR(db->ctx, "Could not allocated visited networks: %m\n");
+ r = 1;
+ goto ERROR;
+ }
// Allocate stack
- int r = loc_network_list_new(e->ctx, &e->stack);
- if (r) {
- loc_database_enumerator_free(e);
- return r;
- }
+ r = loc_network_list_new(e->ctx, &e->stack);
+ if (r)
+ goto ERROR;
// Initialize bogon search
loc_address_reset(&e->gap6_start, AF_INET6);
*enumerator = e;
return 0;
+
+ERROR:
+ if (e)
+ loc_database_enumerator_free(e);
+
+ return r;
}
LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator) {
struct loc_database* db = enumerator->db;
- while (enumerator->as_index < db->as_count) {
+ while (enumerator->as_index < db->as_objects.count) {
// Fetch the next AS
int r = loc_database_fetch_as(db, as, enumerator->as_index++);
if (r)
// Check if there is any space left on the stack
if (e->network_stack_depth >= MAX_STACK_DEPTH) {
ERROR(e->ctx, "Maximum stack size reached: %d\n", e->network_stack_depth);
- return -1;
+ return 1;
+ }
+
+ // Check if the node is in range
+ if (offset >= (off_t)e->db->network_node_objects.count) {
+ ERROR(e->ctx, "Trying to add invalid node with offset %jd/%zu\n",
+ offset, e->db->network_node_objects.count);
+ errno = ERANGE;
+ return 1;
}
// Increase stack size
// Get object from top of the stack
struct loc_node_stack* node = &enumerator->network_stack[enumerator->network_stack_depth];
+ DEBUG(enumerator->ctx, " Got node: %jd\n", node->offset);
+
// Remove the node from the stack if we have already visited it
if (enumerator->networks_visited[node->offset]) {
enumerator->network_stack_depth--;
// Pop node from top of the stack
struct loc_database_network_node_v1* n =
- enumerator->db->network_nodes_v1 + node->offset;
+ (struct loc_database_network_node_v1*)loc_database_object(enumerator->db,
+ &enumerator->db->network_node_objects, sizeof(*n), node->offset);
+ if (!n)
+ return 1;
// Add edges to stack
int r = loc_database_enumerator_stack_push_node(enumerator,
be32toh(n->one), 1, node->depth + 1);
-
if (r)
return r;
r = loc_database_enumerator_stack_push_node(enumerator,
be32toh(n->zero), 0, node->depth + 1);
-
if (r)
return r;
struct loc_database* db = enumerator->db;
- while (enumerator->country_index < db->countries_count) {
+ while (enumerator->country_index < db->country_objects.count) {
// Fetch the next country
int r = loc_database_fetch_country(db, country, enumerator->country_index++);
if (r)
#include <stddef.h>
#include <stdarg.h>
#include <unistd.h>
-#include <errno.h>
#include <string.h>
#include <ctype.h>
LOC_EXPORT int loc_new(struct loc_ctx** ctx) {
struct loc_ctx* c = calloc(1, sizeof(*c));
if (!c)
- return -ENOMEM;
+ return 1;
c->refcount = 1;
c->log_fn = log_stderr;
}
LOC_EXPORT struct loc_ctx* loc_unref(struct loc_ctx* ctx) {
- if (!ctx)
- return NULL;
-
if (--ctx->refcount > 0)
return NULL;
#include <errno.h>
#include <netinet/in.h>
+#include <libloc/compat.h>
+
/*
All of these functions are private and for internal use only
*/
# define s6_addr32 __u6_addr.__u6_addr32
#endif
+#ifndef reallocarray
+# define reallocarray(ptr, nmemb, size) realloc(ptr, nmemb * size)
+#endif
+
#endif
#endif
#include <stdint.h>
#define LOC_DATABASE_MAGIC "LOCDBXX"
+#define LOC_DATABASE_MAGIC_SIZE sizeof(struct loc_database_magic)
enum loc_database_version {
LOC_DATABASE_VERSION_UNSET = 0,
#define LOC_DATABASE_DOMAIN "_v%u._db.location.ipfire.org"
#define LOC_DATABASE_PAGE_SIZE 4096
-#define LOC_SIGNATURE_MAX_LENGTH (LOC_DATABASE_PAGE_SIZE / 2)
+#define LOC_SIGNATURE_MAX_LENGTH 2048
struct loc_database_magic {
char magic[7];
DEBUG(ctx, "Dumping %zu byte(s)\n", len);
+ if (!len)
+ return;
+
// Process every byte in the data
for (i = 0; i < len; i++) {
// Multiple of 16 means new line (with line offset)
struct loc_stringpool;
int loc_stringpool_new(struct loc_ctx* ctx, struct loc_stringpool** pool);
int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
- FILE* f, size_t length, off_t offset);
+ const char* data, const size_t length);
struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool);
struct loc_stringpool* loc_stringpool_unref(struct loc_stringpool* pool);
struct loc_network** elements = reallocarray(list->elements,
list->elements_size + size, sizeof(*list->elements));
if (!elements)
- return -errno;
+ return 1;
list->elements = elements;
list->elements_size += size;
}
LOC_EXPORT struct loc_network_list* loc_network_list_unref(struct loc_network_list* list) {
- if (!list)
- return NULL;
-
if (--list->refcount > 0)
return list;
}
struct loc_network* n = calloc(1, sizeof(*n));
- if (!n) {
- errno = ENOMEM;
+ if (!n)
return 1;
- }
n->ctx = loc_ref(ctx);
n->refcount = 1;
}
LOC_EXPORT struct loc_network* loc_network_unref(struct loc_network* network) {
- if (!network)
- return NULL;
-
if (--network->refcount > 0)
return network;
int r = loc_network_new(ctx, network, address, prefix);
if (r) {
- ERROR(ctx, "Could not allocate a new network: %s", strerror(-r));
+ ERROR(ctx, "Could not allocate a new network: %m\n");
return r;
}
int loc_network_tree_new(struct loc_ctx* ctx, struct loc_network_tree** tree) {
struct loc_network_tree* t = calloc(1, sizeof(*t));
if (!t)
- return -ENOMEM;
+ return 1;
t->ctx = loc_ref(ctx);
t->refcount = 1;
}
struct loc_network_tree_node* loc_network_tree_node_unref(struct loc_network_tree_node* node) {
- if (!node)
- return NULL;
-
if (--node->refcount > 0)
return node;
static int Database_init(DatabaseObject* self, PyObject* args, PyObject* kwargs) {
const char* path = NULL;
+ FILE* f = NULL;
+ // Parse arguments
if (!PyArg_ParseTuple(args, "s", &path))
return -1;
+ // Copy path
self->path = strdup(path);
+ if (!self->path)
+ goto ERROR;
// Open the file for reading
- FILE* f = fopen(self->path, "r");
- if (!f) {
- PyErr_SetFromErrno(PyExc_IOError);
- return -1;
- }
+ f = fopen(self->path, "r");
+ if (!f)
+ goto ERROR;
// Load the database
int r = loc_database_new(loc_ctx, &self->db, f);
- fclose(f);
-
- // Return on any errors
if (r)
- return -1;
+ goto ERROR;
+ fclose(f);
return 0;
+
+ERROR:
+ if (f)
+ fclose(f);
+
+ PyErr_SetFromErrno(PyExc_OSError);
+ return -1;
}
static PyObject* Database_repr(DatabaseObject* self) {
}
static PyObject* Database_get_country(DatabaseObject* self, PyObject* args) {
+ struct loc_country* country = NULL;
const char* country_code = NULL;
if (!PyArg_ParseTuple(args, "s", &country_code))
return NULL;
- struct loc_country* country;
+ // Fetch the country
int r = loc_database_get_country(self->db, &country, country_code);
if (r) {
- Py_RETURN_NONE;
+ switch (errno) {
+ case EINVAL:
+ PyErr_SetString(PyExc_ValueError, "Invalid country code");
+ break;
+
+ default:
+ PyErr_SetFromErrno(PyExc_OSError);
+ break;
+ }
+
+ return NULL;
}
+ // No result
+ if (!country)
+ Py_RETURN_NONE;
+
PyObject* obj = new_country(&CountryType, country);
loc_country_unref(country);
loc_network_unref(network);
return obj;
+ }
// Nothing found
- } else if (r == 1) {
+ if (!errno)
Py_RETURN_NONE;
- // Invalid input
- } else if (r == -EINVAL) {
- PyErr_Format(PyExc_ValueError, "Invalid IP address: %s", address);
- return NULL;
+ // Handle any errors
+ switch (errno) {
+ case EINVAL:
+ PyErr_Format(PyExc_ValueError, "Invalid IP address: %s", address);
+
+ default:
+ PyErr_SetFromErrno(PyExc_OSError);
}
- // Unexpected error
return NULL;
}
struct loc_as_list* asns;
r = loc_as_list_new(loc_ctx, &asns);
if (r) {
- PyErr_SetString(PyExc_SystemError, "Could not create AS list");
+ PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
struct loc_as* as;
r = loc_as_new(loc_ctx, &as, number);
if (r) {
- PyErr_SetString(PyExc_SystemError, "Could not create AS");
+ PyErr_SetFromErrno(PyExc_OSError);
loc_as_list_unref(asns);
loc_as_unref(as);
r = loc_as_list_append(asns, as);
if (r) {
- PyErr_SetString(PyExc_SystemError, "Could not append AS to the list");
+ PyErr_SetFromErrno(PyExc_OSError);
loc_as_list_unref(asns);
loc_as_unref(as);
r = loc_database_enumerator_set_asns(enumerator, asns);
if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
+ PyErr_SetFromErrno(PyExc_OSError);
loc_as_list_unref(asns);
return NULL;
r = loc_database_enumerator_set_flag(enumerator, flags);
if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
+ PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
}
r = loc_database_enumerator_set_family(enumerator, family);
if (r) {
- PyErr_SetFromErrno(PyExc_SystemError);
+ PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
}
# Initialise logging
from . import logger
+
+def open(path=None):
+ """
+ Opens the database at path, or opens the default database.
+ """
+ if not path:
+ path = DATABASE_PATH
+
+ # Open the database
+ return Database(path)
if (PyModule_AddStringConstant(m, "__version__", PACKAGE_VERSION))
return NULL;
+ // Default Database Path
+ if (PyModule_AddStringConstant(m, "DATABASE_PATH", LIBLOC_DEFAULT_DATABASE_PATH))
+ return NULL;
+
// AS
if (PyType_Ready(&ASType) < 0)
return NULL;
return NULL;
Py_INCREF(&DatabaseEnumeratorType);
- //PyModule_AddObject(m, "DatabaseEnumerator", (PyObject *)&DatabaseEnumeratorType);
+ PyModule_AddObject(m, "DatabaseEnumerator", (PyObject *)&DatabaseEnumeratorType);
// Network
if (PyType_Ready(&NetworkType) < 0)
#include <errno.h>
#include <limits.h>
+#include <libloc/compat.h>
#include <libloc/libloc.h>
#include <libloc/network.h>
#include <libloc/network-list.h>
FILE* f = fopen(path, "w+");
if (!f) {
- PyErr_Format(PyExc_IOError, strerror(errno));
+ PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
// Raise any errors
if (r) {
- PyErr_Format(PyExc_IOError, strerror(errno));
+ PyErr_SetFromErrno(PyExc_OSError);
return NULL;
}
# Iterate through every line, filter comments and add remaining ASNs to
# the override table in case they are valid...
with self.db.transaction():
- for sline in t.readlines():
+ for sline in f.readlines():
# The response is assumed to be encoded in UTF-8...
sline = sline.decode("utf-8")
# database
parser.add_argument("--database", "-d",
- default="@databasedir@/database.db", help=_("Path to database"),
+ default=location.DATABASE_PATH, help=_("Path to database"),
)
# public key
#include <libloc/private.h>
#include <libloc/stringpool.h>
-enum loc_stringpool_mode {
- STRINGPOOL_DEFAULT,
- STRINGPOOL_MMAP,
-};
+#define LOC_STRINGPOOL_BLOCK_SIZE (512 * 1024)
struct loc_stringpool {
struct loc_ctx* ctx;
int refcount;
- enum loc_stringpool_mode mode;
-
- char* data;
+ // Reference to any mapped data
+ const char* data;
ssize_t length;
- char* pos;
+ // Reference to own storage
+ char* blocks;
+ size_t size;
};
-static off_t loc_stringpool_get_offset(struct loc_stringpool* pool, const char* pos) {
- if (pos < pool->data)
- return -EFAULT;
-
- if (pos > (pool->data + pool->length))
- return -EFAULT;
-
- return pos - pool->data;
-}
-
-static char* __loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
- if (offset < 0 || offset >= pool->length)
- return NULL;
-
- return pool->data + offset;
-}
-
-static int loc_stringpool_grow(struct loc_stringpool* pool, size_t length) {
- DEBUG(pool->ctx, "Growing string pool to %zu bytes\n", length);
-
- // Save pos pointer
- off_t pos = loc_stringpool_get_offset(pool, pool->pos);
+static int loc_stringpool_grow(struct loc_stringpool* pool, const size_t size) {
+ DEBUG(pool->ctx, "Growing string pool by %zu byte(s)\n", size);
- // Reallocate data section
- pool->data = realloc(pool->data, length);
- if (!pool->data)
- return -ENOMEM;
+ // Increment size
+ pool->size += size;
- pool->length = length;
+ // Reallocate blocks
+ pool->blocks = realloc(pool->blocks, pool->size);
+ if (!pool->blocks) {
+ ERROR(pool->ctx, "Could not grow string pool: %m\n");
+ return 1;
+ }
- // Restore pos
- pool->pos = __loc_stringpool_get(pool, pos);
+ // Update data pointer
+ pool->data = pool->blocks;
return 0;
}
static off_t loc_stringpool_append(struct loc_stringpool* pool, const char* string) {
- if (!string)
- return -EINVAL;
+ if (!string) {
+ errno = EINVAL;
+ return -1;
+ }
DEBUG(pool->ctx, "Appending '%s' to string pool at %p\n", string, pool);
+ // How much space to we need?
+ const size_t length = strlen(string) + 1;
+
// Make sure we have enough space
- int r = loc_stringpool_grow(pool, pool->length + strlen(string) + 1);
- if (r) {
- errno = r;
- return -1;
+ if (pool->length + length > pool->size) {
+ int r = loc_stringpool_grow(pool, LOC_STRINGPOOL_BLOCK_SIZE);
+ if (r)
+ return r;
}
- off_t offset = loc_stringpool_get_offset(pool, pool->pos);
+ off_t offset = pool->length;
- // Copy string byte by byte
- while (*string)
- *pool->pos++ = *string++;
+ // Copy the string
+ memcpy(pool->blocks + offset, string, length);
- // Terminate the string
- *pool->pos++ = '\0';
+ // Update the length of the pool
+ pool->length += length;
return offset;
}
static void loc_stringpool_free(struct loc_stringpool* pool) {
DEBUG(pool->ctx, "Releasing string pool %p\n", pool);
- int r;
-
- switch (pool->mode) {
- case STRINGPOOL_DEFAULT:
- if (pool->data)
- free(pool->data);
- break;
-
- case STRINGPOOL_MMAP:
- if (pool->data) {
- r = munmap(pool->data, pool->length);
- if (r)
- ERROR(pool->ctx, "Could not unmap data at %p: %s\n",
- pool->data, strerror(errno));
- }
- break;
- }
+
+ // Free any data
+ if (pool->blocks)
+ free(pool->blocks);
loc_unref(pool->ctx);
free(pool);
p->ctx = loc_ref(ctx);
p->refcount = 1;
- // Save mode
- p->mode = STRINGPOOL_DEFAULT;
-
*pool = p;
return 0;
}
-static int loc_stringpool_mmap(struct loc_stringpool* pool, FILE* f, size_t length, off_t offset) {
- if (pool->mode != STRINGPOOL_MMAP)
- return -EINVAL;
-
- DEBUG(pool->ctx, "Reading string pool starting from %jd (%zu bytes)\n", (intmax_t)offset, length);
-
- // Map file content into memory
- pool->data = pool->pos = mmap(NULL, length, PROT_READ,
- MAP_PRIVATE, fileno(f), offset);
-
- // Store size of section
- pool->length = length;
-
- if (pool->data == MAP_FAILED)
- return 1;
-
- return 0;
-}
-
int loc_stringpool_open(struct loc_ctx* ctx, struct loc_stringpool** pool,
- FILE* f, size_t length, off_t offset) {
+ const char* data, const size_t length) {
struct loc_stringpool* p = NULL;
// Allocate a new stringpool
int r = loc_stringpool_new(ctx, &p);
if (r)
- return r;
+ goto ERROR;
- // Change mode to mmap
- p->mode = STRINGPOOL_MMAP;
+ // Store data and length
+ p->data = data;
+ p->length = length;
- // Map data into memory
- if (length > 0) {
- r = loc_stringpool_mmap(p, f, length, offset);
- if (r) {
- loc_stringpool_free(p);
- return r;
- }
- }
+ DEBUG(p->ctx, "Opened string pool at %p (%zu bytes)\n", p->data, p->length);
*pool = p;
return 0;
+
+ERROR:
+ if (p)
+ loc_stringpool_free(p);
+
+ return r;
}
struct loc_stringpool* loc_stringpool_ref(struct loc_stringpool* pool) {
return NULL;
}
-static off_t loc_stringpool_get_next_offset(struct loc_stringpool* pool, off_t offset) {
- const char* string = loc_stringpool_get(pool, offset);
- if (!string)
- return offset;
-
- return offset + strlen(string) + 1;
-}
-
const char* loc_stringpool_get(struct loc_stringpool* pool, off_t offset) {
- return __loc_stringpool_get(pool, offset);
+ // Check boundaries
+ if (offset < 0 || offset >= pool->length) {
+ errno = ERANGE;
+ return NULL;
+ }
+
+ // Return any data that we have in memory
+ return pool->data + offset;
}
size_t loc_stringpool_get_size(struct loc_stringpool* pool) {
- return loc_stringpool_get_offset(pool, pool->pos);
+ return pool->length;
}
static off_t loc_stringpool_find(struct loc_stringpool* pool, const char* s) {
- if (!s || !*s)
- return -EINVAL;
+ if (!s || !*s) {
+ errno = EINVAL;
+ return -1;
+ }
off_t offset = 0;
while (offset < pool->length) {
const char* string = loc_stringpool_get(pool, offset);
+
+ // Error!
if (!string)
- break;
+ return 1;
- int r = strcmp(s, string);
- if (r == 0)
+ // Is this a match?
+ if (strcmp(s, string) == 0)
return offset;
- offset = loc_stringpool_get_next_offset(pool, offset);
+ // Shift offset
+ offset += strlen(string) + 1;
}
- return -ENOENT;
+ // Nothing found
+ errno = ENOENT;
+ return -1;
}
off_t loc_stringpool_add(struct loc_stringpool* pool, const char* string) {
while (offset < pool->length) {
const char* string = loc_stringpool_get(pool, offset);
if (!string)
- break;
+ return;
printf("%jd (%zu): %s\n", (intmax_t)offset, strlen(string), string);
- offset = loc_stringpool_get_next_offset(pool, offset);
+ // Shift offset
+ offset += strlen(string) + 1;
}
}
FILE* f = tmpfile();
if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open file for writing: %m\n");
exit(EXIT_FAILURE);
}
err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET);
if (err) {
- fprintf(stderr, "Could not write database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not write database: %m\n");
exit(EXIT_FAILURE);
}
struct loc_database* db;
err = loc_database_new(ctx, &db, f);
if (err) {
- fprintf(stderr, "Could not open database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not open database: %m\n");
exit(EXIT_FAILURE);
}
FILE* f = tmpfile();
if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open file for writing: %m\n");
exit(EXIT_FAILURE);
}
err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET);
if (err) {
- fprintf(stderr, "Could not write database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not write database: %m\n");
exit(EXIT_FAILURE);
}
loc_writer_unref(writer);
struct loc_database* db;
err = loc_database_new(ctx, &db, f);
if (err) {
- fprintf(stderr, "Could not open database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not open database: %m\n");
exit(EXIT_FAILURE);
}
// Lookup an address in the subnet
err = loc_database_get_country(db, &country, "YY");
- if (err) {
+ if (err || !country) {
fprintf(stderr, "Could not find country: YY\n");
exit(EXIT_FAILURE);
}
FILE* f = tmpfile();
if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open file for writing: %m\n");
exit(EXIT_FAILURE);
}
err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET);
if (err) {
- fprintf(stderr, "Could not write database: %s\n", strerror(err));
+ fprintf(stderr, "Could not write database: %m\n");
exit(EXIT_FAILURE);
}
loc_writer_unref(writer);
struct loc_database* db;
err = loc_database_new(ctx, &db, f);
if (err) {
- fprintf(stderr, "Could not open database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not open database: %m\n");
exit(EXIT_FAILURE);
}
FILE* f = tmpfile();
if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open file for writing: %m\n");
exit(EXIT_FAILURE);
}
err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET);
if (err) {
- fprintf(stderr, "Could not write database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not write database: %m\n");
exit(EXIT_FAILURE);
}
loc_writer_unref(writer);
struct loc_database* db;
err = loc_database_new(ctx, &db, f);
if (err) {
- fprintf(stderr, "Could not open database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not open database: %m\n");
exit(EXIT_FAILURE);
}
fprintf(stderr, "Could look up 2001:db8:fffe:1::, but I shouldn't\n");
exit(EXIT_FAILURE);
}
- loc_network_unref(network1);
const struct bit_length_test {
const char* network;
// Open public key
FILE* public_key = fopen(ABS_SRCDIR "/examples/public-key.pem", "r");
if (!public_key) {
- fprintf(stderr, "Could not open public key file: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open public key file: %m\n");
exit(EXIT_FAILURE);
}
// Open private key
FILE* private_key1 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r");
if (!private_key1) {
- fprintf(stderr, "Could not open private key file: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open private key file: %m\n");
exit(EXIT_FAILURE);
}
FILE* private_key2 = fopen(ABS_SRCDIR "/examples/private-key.pem", "r");
if (!private_key2) {
- fprintf(stderr, "Could not open private key file: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open private key file: %m\n");
exit(EXIT_FAILURE);
}
FILE* f = tmpfile();
if (!f) {
- fprintf(stderr, "Could not open file for writing: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open file for writing: %m\n");
exit(EXIT_FAILURE);
}
err = loc_writer_write(writer, f, LOC_DATABASE_VERSION_UNSET);
if (err) {
- fprintf(stderr, "Could not write database: %s\n", strerror(err));
+ fprintf(stderr, "Could not write database: %m\n");
exit(EXIT_FAILURE);
}
loc_writer_unref(writer);
struct loc_database* db;
err = loc_database_new(ctx, &db, f);
if (err) {
- fprintf(stderr, "Could not open database: %s\n", strerror(-err));
+ fprintf(stderr, "Could not open database: %m\n");
exit(EXIT_FAILURE);
}
// Open another public key
public_key = freopen(ABS_SRCDIR "/src/signing-key.pem", "r", public_key);
if (!public_key) {
- fprintf(stderr, "Could not open public key file: %s\n", strerror(errno));
+ fprintf(stderr, "Could not open public key file: %m\n");
exit(EXIT_FAILURE);
}
// Append a string
off_t pos = loc_stringpool_add(pool, "ABC");
if (pos < 0) {
- fprintf(stderr, "Could not add string: %s\n", strerror(-pos));
+ fprintf(stderr, "Could not add string: %m\n");
exit(EXIT_FAILURE);
}
free(string);
if (pos < 0) {
- fprintf(stderr, "Could not add string %d: %s\n", i, strerror(-pos));
+ fprintf(stderr, "Could not add string %d: %m\n", i);
exit(EXIT_FAILURE);
}
}
FILE* fkey1, FILE* fkey2) {
struct loc_writer* w = calloc(1, sizeof(*w));
if (!w)
- return -ENOMEM;
+ return 1;
w->ctx = loc_ref(ctx);
w->refcount = 1;
loc_network_tree_unref(writer->networks);
// Unref the string pool
- loc_stringpool_unref(writer->pool);
+ if (writer->pool)
+ loc_stringpool_unref(writer->pool);
loc_unref(writer->ctx);
free(writer);
static int loc_writer_create_signature(struct loc_writer* writer,
struct loc_database_header_v1* header, FILE* f, EVP_PKEY* private_key,
char* signature, size_t* length) {
+ size_t bytes_read = 0;
+
DEBUG(writer->ctx, "Creating signature...\n");
// Read file from the beginning
// Read magic
struct loc_database_magic magic;
- fread(&magic, 1, sizeof(magic), f);
+ bytes_read = fread(&magic, 1, sizeof(magic), f);
+ if (bytes_read < sizeof(magic)) {
+ ERROR(writer->ctx, "Could not read header: %m\n");
+ r = 1;
+ goto END;
+ }
hexdump(writer->ctx, &magic, sizeof(magic));
// Walk through the file in chunks of 64kB
char buffer[64 * 1024];
while (!feof(f)) {
- size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
+ bytes_read = fread(buffer, 1, sizeof(buffer), f);
if (ferror(f)) {
- ERROR(writer->ctx, "Error reading from file: %s\n", strerror(errno));
- r = errno;
+ ERROR(writer->ctx, "Error reading from file: %m\n");
+ r = 1;
goto END;
}
fwrite(&header, 1, sizeof(header), f);
+ // Flush everything
+ fflush(f);
+
return r;
}
--- /dev/null
+../../src/signing-key.pem
\ No newline at end of file
--- /dev/null
+#!/usr/bin/python3
+###############################################################################
+# #
+# libloc - A library to determine the location of someone on the Internet #
+# #
+# Copyright (C) 2022 IPFire Development Team <info@ipfire.org> #
+# #
+# This library is free software; you can redistribute it and/or #
+# modify it under the terms of the GNU Lesser General Public #
+# License as published by the Free Software Foundation; either #
+# version 2.1 of the License, or (at your option) any later version. #
+# #
+# This library is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
+# Lesser General Public License for more details. #
+# #
+###############################################################################
+
+import location
+import os
+import unittest
+
+TEST_DATA_DIR = os.environ["TEST_DATA_DIR"]
+
+class Test(unittest.TestCase):
+ def setUp(self):
+ path = os.path.join(TEST_DATA_DIR, "location-2022-03-30.db")
+
+ # Load the database
+ self.db = location.Database(path)
+
+ def test_metadata(self):
+ """
+ Check if any metadata matches what we expected
+ """
+ # Vendor
+ self.assertEqual(self.db.vendor, "IPFire Project")
+
+ # Description
+ self.assertEqual(self.db.description,
+ "This database has been obtained from https://location.ipfire.org/\n\nFind the full license terms at https://creativecommons.org/licenses/by-sa/4.0/")
+
+ # License
+ self.assertEqual(self.db.license, "CC BY-SA 4.0")
+
+ # Created At
+ self.assertEqual(self.db.created_at, 1648619023)
+
+ def test_fetch_network(self):
+ """
+ Try fetching some results that should exist
+ """
+ n = self.db.lookup("81.3.27.38")
+ self.assertIsInstance(n, location.Network)
+
+ n = self.db.lookup("1.1.1.1")
+ self.assertIsInstance(n, location.Network)
+
+ n = self.db.lookup("8.8.8.8")
+ self.assertIsInstance(n, location.Network)
+
+ def test_fetch_network_nonexistant(self):
+ """
+ Try to fetch something that should not exist
+ """
+ n = self.db.lookup("255.255.255.255")
+ self.assertIsNone(n)
+
+ def test_fetch_network_invalid(self):
+ """
+ Feed some invalid inputs into the lookup function
+ """
+ with self.assertRaises(ValueError):
+ self.db.lookup("XXX")
+
+ with self.assertRaises(ValueError):
+ self.db.lookup("455.455.455.455")
+
+ def test_verify(self):
+ """
+ Verify the database
+ """
+ # Path to the signature file
+ path = os.path.join(TEST_DATA_DIR, "signing-key.pem")
+
+ # Try to verify with an invalid signature
+ with self.assertRaises(TypeError):
+ self.db.verify(None)
+
+ # Perform verification with the correct key
+ with open(path, "r") as f:
+ self.assertTrue(self.db.verify(f))
+
+ # Perform verification with invalid keys
+ with open("/dev/null", "r") as f:
+ self.assertFalse(self.db.verify(f))
+
+ with open("/dev/urandom", "r") as f:
+ self.assertFalse(self.db.verify(f))
+
+ def test_search_as(self):
+ """
+ Try to fetch an AS
+ """
+ # Fetch an existing AS
+ self.assertIsInstance(self.db.get_as(204867), location.AS)
+
+ # Fetch a non-existing AS
+ self.assertIsNone(self.db.get_as(0))
+
+ # Fetch an AS with a number that is out of range
+ with self.assertRaises(OverflowError):
+ self.db.get_as(2**32 + 1)
+
+ def test_get_country(self):
+ """
+ Try fetching a country
+ """
+ # Fetch an existing country
+ self.assertIsInstance(self.db.get_country("DE"), location.Country)
+
+ # Fetch a non-existing country
+ self.assertIsNone(self.db.get_country("AA"))
+
+ # Fetch a country with an invalid country code
+ with self.assertRaises(ValueError):
+ self.db.get_country("XXX")
+
+ def test_list_bogons(self):
+ """
+ Generate a list of bogons
+ """
+ # Fetch all bogons
+ bogons = self.db.list_bogons()
+
+ # We should have received an enumerator full of networks
+ self.assertIsInstance(bogons, location.DatabaseEnumerator)
+ for bogon in bogons:
+ self.assertIsInstance(bogon, location.Network)
+
+
+if __name__ == "__main__":
+ unittest.main()
projects / location / debian / libloc.git / commitdiff
