*/
#include <arpa/inet.h>
+#include <ctype.h>
#include <errno.h>
+#include <netinet/in.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/mman.h>
#include <sys/types.h>
+#include <time.h>
#include <unistd.h>
+#ifdef HAVE_ENDIAN_H
+# include <endian.h>
+#endif
+
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+
#include <loc/libloc.h>
+#include <loc/as.h>
+#include <loc/compat.h>
+#include <loc/country.h>
+#include <loc/database.h>
#include <loc/format.h>
-
-#include "libloc-private.h"
-#include "as.h"
-#include "database.h"
-#include "stringpool.h"
+#include <loc/network.h>
+#include <loc/private.h>
+#include <loc/stringpool.h>
struct loc_database {
struct loc_ctx* ctx;
int refcount;
- FILE* file;
+ FILE* f;
+
unsigned int version;
+ time_t created_at;
off_t vendor;
off_t description;
+ off_t license;
+
+ char* signature;
+ size_t signature_length;
// ASes in the database
- struct loc_as** as;
+ struct loc_database_as_v0* as_v0;
size_t as_count;
+ // Network tree
+ struct loc_database_network_node_v0* network_nodes_v0;
+ size_t network_nodes_count;
+
+ // Networks
+ struct loc_database_network_v0* networks_v0;
+ size_t networks_count;
+
+ // Countries
+ struct loc_database_country_v0* countries_v0;
+ size_t countries_count;
+
struct loc_stringpool* pool;
};
-LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, size_t pool_size) {
+#define MAX_STACK_DEPTH 256
+
+struct loc_node_stack {
+ off_t offset;
+ int i; // Is this node 0 or 1?
+ int depth;
+};
+
+struct loc_database_enumerator {
+ struct loc_ctx* ctx;
+ struct loc_database* db;
+ enum loc_database_enumerator_mode mode;
+ int refcount;
+
+ // Search string
+ char* string;
+ char country_code[3];
+ uint32_t asn;
+ enum loc_network_flags flags;
+ int family;
+
+ // Index of the AS we are looking at
+ unsigned int as_index;
+
+ // Network state
+ struct in6_addr network_address;
+ struct loc_node_stack network_stack[MAX_STACK_DEPTH];
+ int network_stack_depth;
+ unsigned int* networks_visited;
+};
+
+static int loc_database_read_magic(struct loc_database* db) {
+ struct loc_database_magic magic;
+
+ // Read from file
+ size_t bytes_read = fread(&magic, 1, sizeof(magic), db->f);
+
+ // Check if we have been able to read enough data
+ if (bytes_read < sizeof(magic)) {
+ ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
+ DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
+ return -ENOMSG;
+ }
+
+ // Compare magic bytes
+ if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
+ DEBUG(db->ctx, "Magic value matches\n");
+
+ // Parse version
+ db->version = be16toh(magic.version);
+ DEBUG(db->ctx, "Database version is %u\n", db->version);
+
+ return 0;
+ }
+
+ ERROR(db->ctx, "Database format is not compatible\n");
+
+ // Return an error
+ return 1;
+}
+
+static int loc_database_read_as_section_v0(struct loc_database* db,
+ const struct loc_database_header_v0* header) {
+ off_t as_offset = be32toh(header->as_offset);
+ size_t as_length = be32toh(header->as_length);
+
+ DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", (intmax_t)as_offset, as_length);
+
+ if (as_length > 0) {
+ db->as_v0 = mmap(NULL, as_length, PROT_READ,
+ MAP_SHARED, fileno(db->f), as_offset);
+
+ if (db->as_v0 == MAP_FAILED)
+ return -errno;
+ }
+
+ db->as_count = as_length / sizeof(*db->as_v0);
+
+ INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
+
+ return 0;
+}
+
+static int loc_database_read_network_nodes_section_v0(struct loc_database* db,
+ const struct loc_database_header_v0* header) {
+ off_t network_nodes_offset = be32toh(header->network_tree_offset);
+ size_t network_nodes_length = be32toh(header->network_tree_length);
+
+ DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n",
+ (intmax_t)network_nodes_offset, network_nodes_length);
+
+ if (network_nodes_length > 0) {
+ db->network_nodes_v0 = mmap(NULL, network_nodes_length, PROT_READ,
+ MAP_SHARED, fileno(db->f), network_nodes_offset);
+
+ if (db->network_nodes_v0 == MAP_FAILED)
+ return -errno;
+ }
+
+ db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v0);
+
+ INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count);
+
+ return 0;
+}
+
+static int loc_database_read_networks_section_v0(struct loc_database* db,
+ const struct loc_database_header_v0* header) {
+ off_t networks_offset = be32toh(header->network_data_offset);
+ size_t networks_length = be32toh(header->network_data_length);
+
+ DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n",
+ (intmax_t)networks_offset, networks_length);
+
+ if (networks_length > 0) {
+ db->networks_v0 = mmap(NULL, networks_length, PROT_READ,
+ MAP_SHARED, fileno(db->f), networks_offset);
+
+ if (db->networks_v0 == MAP_FAILED)
+ return -errno;
+ }
+
+ db->networks_count = networks_length / sizeof(*db->networks_v0);
+
+ INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count);
+
+ return 0;
+}
+
+static int loc_database_read_countries_section_v0(struct loc_database* db,
+ const struct loc_database_header_v0* header) {
+ off_t countries_offset = be32toh(header->countries_offset);
+ size_t countries_length = be32toh(header->countries_length);
+
+ DEBUG(db->ctx, "Reading countries section from %jd (%zu bytes)\n",
+ (intmax_t)countries_offset, countries_length);
+
+ if (countries_length > 0) {
+ db->countries_v0 = mmap(NULL, countries_length, PROT_READ,
+ MAP_SHARED, fileno(db->f), countries_offset);
+
+ if (db->countries_v0 == MAP_FAILED)
+ return -errno;
+ }
+
+ db->countries_count = countries_length / sizeof(*db->countries_v0);
+
+ INFO(db->ctx, "Read %zu countries from the database\n",
+ db->countries_count);
+
+ return 0;
+}
+
+static int loc_database_read_header_v0(struct loc_database* db) {
+ struct loc_database_header_v0 header;
+
+ // Read from file
+ size_t size = fread(&header, 1, sizeof(header), db->f);
+
+ if (size < sizeof(header)) {
+ ERROR(db->ctx, "Could not read enough data for header\n");
+ return -ENOMSG;
+ }
+
+ // Copy over data
+ db->created_at = be64toh(header.created_at);
+ db->vendor = be32toh(header.vendor);
+ db->description = be32toh(header.description);
+ db->license = be32toh(header.license);
+
+ // Read signature
+ db->signature_length = be32toh(header.signature_length);
+ if (db->signature_length) {
+ // Check for a plausible signature length
+ if (db->signature_length > LOC_SIGNATURE_MAX_LENGTH) {
+ ERROR(db->ctx, "Signature too long: %ld\n", db->signature_length);
+ return -EINVAL;
+ }
+
+ DEBUG(db->ctx, "Reading signature of %ld bytes\n",
+ db->signature_length);
+
+ db->signature = malloc(db->signature_length);
+ for (unsigned int i = 0; i < db->signature_length; i++)
+ db->signature[i] = header.signature[i];
+ }
+
+ // Open pool
+ off_t pool_offset = be32toh(header.pool_offset);
+ size_t pool_length = be32toh(header.pool_length);
+
+ int r = loc_stringpool_open(db->ctx, &db->pool,
+ db->f, pool_length, pool_offset);
+ if (r)
+ return r;
+
+ // AS section
+ r = loc_database_read_as_section_v0(db, &header);
+ if (r)
+ return r;
+
+ // Network Nodes
+ r = loc_database_read_network_nodes_section_v0(db, &header);
+ if (r)
+ return r;
+
+ // Networks
+ r = loc_database_read_networks_section_v0(db, &header);
+ if (r)
+ return r;
+
+ // countries
+ r = loc_database_read_countries_section_v0(db, &header);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+static int loc_database_read_header(struct loc_database* db) {
+ switch (db->version) {
+ case 0:
+ return loc_database_read_header_v0(db);
+
+ default:
+ ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
+ return 1;
+ }
+}
+
+static int loc_database_read(struct loc_database* db, FILE* f) {
+ clock_t start = clock();
+
+ int fd = fileno(f);
+
+ // Clone file descriptor
+ fd = dup(fd);
+ if (!fd) {
+ ERROR(db->ctx, "Could not duplicate file descriptor\n");
+ return -1;
+ }
+
+ // Reopen the file so that we can keep our own file handle
+ db->f = fdopen(fd, "r");
+ if (!db->f) {
+ ERROR(db->ctx, "Could not re-open database file\n");
+ return -1;
+ }
+
+ // Rewind to the start of the file
+ rewind(db->f);
+
+ // Read magic bytes
+ int r = loc_database_read_magic(db);
+ if (r)
+ return r;
+
+ // Read the header
+ r = loc_database_read_header(db);
+ if (r)
+ return r;
+
+ clock_t end = clock();
+
+ INFO(db->ctx, "Opened database in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
+
+ return 0;
+}
+
+LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
+ // Fail on invalid file handle
+ if (!f)
+ return -EINVAL;
+
struct loc_database* db = calloc(1, sizeof(*db));
if (!db)
return -ENOMEM;
db->ctx = loc_ref(ctx);
db->refcount = 1;
- DEBUG(db->ctx, "Database allocated at %p\n", db);
+ DEBUG(db->ctx, "Database object allocated at %p\n", db);
- // Create string pool
- int r = loc_stringpool_new(db->ctx, &db->pool, pool_size);
+ int r = loc_database_read(db, f);
if (r) {
loc_database_unref(db);
return r;
return 0;
}
-LOC_EXPORT int loc_database_open(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
- int r = loc_database_new(ctx, database, 0);
- if (r)
- return r;
-
- return loc_database_read(*database, f);
-}
-
LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
db->refcount++;
}
static void loc_database_free(struct loc_database* db) {
+ int r;
+
DEBUG(db->ctx, "Releasing database %p\n", db);
- // Remove references to all ASes
- if (db->as) {
- for (unsigned int i = 0; i < db->as_count; i++) {
- loc_as_unref(db->as[i]);
- }
- free(db->as);
+ // Removing all ASes
+ if (db->as_v0) {
+ r = munmap(db->as_v0, db->as_count * sizeof(*db->as_v0));
+ if (r)
+ ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno));
+ }
+
+ // Remove mapped network sections
+ if (db->networks_v0) {
+ r = munmap(db->networks_v0, db->networks_count * sizeof(*db->networks_v0));
+ if (r)
+ ERROR(db->ctx, "Could not unmap networks section: %s\n", strerror(errno));
+ }
+
+ // Remove mapped network nodes section
+ if (db->network_nodes_v0) {
+ r = munmap(db->network_nodes_v0, db->network_nodes_count * sizeof(*db->network_nodes_v0));
+ if (r)
+ ERROR(db->ctx, "Could not unmap network nodes section: %s\n", strerror(errno));
}
loc_stringpool_unref(db->pool);
- // Close file
- if (db->file)
- fclose(db->file);
+ // Free signature
+ if (db->signature)
+ free(db->signature);
+
+ // Close database file
+ if (db->f)
+ fclose(db->f);
loc_unref(db->ctx);
free(db);
return NULL;
}
-LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
- return loc_stringpool_get(db->pool, db->vendor);
+LOC_EXPORT int loc_database_verify(struct loc_database* db, FILE* f) {
+ // Cannot do this when no signature is available
+ if (!db->signature) {
+ DEBUG(db->ctx, "No signature available to verify\n");
+ return 1;
+ }
+
+ // Start the stopwatch
+ clock_t start = clock();
+
+ // Load public key
+ EVP_PKEY* pkey = PEM_read_PUBKEY(f, NULL, NULL, NULL);
+ if (!pkey) {
+ char* error = ERR_error_string(ERR_get_error(), NULL);
+ ERROR(db->ctx, "Could not parse public key: %s\n", error);
+
+ return -1;
+ }
+
+ int r = 0;
+
+ EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
+
+ // Initialise hash function
+ r = EVP_DigestVerifyInit(mdctx, NULL, NULL, NULL, pkey);
+ if (r != 1) {
+ ERROR(db->ctx, "Error initializing signature validation: %s\n",
+ ERR_error_string(ERR_get_error(), NULL));
+ r = 1;
+
+ goto CLEANUP;
+ }
+
+ // Reset file to start
+ rewind(db->f);
+
+ // Read magic
+ struct loc_database_magic magic;
+ fread(&magic, 1, sizeof(magic), db->f);
+
+ hexdump(db->ctx, &magic, sizeof(magic));
+
+ // Feed magic into the hash
+ r = EVP_DigestVerifyUpdate(mdctx, &magic, sizeof(magic));
+ if (r != 1) {
+ ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
+ r = 1;
+
+ goto CLEANUP;
+ }
+
+ // Read the header
+ struct loc_database_header_v0 header_v0;
+
+ switch (db->version) {
+ case 0:
+ fread(&header_v0, 1, sizeof(header_v0), db->f);
+
+ // Clear signature
+ for (unsigned int i = 0; i < sizeof(header_v0.signature); i++) {
+ header_v0.signature[i] = '\0';
+ }
+
+ hexdump(db->ctx, &header_v0, sizeof(header_v0));
+
+ // Feed header into the hash
+ r = EVP_DigestVerifyUpdate(mdctx, &header_v0, sizeof(header_v0));
+ if (r != 1) {
+ ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
+ r = 1;
+
+ goto CLEANUP;
+ }
+ break;
+
+ default:
+ ERROR(db->ctx, "Cannot compute hash for database with format %d\n",
+ db->version);
+ r = -EINVAL;
+ goto CLEANUP;
+ }
+
+ // Walk through the file in chunks of 64kB
+ char buffer[64 * 1024];
+
+ while (!feof(db->f)) {
+ size_t bytes_read = fread(buffer, 1, sizeof(buffer), db->f);
+
+ hexdump(db->ctx, buffer, bytes_read);
+
+ r = EVP_DigestVerifyUpdate(mdctx, buffer, bytes_read);
+ if (r != 1) {
+ ERROR(db->ctx, "%s\n", ERR_error_string(ERR_get_error(), NULL));
+ r = 1;
+
+ goto CLEANUP;
+ }
+ }
+
+ // Finish
+ r = EVP_DigestVerifyFinal(mdctx,
+ (unsigned char*)db->signature, db->signature_length);
+
+ if (r == 0) {
+ DEBUG(db->ctx, "The signature is invalid\n");
+ r = 1;
+ } else if (r == 1) {
+ DEBUG(db->ctx, "The signature is valid\n");
+ r = 0;
+ } else {
+ ERROR(db->ctx, "Error verifying the signature: %s\n",
+ ERR_error_string(ERR_get_error(), NULL));
+ r = 1;
+ }
+
+ // Dump signature
+ hexdump(db->ctx, db->signature, db->signature_length);
+
+ clock_t end = clock();
+ DEBUG(db->ctx, "Signature checked in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
+
+CLEANUP:
+ // Cleanup
+ EVP_MD_CTX_free(mdctx);
+ EVP_PKEY_free(pkey);
+
+ return r;
}
-LOC_EXPORT int loc_database_set_vendor(struct loc_database* db, const char* vendor) {
- // Add the string to the string pool
- off_t offset = loc_stringpool_add(db->pool, vendor);
- if (offset < 0)
- return offset;
+LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
+ return db->created_at;
+}
- db->vendor = offset;
- return 0;
+LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
+ return loc_stringpool_get(db->pool, db->vendor);
}
LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
return loc_stringpool_get(db->pool, db->description);
}
-LOC_EXPORT int loc_database_set_description(struct loc_database* db, const char* description) {
- // Add the string to the string pool
- off_t offset = loc_stringpool_add(db->pool, description);
- if (offset < 0)
- return offset;
-
- db->description = offset;
- return 0;
+LOC_EXPORT const char* loc_database_get_license(struct loc_database* db) {
+ return loc_stringpool_get(db->pool, db->license);
}
LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
return db->as_count;
}
-static int loc_database_has_as(struct loc_database* db, struct loc_as* as) {
- for (unsigned int i = 0; i < db->as_count; i++) {
- if (loc_as_cmp(as, db->as[i]) == 0)
- return i;
+// Returns the AS at position pos
+static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
+ if ((size_t)pos >= db->as_count)
+ return -EINVAL;
+
+ DEBUG(db->ctx, "Fetching AS at position %jd\n", (intmax_t)pos);
+
+ int r;
+ switch (db->version) {
+ case 0:
+ r = loc_as_new_from_database_v0(db->ctx, db->pool, as, db->as_v0 + pos);
+ break;
+
+ default:
+ return -1;
}
- return -1;
-}
+ if (r == 0) {
+ DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as));
+ }
-static int __loc_as_cmp(const void* as1, const void* as2) {
- return loc_as_cmp(*(struct loc_as**)as1, *(struct loc_as**)as2);
+ return r;
}
-static void loc_database_sort_ases(struct loc_database* db) {
- qsort(db->as, db->as_count, sizeof(*db->as), __loc_as_cmp);
-}
+// Performs a binary search to find the AS in the list
+LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) {
+ off_t lo = 0;
+ off_t hi = db->as_count - 1;
-static struct loc_as* __loc_database_add_as(struct loc_database* db, struct loc_as* as) {
- // Check if AS exists already
- int i = loc_database_has_as(db, as);
- if (i >= 0) {
- loc_as_unref(as);
+ // Save start time
+ clock_t start = clock();
- // Select already existing AS
- as = db->as[i];
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
- return loc_as_ref(as);
- }
+ // Fetch AS in the middle between lo and hi
+ int r = loc_database_fetch_as(db, as, i);
+ if (r)
+ return r;
- db->as_count++;
+ // Check if this is a match
+ uint32_t as_number = loc_as_get_number(*as);
+ if (as_number == number) {
+ clock_t end = clock();
- // Make space for the new entry
- db->as = realloc(db->as, sizeof(*db->as) * db->as_count);
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found AS%u in %.4fms\n", as_number,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
- // Add the new entry at the end
- db->as[db->as_count - 1] = loc_as_ref(as);
+ return 0;
+ }
- // Sort everything
- loc_database_sort_ases(db);
+ // If it wasn't, we release the AS and
+ // adjust our search pointers
+ loc_as_unref(*as);
- return as;
-}
+ if (as_number < number) {
+ lo = i + 1;
+ } else
+ hi = i - 1;
+ }
-LOC_EXPORT struct loc_as* loc_database_add_as(struct loc_database* db, uint32_t number) {
- struct loc_as* as;
- int r = loc_as_new(db->ctx, db->pool, &as, number);
- if (r)
- return NULL;
+ // Nothing found
+ *as = NULL;
- return __loc_database_add_as(db, as);
+ return 1;
}
-static int loc_database_read_magic(struct loc_database* db) {
- struct loc_database_magic magic;
+// Returns the network at position pos
+static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network,
+ struct in6_addr* address, unsigned int prefix, off_t pos) {
+ if ((size_t)pos >= db->networks_count) {
+ DEBUG(db->ctx, "Network ID out of range: %jd/%jd\n",
+ (intmax_t)pos, (intmax_t)db->networks_count);
+ return -EINVAL;
+ }
- // Read from file
- size_t bytes_read = fread(&magic, 1, sizeof(magic), db->file);
- // Check if we have been able to read enough data
- if (bytes_read < sizeof(magic)) {
- ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
- DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
- return -ENOMSG;
- }
+ DEBUG(db->ctx, "Fetching network at position %jd\n", (intmax_t)pos);
- // Compare magic bytes
- if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
- DEBUG(db->ctx, "Magic value matches\n");
+ int r;
+ switch (db->version) {
+ case 0:
+ r = loc_network_new_from_database_v0(db->ctx, network,
+ address, prefix, db->networks_v0 + pos);
+ break;
- // Parse version
- db->version = ntohs(magic.version);
- DEBUG(db->ctx, "Database version is %u\n", db->version);
+ default:
+ return -1;
+ }
- return 0;
+ if (r == 0) {
+ char* string = loc_network_str(*network);
+ DEBUG(db->ctx, "Got network %s\n", string);
+ free(string);
}
- ERROR(db->ctx, "Database format is not compatible\n");
+ return r;
+}
- // Return an error
- return 1;
+static int __loc_database_node_is_leaf(const struct loc_database_network_node_v0* node) {
+ return (node->network != htobe32(0xffffffff));
}
-static int loc_database_read_as_section_v0(struct loc_database* db,
- off_t as_offset, size_t as_length) {
- struct loc_database_as_v0 dbobj;
+static int __loc_database_lookup_handle_leaf(struct loc_database* db, const struct in6_addr* address,
+ struct loc_network** network, struct in6_addr* network_address, unsigned int prefix,
+ const struct loc_database_network_node_v0* node) {
+ off_t network_index = be32toh(node->network);
- // Read from the start of the section
- int r = fseek(db->file, as_offset, SEEK_SET);
- if (r)
+ DEBUG(db->ctx, "Handling leaf node at %jd (%jd)\n", (intmax_t)(node - db->network_nodes_v0), (intmax_t)network_index);
+
+ // Fetch the network
+ int r = loc_database_fetch_network(db, network,
+ network_address, prefix, network_index);
+ if (r) {
+ ERROR(db->ctx, "Could not fetch network %jd from database\n", (intmax_t)network_index);
return r;
+ }
- // Read all ASes
- size_t as_count = as_length / sizeof(dbobj);
- for (unsigned int i = 0; i < as_count; i++) {
- size_t bytes_read = fread(&dbobj, 1, sizeof(dbobj), db->file);
- if (bytes_read < sizeof(dbobj)) {
- ERROR(db->ctx, "Could not read an AS object\n");
- return -ENOMSG;
- }
+ // Check if the given IP address is inside the network
+ r = loc_network_match_address(*network, address);
+ if (r) {
+ DEBUG(db->ctx, "Searched address is not part of the network\n");
- // Allocate a new AS
- struct loc_as* as;
- r = loc_as_new_from_database_v0(db->ctx, db->pool, &as, &dbobj);
- if (r)
+ loc_network_unref(*network);
+ *network = NULL;
+ return 1;
+ }
+
+ // A network was found and the IP address matches
+ return 0;
+}
+
+// Searches for an exact match along the path
+static int __loc_database_lookup(struct loc_database* db, const struct in6_addr* address,
+ struct loc_network** network, struct in6_addr* network_address,
+ const struct loc_database_network_node_v0* node, unsigned int level) {
+ int r;
+ off_t node_index;
+
+ // Follow the path
+ int bit = in6_addr_get_bit(address, level);
+ in6_addr_set_bit(network_address, level, bit);
+
+ if (bit == 0)
+ node_index = be32toh(node->zero);
+ else
+ node_index = be32toh(node->one);
+
+ // If the node index is zero, the tree ends here
+ // and we cannot descend any further
+ if (node_index > 0) {
+ // Check boundaries
+ if ((size_t)node_index >= db->network_nodes_count)
+ return -EINVAL;
+
+ // Move on to the next node
+ r = __loc_database_lookup(db, address, network, network_address,
+ db->network_nodes_v0 + node_index, level + 1);
+
+ // End here if a result was found
+ if (r == 0)
+ return r;
+
+ // Raise any errors
+ else if (r < 0)
return r;
- // Attach it to the database
- as = __loc_database_add_as(db, as);
- loc_as_unref(as);
+ DEBUG(db->ctx, "No match found below level %u\n", level);
+ } else {
+ DEBUG(db->ctx, "Tree ended at level %u\n", level);
}
- INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
+ // If this node has a leaf, we will check if it matches
+ if (__loc_database_node_is_leaf(node)) {
+ r = __loc_database_lookup_handle_leaf(db, address, network, network_address, level, node);
+ if (r <= 0)
+ return r;
+ }
- return 0;
+ return 1;
}
-static int loc_database_read_header_v0(struct loc_database* db) {
- struct loc_database_header_v0 header;
+LOC_EXPORT int loc_database_lookup(struct loc_database* db,
+ struct in6_addr* address, struct loc_network** network) {
+ struct in6_addr network_address;
+ memset(&network_address, 0, sizeof(network_address));
- // Read from file
- size_t size = fread(&header, 1, sizeof(header), db->file);
+ *network = NULL;
- if (size < sizeof(header)) {
- ERROR(db->ctx, "Could not read enough data for header\n");
- return -ENOMSG;
- }
+ // Save start time
+ clock_t start = clock();
- // Copy over data
- db->vendor = ntohl(header.vendor);
- db->description = ntohl(header.description);
+ int r = __loc_database_lookup(db, address, network, &network_address,
+ db->network_nodes_v0, 0);
- // Open pool
- off_t pool_offset = ntohl(header.pool_offset);
- size_t pool_length = ntohl(header.pool_length);
+ clock_t end = clock();
- int r = loc_stringpool_read(db->pool, db->file, pool_offset, pool_length);
- if (r)
- return r;
+ // Log how fast this has been
+ DEBUG(db->ctx, "Executed network search in %.4fms\n",
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
- // AS section
- off_t as_offset = ntohl(header.as_offset);
- size_t as_length = ntohl(header.as_length);
+ return r;
+}
+
+LOC_EXPORT int loc_database_lookup_from_string(struct loc_database* db,
+ const char* string, struct loc_network** network) {
+ struct in6_addr address;
- r = loc_database_read_as_section_v0(db, as_offset, as_length);
+ int r = loc_parse_address(db->ctx, string, &address);
if (r)
return r;
- return 0;
+ return loc_database_lookup(db, &address, network);
}
-static int loc_database_read_header(struct loc_database* db) {
+// Returns the country at position pos
+static int loc_database_fetch_country(struct loc_database* db,
+ struct loc_country** country, off_t pos) {
+ if ((size_t)pos >= db->countries_count)
+ return -EINVAL;
+
+ DEBUG(db->ctx, "Fetching country at position %jd\n", (intmax_t)pos);
+
+ int r;
switch (db->version) {
case 0:
- return loc_database_read_header_v0(db);
+ r = loc_country_new_from_database_v0(db->ctx, db->pool, country, db->countries_v0 + pos);
+ break;
default:
- ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
- return 1;
+ return -1;
}
+
+ if (r == 0) {
+ DEBUG(db->ctx, "Got country %s\n", loc_country_get_code(*country));
+ }
+
+ return r;
}
-LOC_EXPORT int loc_database_read(struct loc_database* db, FILE* f) {
- // Copy the file pointer and work on that so we don't care if
- // the calling function closes the file
- int fd = fileno(f);
+// Performs a binary search to find the country in the list
+LOC_EXPORT int loc_database_get_country(struct loc_database* db,
+ struct loc_country** country, const char* code) {
+ off_t lo = 0;
+ off_t hi = db->countries_count - 1;
- // Make a copy
- fd = dup(fd);
+ // Save start time
+ clock_t start = clock();
- // Retrieve a file pointer
- db->file = fdopen(fd, "r");
- if (!db->file)
- return -errno;
+ while (lo <= hi) {
+ off_t i = (lo + hi) / 2;
- int r = fseek(db->file, 0, SEEK_SET);
- if (r)
- return r;
+ // Fetch country in the middle between lo and hi
+ int r = loc_database_fetch_country(db, country, i);
+ if (r)
+ return r;
- // Read magic bytes
- r = loc_database_read_magic(db);
- if (r)
- return r;
+ // Check if this is a match
+ const char* cc = loc_country_get_code(*country);
+ int result = strcmp(code, cc);
- // Read the header
- r = loc_database_read_header(db);
- if (r)
- return r;
+ if (result == 0) {
+ clock_t end = clock();
+
+ // Log how fast this has been
+ DEBUG(db->ctx, "Found country %s in %.4fms\n", cc,
+ (double)(end - start) / CLOCKS_PER_SEC * 1000);
+
+ return 0;
+ }
+
+ // If it wasn't, we release the country and
+ // adjust our search pointers
+ loc_country_unref(*country);
+
+ if (result > 0) {
+ lo = i + 1;
+ } else
+ hi = i - 1;
+ }
+
+ // Nothing found
+ *country = NULL;
+
+ return 1;
+}
+
+// Enumerator
+LOC_EXPORT int loc_database_enumerator_new(struct loc_database_enumerator** enumerator,
+ struct loc_database* db, enum loc_database_enumerator_mode mode) {
+ struct loc_database_enumerator* e = calloc(1, sizeof(*e));
+ if (!e)
+ return -ENOMEM;
+
+ // Reference context
+ e->ctx = loc_ref(db->ctx);
+ e->db = loc_database_ref(db);
+ e->mode = mode;
+ e->refcount = 1;
+
+ // Initialise graph search
+ //e->network_stack[++e->network_stack_depth] = 0;
+ e->network_stack_depth = 1;
+ e->networks_visited = calloc(db->network_nodes_count, sizeof(*e->networks_visited));
+
+ DEBUG(e->ctx, "Database enumerator object allocated at %p\n", e);
+
+ *enumerator = e;
return 0;
}
-static void loc_database_make_magic(struct loc_database* db, struct loc_database_magic* magic) {
- // Copy magic bytes
- for (unsigned int i = 0; i < strlen(LOC_DATABASE_MAGIC); i++)
- magic->magic[i] = LOC_DATABASE_MAGIC[i];
+LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_ref(struct loc_database_enumerator* enumerator) {
+ enumerator->refcount++;
- // Set version
- magic->version = htons(LOC_DATABASE_VERSION);
+ return enumerator;
}
-static void loc_database_align_page_boundary(off_t* offset, FILE* f) {
- // Move to next page boundary
- while (*offset % LOC_DATABASE_PAGE_SIZE > 0)
- *offset += fwrite("", 1, 1, f);
+static void loc_database_enumerator_free(struct loc_database_enumerator* enumerator) {
+ DEBUG(enumerator->ctx, "Releasing database enumerator %p\n", enumerator);
+
+ // Release all references
+ loc_database_unref(enumerator->db);
+ loc_unref(enumerator->ctx);
+
+ if (enumerator->string)
+ free(enumerator->string);
+
+ // Free network search
+ free(enumerator->networks_visited);
+
+ free(enumerator);
}
-static int loc_database_write_pool(struct loc_database* db, struct loc_database_header_v0* header, off_t* offset, FILE* f) {
- // Save the offset of the pool section
- DEBUG(db->ctx, "Pool starts at %jd bytes\n", *offset);
- header->pool_offset = htonl(*offset);
+LOC_EXPORT struct loc_database_enumerator* loc_database_enumerator_unref(struct loc_database_enumerator* enumerator) {
+ if (!enumerator)
+ return NULL;
+
+ if (--enumerator->refcount > 0)
+ return enumerator;
+
+ loc_database_enumerator_free(enumerator);
+ return NULL;
+}
- // Write the pool
- size_t pool_length = loc_stringpool_write(db->pool, f);
- *offset += pool_length;
+LOC_EXPORT int loc_database_enumerator_set_string(struct loc_database_enumerator* enumerator, const char* string) {
+ enumerator->string = strdup(string);
- DEBUG(db->ctx, "Pool has a length of %zu bytes\n", pool_length);
- header->pool_length = htonl(pool_length);
+ // Make the string lowercase
+ for (char *p = enumerator->string; *p; p++)
+ *p = tolower(*p);
return 0;
}
-static int loc_database_write_as_section(struct loc_database* db,
- struct loc_database_header_v0* header, off_t* offset, FILE* f) {
- DEBUG(db->ctx, "AS section starts at %jd bytes\n", *offset);
- header->as_offset = htonl(*offset);
+LOC_EXPORT int loc_database_enumerator_set_country_code(struct loc_database_enumerator* enumerator, const char* country_code) {
+ // Set empty country code
+ if (!country_code || !*country_code) {
+ *enumerator->country_code = '\0';
+ return 0;
+ }
- size_t as_length = 0;
+ // Treat A1, A2, A3 as special country codes,
+ // but perform search for flags instead
+ if (strcmp(country_code, "A1") == 0) {
+ return loc_database_enumerator_set_flag(enumerator,
+ LOC_NETWORK_FLAG_ANONYMOUS_PROXY);
+ } else if (strcmp(country_code, "A2") == 0) {
+ return loc_database_enumerator_set_flag(enumerator,
+ LOC_NETWORK_FLAG_SATELLITE_PROVIDER);
+ } else if (strcmp(country_code, "A3") == 0) {
+ return loc_database_enumerator_set_flag(enumerator,
+ LOC_NETWORK_FLAG_ANYCAST);
+ }
- struct loc_database_as_v0 dbas;
- for (unsigned int i = 0; i < db->as_count; i++) {
- // Convert AS into database format
- loc_as_to_database_v0(db->as[i], &dbas);
+ // Country codes must be two characters
+ if (!loc_country_code_is_valid(country_code))
+ return -EINVAL;
- // Write to disk
- offset += fwrite(&dbas, 1, sizeof(dbas), f);
- as_length += sizeof(dbas);
+ for (unsigned int i = 0; i < 3; i++) {
+ enumerator->country_code[i] = country_code[i];
}
- DEBUG(db->ctx, "AS section has a length of %zu bytes\n", as_length);
- header->as_length = htonl(as_length);
+ return 0;
+}
+
+LOC_EXPORT int loc_database_enumerator_set_asn(
+ struct loc_database_enumerator* enumerator, unsigned int asn) {
+ enumerator->asn = asn;
return 0;
}
-LOC_EXPORT int loc_database_write(struct loc_database* db, FILE* f) {
- struct loc_database_magic magic;
- loc_database_make_magic(db, &magic);
+LOC_EXPORT int loc_database_enumerator_set_flag(
+ struct loc_database_enumerator* enumerator, enum loc_network_flags flag) {
+ enumerator->flags |= flag;
- // Make the header
- struct loc_database_header_v0 header;
- header.vendor = htonl(db->vendor);
- header.description = htonl(db->description);
+ return 0;
+}
- int r;
- off_t offset = 0;
+LOC_EXPORT int loc_database_enumerator_set_family(
+ struct loc_database_enumerator* enumerator, int family) {
+ enumerator->family = family;
- // Start writing at the beginning of the file
- r = fseek(f, 0, SEEK_SET);
- if (r)
- return r;
+ return 0;
+}
- // Write the magic
- offset += fwrite(&magic, 1, sizeof(magic), f);
+LOC_EXPORT int loc_database_enumerator_next_as(
+ struct loc_database_enumerator* enumerator, struct loc_as** as) {
+ *as = NULL;
- // Skip the space we need to write the header later
- r = fseek(f, sizeof(header), SEEK_CUR);
- if (r) {
- DEBUG(db->ctx, "Could not seek to position after header\n");
- return r;
+ // Do not do anything if not in AS mode
+ if (enumerator->mode != LOC_DB_ENUMERATE_ASES)
+ return 0;
+
+ struct loc_database* db = enumerator->db;
+
+ while (enumerator->as_index < db->as_count) {
+ // Fetch the next AS
+ int r = loc_database_fetch_as(db, as, enumerator->as_index++);
+ if (r)
+ return r;
+
+ r = loc_as_match_string(*as, enumerator->string);
+ if (r == 1) {
+ DEBUG(enumerator->ctx, "AS%d (%s) matches %s\n",
+ loc_as_get_number(*as), loc_as_get_name(*as), enumerator->string);
+
+ return 0;
+ }
+
+ // No match
+ loc_as_unref(*as);
+ *as = NULL;
}
- offset += sizeof(header);
- loc_database_align_page_boundary(&offset, f);
+ // Reset the index
+ enumerator->as_index = 0;
- // Write pool
- r = loc_database_write_pool(db, &header, &offset, f);
- if (r)
- return r;
+ // We have searched through all of them
+ return 0;
+}
- loc_database_align_page_boundary(&offset, f);
+static int loc_database_enumerator_stack_push_node(
+ struct loc_database_enumerator* e, off_t offset, int i, int depth) {
+ // Do not add empty nodes
+ if (!offset)
+ return 0;
- // Write all ASes
- r = loc_database_write_as_section(db, &header, &offset, f);
- if (r)
- return r;
+ // Check if there is any space left on the stack
+ if (e->network_stack_depth >= MAX_STACK_DEPTH) {
+ ERROR(e->ctx, "Maximum stack size reached: %d\n", e->network_stack_depth);
+ return -1;
+ }
- // Write the header
- r = fseek(f, sizeof(magic), SEEK_SET);
- if (r)
- return r;
+ // Increase stack size
+ int s = ++e->network_stack_depth;
+
+ DEBUG(e->ctx, "Added node %jd to stack (%d)\n", (intmax_t)offset, depth);
+
+ e->network_stack[s].offset = offset;
+ e->network_stack[s].i = i;
+ e->network_stack[s].depth = depth;
+
+ return 0;
+}
+
+LOC_EXPORT int loc_database_enumerator_next_network(
+ struct loc_database_enumerator* enumerator, struct loc_network** network) {
+ // Reset network
+ *network = NULL;
+
+ // Do not do anything if not in network mode
+ if (enumerator->mode != LOC_DB_ENUMERATE_NETWORKS)
+ return 0;
+
+ int r;
+
+ DEBUG(enumerator->ctx, "Called with a stack of %u nodes\n",
+ enumerator->network_stack_depth);
+
+ // Perform DFS
+ while (enumerator->network_stack_depth > 0) {
+ DEBUG(enumerator->ctx, "Stack depth: %u\n", enumerator->network_stack_depth);
+
+ // Get object from top of the stack
+ struct loc_node_stack* node = &enumerator->network_stack[enumerator->network_stack_depth];
+
+ // Remove the node from the stack if we have already visited it
+ if (enumerator->networks_visited[node->offset]) {
+ enumerator->network_stack_depth--;
+ continue;
+ }
+
+ // Mark the bits on the path correctly
+ in6_addr_set_bit(&enumerator->network_address,
+ (node->depth > 0) ? node->depth - 1 : 0, node->i);
+
+ DEBUG(enumerator->ctx, "Looking at node %jd\n", (intmax_t)node->offset);
+ enumerator->networks_visited[node->offset]++;
+
+ // Pop node from top of the stack
+ struct loc_database_network_node_v0* n =
+ enumerator->db->network_nodes_v0 + node->offset;
+
+ // Add edges to stack
+ r = loc_database_enumerator_stack_push_node(enumerator,
+ be32toh(n->one), 1, node->depth + 1);
+
+ if (r)
+ return r;
+
+ r = loc_database_enumerator_stack_push_node(enumerator,
+ be32toh(n->zero), 0, node->depth + 1);
+
+ if (r)
+ return r;
+
+ // Check if this node is a leaf and has a network object
+ if (__loc_database_node_is_leaf(n)) {
+ off_t network_index = be32toh(n->network);
+
+ DEBUG(enumerator->ctx, "Node has a network at %jd\n", (intmax_t)network_index);
+
+ // Fetch the network object
+ r = loc_database_fetch_network(enumerator->db, network,
+ &enumerator->network_address, node->depth, network_index);
+
+ // Break on any errors
+ if (r)
+ return r;
+
+ // Check if we are interested in this network
+
+ // Skip if the family does not match
+ if (enumerator->family && loc_network_address_family(*network) != enumerator->family) {
+ loc_network_unref(*network);
+ *network = NULL;
+
+ continue;
+ }
+
+ // Skip if the country code does not match
+ if (*enumerator->country_code &&
+ !loc_network_match_country_code(*network, enumerator->country_code)) {
+ loc_network_unref(*network);
+ *network = NULL;
+
+ continue;
+ }
+
+ // Skip if the ASN does not match
+ if (enumerator->asn &&
+ !loc_network_match_asn(*network, enumerator->asn)) {
+ loc_network_unref(*network);
+ *network = NULL;
+
+ continue;
+ }
+
+ // Skip if flags do not match
+ if (enumerator->flags &&
+ !loc_network_match_flag(*network, enumerator->flags)) {
+ loc_network_unref(*network);
+ *network = NULL;
+ }
+
+ return 0;
+ }
+ }
+
+ // Reached the end of the search
- offset += fwrite(&header, 1, sizeof(header), f);
+ // Mark all nodes as non-visited
+ for (unsigned int i = 0; i < enumerator->db->network_nodes_count; i++)
+ enumerator->networks_visited[i] = 0;
return 0;
}