src/database.c

   1 /*
   2         libloc - A library to determine the location of someone on the Internet
   3
   4         Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
   5
   6         This library is free software; you can redistribute it and/or
   7         modify it under the terms of the GNU Lesser General Public
   8         License as published by the Free Software Foundation; either
   9         version 2.1 of the License, or (at your option) any later version.
  10
  11         This library is distributed in the hope that it will be useful,
  12         but WITHOUT ANY WARRANTY; without even the implied warranty of
  13         MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14         Lesser General Public License for more details.
  15 */
  16
  17 #include <endian.h>
  18 #include <errno.h>
  19 #include <stddef.h>
  20 #include <stdint.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <sys/mman.h>
  25 #include <sys/types.h>
  26 #include <time.h>
  27 #include <unistd.h>
  28
  29 #include <loc/libloc.h>
  30 #include <loc/as.h>
  31 #include <loc/database.h>
  32 #include <loc/format.h>
  33 #include <loc/private.h>
  34 #include <loc/stringpool.h>
  35
  36 struct loc_database {
  37         struct loc_ctx* ctx;
  38         int refcount;
  39
  40         unsigned int version;
  41         time_t created_at;
  42         off_t vendor;
  43         off_t description;
  44
  45         // ASes in the database
  46         struct loc_database_as_v0* as_v0;
  47         size_t as_count;
  48
  49         // Network tree
  50         struct loc_database_network_node_v0* network_nodes_v0;
  51         size_t network_nodes_count;
  52
  53         // Networks
  54         struct loc_database_network_v0* networks_v0;
  55         size_t networks_count;
  56
  57         struct loc_stringpool* pool;
  58 };
  59
  60 static int loc_database_read_magic(struct loc_database* db, FILE* f) {
  61         struct loc_database_magic magic;
  62
  63         // Read from file
  64         size_t bytes_read = fread(&magic, 1, sizeof(magic), f);
  65
  66         // Check if we have been able to read enough data
  67         if (bytes_read < sizeof(magic)) {
  68                 ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
  69                 DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
  70                 return -ENOMSG;
  71         }
  72
  73         // Compare magic bytes
  74         if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
  75                 DEBUG(db->ctx, "Magic value matches\n");
  76
  77                 // Parse version
  78                 db->version = be16toh(magic.version);
  79                 DEBUG(db->ctx, "Database version is %u\n", db->version);
  80
  81                 return 0;
  82         }
  83
  84         ERROR(db->ctx, "Database format is not compatible\n");
  85
  86         // Return an error
  87         return 1;
  88 }
  89
  90 static int loc_database_read_as_section_v0(struct loc_database* db,
  91                 FILE* f, const struct loc_database_header_v0* header) {
  92         off_t as_offset  = be32toh(header->as_offset);
  93         size_t as_length = be32toh(header->as_length);
  94
  95         DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", as_offset, as_length);
  96
  97         if (as_length > 0) {
  98                 db->as_v0 = mmap(NULL, as_length, PROT_READ,
  99                         MAP_SHARED, fileno(f), as_offset);
 100
 101                 if (db->as_v0 == MAP_FAILED)
 102                         return -errno;
 103         }
 104
 105         db->as_count = as_length / sizeof(*db->as_v0);
 106
 107         INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
 108
 109         return 0;
 110 }
 111
 112 static int loc_database_read_network_nodes_section_v0(struct loc_database* db,
 113                 FILE* f, const struct loc_database_header_v0* header) {
 114         off_t network_nodes_offset  = be32toh(header->network_tree_offset);
 115         size_t network_nodes_length = be32toh(header->network_tree_length);
 116
 117         DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n",
 118                 network_nodes_offset, network_nodes_length);
 119
 120         if (network_nodes_length > 0) {
 121                 db->network_nodes_v0 = mmap(NULL, network_nodes_length, PROT_READ,
 122                         MAP_SHARED, fileno(f), network_nodes_offset);
 123
 124                 if (db->network_nodes_v0 == MAP_FAILED)
 125                         return -errno;
 126         }
 127
 128         db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v0);
 129
 130         INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count);
 131
 132         return 0;
 133 }
 134
 135 static int loc_database_read_networks_section_v0(struct loc_database* db,
 136                 FILE* f, const struct loc_database_header_v0* header) {
 137         off_t networks_offset  = be32toh(header->network_data_offset);
 138         size_t networks_length = be32toh(header->network_data_length);
 139
 140         DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n",
 141                 networks_offset, networks_length);
 142
 143         if (networks_length > 0) {
 144                 db->networks_v0 = mmap(NULL, networks_length, PROT_READ,
 145                         MAP_SHARED, fileno(f), networks_offset);
 146
 147                 if (db->networks_v0 == MAP_FAILED)
 148                         return -errno;
 149         }
 150
 151         db->networks_count = networks_length / sizeof(*db->networks_v0);
 152
 153         INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count);
 154
 155         return 0;
 156 }
 157
 158 static int loc_database_read_header_v0(struct loc_database* db, FILE* f) {
 159         struct loc_database_header_v0 header;
 160
 161         // Read from file
 162         size_t size = fread(&header, 1, sizeof(header), f);
 163
 164         if (size < sizeof(header)) {
 165                 ERROR(db->ctx, "Could not read enough data for header\n");
 166                 return -ENOMSG;
 167         }
 168
 169         // Copy over data
 170         db->created_at  = be64toh(header.created_at);
 171         db->vendor      = be32toh(header.vendor);
 172         db->description = be32toh(header.description);
 173
 174         // Open pool
 175         off_t pool_offset  = be32toh(header.pool_offset);
 176         size_t pool_length = be32toh(header.pool_length);
 177
 178         int r = loc_stringpool_open(db->ctx, &db->pool,
 179                 f, pool_length, pool_offset);
 180         if (r)
 181                 return r;
 182
 183         // AS section
 184         r = loc_database_read_as_section_v0(db, f, &header);
 185         if (r)
 186                 return r;
 187
 188         // Network Nodes
 189         r = loc_database_read_network_nodes_section_v0(db, f, &header);
 190         if (r)
 191                 return r;
 192
 193         // Networks
 194         r = loc_database_read_networks_section_v0(db, f, &header);
 195         if (r)
 196                 return r;
 197
 198         return 0;
 199 }
 200
 201 static int loc_database_read_header(struct loc_database* db, FILE* f) {
 202         switch (db->version) {
 203                 case 0:
 204                         return loc_database_read_header_v0(db, f);
 205
 206                 default:
 207                         ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
 208                         return 1;
 209         }
 210 }
 211
 212 static int loc_database_read(struct loc_database* db, FILE* f) {
 213         clock_t start = clock();
 214
 215         // Read magic bytes
 216         int r = loc_database_read_magic(db, f);
 217         if (r)
 218                 return r;
 219
 220         // Read the header
 221         r = loc_database_read_header(db, f);
 222         if (r)
 223                 return r;
 224
 225         clock_t end = clock();
 226
 227         INFO(db->ctx, "Opened database in %.8fs\n",
 228                 (double)(end - start) / CLOCKS_PER_SEC);
 229
 230         return 0;
 231 }
 232
 233 LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
 234         // Fail on invalid file handle
 235         if (!f)
 236                 return -EINVAL;
 237
 238         struct loc_database* db = calloc(1, sizeof(*db));
 239         if (!db)
 240                 return -ENOMEM;
 241
 242         // Reference context
 243         db->ctx = loc_ref(ctx);
 244         db->refcount = 1;
 245
 246         DEBUG(db->ctx, "Database object allocated at %p\n", db);
 247
 248         int r = loc_database_read(db, f);
 249         if (r) {
 250                 loc_database_unref(db);
 251                 return r;
 252         }
 253
 254         *database = db;
 255
 256         return 0;
 257 }
 258
 259 LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
 260         db->refcount++;
 261
 262         return db;
 263 }
 264
 265 static void loc_database_free(struct loc_database* db) {
 266         DEBUG(db->ctx, "Releasing database %p\n", db);
 267
 268         // Removing all ASes
 269         if (db->as_v0) {
 270                 int r = munmap(db->as_v0, db->as_count * sizeof(*db->as_v0));
 271                 if (r)
 272                         ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno));
 273         }
 274
 275         loc_stringpool_unref(db->pool);
 276
 277         loc_unref(db->ctx);
 278         free(db);
 279 }
 280
 281 LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
 282         if (--db->refcount > 0)
 283                 return NULL;
 284
 285         loc_database_free(db);
 286         return NULL;
 287 }
 288
 289 LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
 290         return db->created_at;
 291 }
 292
 293 LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
 294         return loc_stringpool_get(db->pool, db->vendor);
 295 }
 296
 297 LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
 298         return loc_stringpool_get(db->pool, db->description);
 299 }
 300
 301 LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
 302         return db->as_count;
 303 }
 304
 305 // Returns the AS at position pos
 306 static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
 307         if ((size_t)pos >= db->as_count)
 308                 return -EINVAL;
 309
 310         DEBUG(db->ctx, "Fetching AS at position %jd\n", pos);
 311
 312         int r;
 313         switch (db->version) {
 314                 case 0:
 315                         r = loc_as_new_from_database_v0(db->ctx, db->pool, as, db->as_v0 + pos);
 316                         break;
 317
 318                 default:
 319                         return -1;
 320         }
 321
 322         if (r == 0) {
 323                 DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as));
 324         }
 325
 326         return r;
 327 }
 328
 329 // Performs a binary search to find the AS in the list
 330 LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) {
 331         off_t lo = 0;
 332         off_t hi = db->as_count - 1;
 333
 334         // Save start time
 335         clock_t start = clock();
 336
 337         while (lo <= hi) {
 338                 off_t i = (lo + hi) / 2;
 339
 340                 // Fetch AS in the middle between lo and hi
 341                 int r = loc_database_fetch_as(db, as, i);
 342                 if (r)
 343                         return r;
 344
 345                 // Check if this is a match
 346                 uint32_t as_number = loc_as_get_number(*as);
 347                 if (as_number == number) {
 348                         clock_t end = clock();
 349
 350                         // Log how fast this has been
 351                         DEBUG(db->ctx, "Found AS%u in %.8fs\n", as_number,
 352                                 (double)(end - start) / CLOCKS_PER_SEC);
 353
 354                         return 0;
 355                 }
 356
 357                 // If it wasn't, we release the AS and
 358                 // adjust our search pointers
 359                 loc_as_unref(*as);
 360
 361                 if (as_number < number) {
 362                         lo = i + 1;
 363                 } else
 364                         hi = i - 1;
 365         }
 366
 367         // Nothing found
 368         *as = NULL;
 369
 370         return 1;
 371 }