database: Implement reading a network from the database
[people/ms/libloc.git] / src / database.c
CommitLineData
2601e83e
MT
1/*
2 libloc - A library to determine the location of someone on the Internet
3
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
5
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15*/
16
0676cd80 17#include <endian.h>
2601e83e 18#include <errno.h>
10778041 19#include <netinet/in.h>
2601e83e
MT
20#include <stddef.h>
21#include <stdint.h>
22#include <stdio.h>
23#include <stdlib.h>
24#include <string.h>
c182393f 25#include <sys/mman.h>
2601e83e 26#include <sys/types.h>
96ea74a5 27#include <time.h>
3f35869a 28#include <unistd.h>
2601e83e
MT
29
30#include <loc/libloc.h>
9fc7f001
MT
31#include <loc/as.h>
32#include <loc/database.h>
a5db3e49 33#include <loc/format.h>
10778041 34#include <loc/network.h>
9fc7f001
MT
35#include <loc/private.h>
36#include <loc/stringpool.h>
2601e83e
MT
37
38struct loc_database {
39 struct loc_ctx* ctx;
40 int refcount;
41
42 unsigned int version;
96ea74a5 43 time_t created_at;
2601e83e
MT
44 off_t vendor;
45 off_t description;
46
a5db3e49 47 // ASes in the database
c182393f 48 struct loc_database_as_v0* as_v0;
a5db3e49
MT
49 size_t as_count;
50
f66b7b09
MT
51 // Network tree
52 struct loc_database_network_node_v0* network_nodes_v0;
53 size_t network_nodes_count;
54
a735a563
MT
55 // Networks
56 struct loc_database_network_v0* networks_v0;
57 size_t networks_count;
58
2601e83e
MT
59 struct loc_stringpool* pool;
60};
61
a7431f1a 62static int loc_database_read_magic(struct loc_database* db, FILE* f) {
2601e83e
MT
63 struct loc_database_magic magic;
64
65 // Read from file
a7431f1a 66 size_t bytes_read = fread(&magic, 1, sizeof(magic), f);
2601e83e
MT
67
68 // Check if we have been able to read enough data
69 if (bytes_read < sizeof(magic)) {
70 ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
71 DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
72 return -ENOMSG;
73 }
74
75 // Compare magic bytes
76 if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
77 DEBUG(db->ctx, "Magic value matches\n");
78
79 // Parse version
0676cd80 80 db->version = be16toh(magic.version);
2601e83e
MT
81 DEBUG(db->ctx, "Database version is %u\n", db->version);
82
83 return 0;
84 }
85
86 ERROR(db->ctx, "Database format is not compatible\n");
87
88 // Return an error
89 return 1;
90}
91
a5db3e49 92static int loc_database_read_as_section_v0(struct loc_database* db,
edb4ba7c
MT
93 FILE* f, const struct loc_database_header_v0* header) {
94 off_t as_offset = be32toh(header->as_offset);
95 size_t as_length = be32toh(header->as_length);
96
c182393f 97 DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", as_offset, as_length);
a5db3e49 98
c182393f
MT
99 if (as_length > 0) {
100 db->as_v0 = mmap(NULL, as_length, PROT_READ,
a7431f1a 101 MAP_SHARED, fileno(f), as_offset);
a5db3e49 102
c182393f
MT
103 if (db->as_v0 == MAP_FAILED)
104 return -errno;
a5db3e49
MT
105 }
106
c182393f
MT
107 db->as_count = as_length / sizeof(*db->as_v0);
108
a5db3e49
MT
109 INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
110
111 return 0;
112}
113
f66b7b09 114static int loc_database_read_network_nodes_section_v0(struct loc_database* db,
edb4ba7c
MT
115 FILE* f, const struct loc_database_header_v0* header) {
116 off_t network_nodes_offset = be32toh(header->network_tree_offset);
117 size_t network_nodes_length = be32toh(header->network_tree_length);
118
f66b7b09
MT
119 DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n",
120 network_nodes_offset, network_nodes_length);
121
122 if (network_nodes_length > 0) {
123 db->network_nodes_v0 = mmap(NULL, network_nodes_length, PROT_READ,
124 MAP_SHARED, fileno(f), network_nodes_offset);
125
126 if (db->network_nodes_v0 == MAP_FAILED)
127 return -errno;
128 }
129
130 db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v0);
131
132 INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count);
133
134 return 0;
135}
136
a735a563
MT
137static int loc_database_read_networks_section_v0(struct loc_database* db,
138 FILE* f, const struct loc_database_header_v0* header) {
139 off_t networks_offset = be32toh(header->network_data_offset);
140 size_t networks_length = be32toh(header->network_data_length);
141
142 DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n",
143 networks_offset, networks_length);
144
145 if (networks_length > 0) {
146 db->networks_v0 = mmap(NULL, networks_length, PROT_READ,
147 MAP_SHARED, fileno(f), networks_offset);
148
149 if (db->networks_v0 == MAP_FAILED)
150 return -errno;
151 }
152
153 db->networks_count = networks_length / sizeof(*db->networks_v0);
154
155 INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count);
156
157 return 0;
158}
159
a7431f1a 160static int loc_database_read_header_v0(struct loc_database* db, FILE* f) {
2601e83e
MT
161 struct loc_database_header_v0 header;
162
163 // Read from file
a7431f1a 164 size_t size = fread(&header, 1, sizeof(header), f);
2601e83e
MT
165
166 if (size < sizeof(header)) {
167 ERROR(db->ctx, "Could not read enough data for header\n");
168 return -ENOMSG;
169 }
170
171 // Copy over data
96ea74a5 172 db->created_at = be64toh(header.created_at);
0676cd80
MT
173 db->vendor = be32toh(header.vendor);
174 db->description = be32toh(header.description);
2601e83e
MT
175
176 // Open pool
0676cd80
MT
177 off_t pool_offset = be32toh(header.pool_offset);
178 size_t pool_length = be32toh(header.pool_length);
2601e83e 179
0e974d4b 180 int r = loc_stringpool_open(db->ctx, &db->pool,
a7431f1a 181 f, pool_length, pool_offset);
2601e83e
MT
182 if (r)
183 return r;
184
a5db3e49 185 // AS section
edb4ba7c 186 r = loc_database_read_as_section_v0(db, f, &header);
a5db3e49
MT
187 if (r)
188 return r;
189
f66b7b09 190 // Network Nodes
edb4ba7c 191 r = loc_database_read_network_nodes_section_v0(db, f, &header);
f66b7b09
MT
192 if (r)
193 return r;
194
a735a563
MT
195 // Networks
196 r = loc_database_read_networks_section_v0(db, f, &header);
197 if (r)
198 return r;
199
2601e83e
MT
200 return 0;
201}
202
a7431f1a 203static int loc_database_read_header(struct loc_database* db, FILE* f) {
2601e83e
MT
204 switch (db->version) {
205 case 0:
a7431f1a 206 return loc_database_read_header_v0(db, f);
2601e83e
MT
207
208 default:
209 ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
210 return 1;
211 }
212}
213
a7431f1a 214static int loc_database_read(struct loc_database* db, FILE* f) {
02879100
MT
215 clock_t start = clock();
216
217 // Read magic bytes
a7431f1a 218 int r = loc_database_read_magic(db, f);
02879100
MT
219 if (r)
220 return r;
221
222 // Read the header
a7431f1a 223 r = loc_database_read_header(db, f);
02879100
MT
224 if (r)
225 return r;
226
227 clock_t end = clock();
228
229 INFO(db->ctx, "Opened database in %.8fs\n",
230 (double)(end - start) / CLOCKS_PER_SEC);
231
232 return 0;
233}
234
c182393f 235LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
a7431f1a
MT
236 // Fail on invalid file handle
237 if (!f)
238 return -EINVAL;
239
c182393f
MT
240 struct loc_database* db = calloc(1, sizeof(*db));
241 if (!db)
242 return -ENOMEM;
243
244 // Reference context
245 db->ctx = loc_ref(ctx);
246 db->refcount = 1;
247
248 DEBUG(db->ctx, "Database object allocated at %p\n", db);
249
a7431f1a 250 int r = loc_database_read(db, f);
02879100
MT
251 if (r) {
252 loc_database_unref(db);
2601e83e 253 return r;
02879100 254 }
2601e83e 255
c182393f
MT
256 *database = db;
257
2601e83e 258 return 0;
2601e83e
MT
259}
260
c182393f
MT
261LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
262 db->refcount++;
263
264 return db;
8f5b676a
MT
265}
266
c182393f
MT
267static void loc_database_free(struct loc_database* db) {
268 DEBUG(db->ctx, "Releasing database %p\n", db);
c34e76f1 269
c182393f
MT
270 // Removing all ASes
271 if (db->as_v0) {
272 int r = munmap(db->as_v0, db->as_count * sizeof(*db->as_v0));
273 if (r)
274 ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno));
275 }
c34e76f1 276
c182393f 277 loc_stringpool_unref(db->pool);
c34e76f1 278
c182393f
MT
279 loc_unref(db->ctx);
280 free(db);
c34e76f1
MT
281}
282
c182393f
MT
283LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
284 if (--db->refcount > 0)
285 return NULL;
78ace4ed 286
c182393f
MT
287 loc_database_free(db);
288 return NULL;
289}
78ace4ed 290
c182393f
MT
291LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
292 return db->created_at;
293}
78ace4ed 294
c182393f
MT
295LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
296 return loc_stringpool_get(db->pool, db->vendor);
297}
78ace4ed 298
c182393f
MT
299LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
300 return loc_stringpool_get(db->pool, db->description);
301}
78ace4ed 302
c182393f
MT
303LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
304 return db->as_count;
78ace4ed
MT
305}
306
c182393f
MT
307// Returns the AS at position pos
308static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
309 if ((size_t)pos >= db->as_count)
310 return -EINVAL;
2601e83e 311
c182393f 312 DEBUG(db->ctx, "Fetching AS at position %jd\n", pos);
2601e83e
MT
313
314 int r;
c182393f
MT
315 switch (db->version) {
316 case 0:
317 r = loc_as_new_from_database_v0(db->ctx, db->pool, as, db->as_v0 + pos);
318 break;
2601e83e 319
c182393f
MT
320 default:
321 return -1;
322 }
2601e83e 323
c182393f
MT
324 if (r == 0) {
325 DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as));
2601e83e 326 }
2601e83e 327
c182393f
MT
328 return r;
329}
c34e76f1 330
c182393f
MT
331// Performs a binary search to find the AS in the list
332LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) {
333 off_t lo = 0;
334 off_t hi = db->as_count - 1;
c34e76f1 335
8f3e2a06
MT
336 // Save start time
337 clock_t start = clock();
338
c182393f
MT
339 while (lo <= hi) {
340 off_t i = (lo + hi) / 2;
8f5b676a 341
c182393f
MT
342 // Fetch AS in the middle between lo and hi
343 int r = loc_database_fetch_as(db, as, i);
344 if (r)
345 return r;
a5db3e49 346
c182393f
MT
347 // Check if this is a match
348 uint32_t as_number = loc_as_get_number(*as);
8f3e2a06
MT
349 if (as_number == number) {
350 clock_t end = clock();
351
352 // Log how fast this has been
353 DEBUG(db->ctx, "Found AS%u in %.8fs\n", as_number,
354 (double)(end - start) / CLOCKS_PER_SEC);
355
c182393f 356 return 0;
8f3e2a06 357 }
c182393f
MT
358
359 // If it wasn't, we release the AS and
360 // adjust our search pointers
361 loc_as_unref(*as);
362
363 if (as_number < number) {
364 lo = i + 1;
365 } else
366 hi = i - 1;
367 }
2601e83e 368
c182393f
MT
369 // Nothing found
370 *as = NULL;
2601e83e 371
8f3e2a06 372 return 1;
2601e83e 373}
10778041
MT
374
375// Returns the network at position pos
376static int loc_database_fetch_network(struct loc_database* db, struct loc_network** network, struct in6_addr* address, off_t pos) {
377 if ((size_t)pos >= db->networks_count)
378 return -EINVAL;
379
380 DEBUG(db->ctx, "Fetching network at position %jd\n", pos);
381
382 int r;
383 switch (db->version) {
384 case 0:
385 r = loc_network_new_from_database_v0(db->ctx, network, address, db->networks_v0 + pos);
386 break;
387
388 default:
389 return -1;
390 }
391
392 if (r == 0) {
393 char* string = loc_network_str(*network);
394 DEBUG(db->ctx, "Got network %s\n", string);
395 free(string);
396 }
397
398 return r;
399}