database: Load networks from database
[people/ms/libloc.git] / src / database.c
CommitLineData
2601e83e
MT
1/*
2 libloc - A library to determine the location of someone on the Internet
3
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
5
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15*/
16
0676cd80 17#include <endian.h>
2601e83e
MT
18#include <errno.h>
19#include <stddef.h>
20#include <stdint.h>
21#include <stdio.h>
22#include <stdlib.h>
23#include <string.h>
c182393f 24#include <sys/mman.h>
2601e83e 25#include <sys/types.h>
96ea74a5 26#include <time.h>
3f35869a 27#include <unistd.h>
2601e83e
MT
28
29#include <loc/libloc.h>
9fc7f001
MT
30#include <loc/as.h>
31#include <loc/database.h>
a5db3e49 32#include <loc/format.h>
9fc7f001
MT
33#include <loc/private.h>
34#include <loc/stringpool.h>
2601e83e
MT
35
36struct loc_database {
37 struct loc_ctx* ctx;
38 int refcount;
39
40 unsigned int version;
96ea74a5 41 time_t created_at;
2601e83e
MT
42 off_t vendor;
43 off_t description;
44
a5db3e49 45 // ASes in the database
c182393f 46 struct loc_database_as_v0* as_v0;
a5db3e49
MT
47 size_t as_count;
48
f66b7b09
MT
49 // Network tree
50 struct loc_database_network_node_v0* network_nodes_v0;
51 size_t network_nodes_count;
52
a735a563
MT
53 // Networks
54 struct loc_database_network_v0* networks_v0;
55 size_t networks_count;
56
2601e83e
MT
57 struct loc_stringpool* pool;
58};
59
a7431f1a 60static int loc_database_read_magic(struct loc_database* db, FILE* f) {
2601e83e
MT
61 struct loc_database_magic magic;
62
63 // Read from file
a7431f1a 64 size_t bytes_read = fread(&magic, 1, sizeof(magic), f);
2601e83e
MT
65
66 // Check if we have been able to read enough data
67 if (bytes_read < sizeof(magic)) {
68 ERROR(db->ctx, "Could not read enough data to validate magic bytes\n");
69 DEBUG(db->ctx, "Read %zu bytes, but needed %zu\n", bytes_read, sizeof(magic));
70 return -ENOMSG;
71 }
72
73 // Compare magic bytes
74 if (memcmp(LOC_DATABASE_MAGIC, magic.magic, strlen(LOC_DATABASE_MAGIC)) == 0) {
75 DEBUG(db->ctx, "Magic value matches\n");
76
77 // Parse version
0676cd80 78 db->version = be16toh(magic.version);
2601e83e
MT
79 DEBUG(db->ctx, "Database version is %u\n", db->version);
80
81 return 0;
82 }
83
84 ERROR(db->ctx, "Database format is not compatible\n");
85
86 // Return an error
87 return 1;
88}
89
a5db3e49 90static int loc_database_read_as_section_v0(struct loc_database* db,
edb4ba7c
MT
91 FILE* f, const struct loc_database_header_v0* header) {
92 off_t as_offset = be32toh(header->as_offset);
93 size_t as_length = be32toh(header->as_length);
94
c182393f 95 DEBUG(db->ctx, "Reading AS section from %jd (%zu bytes)\n", as_offset, as_length);
a5db3e49 96
c182393f
MT
97 if (as_length > 0) {
98 db->as_v0 = mmap(NULL, as_length, PROT_READ,
a7431f1a 99 MAP_SHARED, fileno(f), as_offset);
a5db3e49 100
c182393f
MT
101 if (db->as_v0 == MAP_FAILED)
102 return -errno;
a5db3e49
MT
103 }
104
c182393f
MT
105 db->as_count = as_length / sizeof(*db->as_v0);
106
a5db3e49
MT
107 INFO(db->ctx, "Read %zu ASes from the database\n", db->as_count);
108
109 return 0;
110}
111
f66b7b09 112static int loc_database_read_network_nodes_section_v0(struct loc_database* db,
edb4ba7c
MT
113 FILE* f, const struct loc_database_header_v0* header) {
114 off_t network_nodes_offset = be32toh(header->network_tree_offset);
115 size_t network_nodes_length = be32toh(header->network_tree_length);
116
f66b7b09
MT
117 DEBUG(db->ctx, "Reading network nodes section from %jd (%zu bytes)\n",
118 network_nodes_offset, network_nodes_length);
119
120 if (network_nodes_length > 0) {
121 db->network_nodes_v0 = mmap(NULL, network_nodes_length, PROT_READ,
122 MAP_SHARED, fileno(f), network_nodes_offset);
123
124 if (db->network_nodes_v0 == MAP_FAILED)
125 return -errno;
126 }
127
128 db->network_nodes_count = network_nodes_length / sizeof(*db->network_nodes_v0);
129
130 INFO(db->ctx, "Read %zu network nodes from the database\n", db->network_nodes_count);
131
132 return 0;
133}
134
a735a563
MT
135static int loc_database_read_networks_section_v0(struct loc_database* db,
136 FILE* f, const struct loc_database_header_v0* header) {
137 off_t networks_offset = be32toh(header->network_data_offset);
138 size_t networks_length = be32toh(header->network_data_length);
139
140 DEBUG(db->ctx, "Reading networks section from %jd (%zu bytes)\n",
141 networks_offset, networks_length);
142
143 if (networks_length > 0) {
144 db->networks_v0 = mmap(NULL, networks_length, PROT_READ,
145 MAP_SHARED, fileno(f), networks_offset);
146
147 if (db->networks_v0 == MAP_FAILED)
148 return -errno;
149 }
150
151 db->networks_count = networks_length / sizeof(*db->networks_v0);
152
153 INFO(db->ctx, "Read %zu networks from the database\n", db->networks_count);
154
155 return 0;
156}
157
a7431f1a 158static int loc_database_read_header_v0(struct loc_database* db, FILE* f) {
2601e83e
MT
159 struct loc_database_header_v0 header;
160
161 // Read from file
a7431f1a 162 size_t size = fread(&header, 1, sizeof(header), f);
2601e83e
MT
163
164 if (size < sizeof(header)) {
165 ERROR(db->ctx, "Could not read enough data for header\n");
166 return -ENOMSG;
167 }
168
169 // Copy over data
96ea74a5 170 db->created_at = be64toh(header.created_at);
0676cd80
MT
171 db->vendor = be32toh(header.vendor);
172 db->description = be32toh(header.description);
2601e83e
MT
173
174 // Open pool
0676cd80
MT
175 off_t pool_offset = be32toh(header.pool_offset);
176 size_t pool_length = be32toh(header.pool_length);
2601e83e 177
0e974d4b 178 int r = loc_stringpool_open(db->ctx, &db->pool,
a7431f1a 179 f, pool_length, pool_offset);
2601e83e
MT
180 if (r)
181 return r;
182
a5db3e49 183 // AS section
edb4ba7c 184 r = loc_database_read_as_section_v0(db, f, &header);
a5db3e49
MT
185 if (r)
186 return r;
187
f66b7b09 188 // Network Nodes
edb4ba7c 189 r = loc_database_read_network_nodes_section_v0(db, f, &header);
f66b7b09
MT
190 if (r)
191 return r;
192
a735a563
MT
193 // Networks
194 r = loc_database_read_networks_section_v0(db, f, &header);
195 if (r)
196 return r;
197
2601e83e
MT
198 return 0;
199}
200
a7431f1a 201static int loc_database_read_header(struct loc_database* db, FILE* f) {
2601e83e
MT
202 switch (db->version) {
203 case 0:
a7431f1a 204 return loc_database_read_header_v0(db, f);
2601e83e
MT
205
206 default:
207 ERROR(db->ctx, "Incompatible database version: %u\n", db->version);
208 return 1;
209 }
210}
211
a7431f1a 212static int loc_database_read(struct loc_database* db, FILE* f) {
02879100
MT
213 clock_t start = clock();
214
215 // Read magic bytes
a7431f1a 216 int r = loc_database_read_magic(db, f);
02879100
MT
217 if (r)
218 return r;
219
220 // Read the header
a7431f1a 221 r = loc_database_read_header(db, f);
02879100
MT
222 if (r)
223 return r;
224
225 clock_t end = clock();
226
227 INFO(db->ctx, "Opened database in %.8fs\n",
228 (double)(end - start) / CLOCKS_PER_SEC);
229
230 return 0;
231}
232
c182393f 233LOC_EXPORT int loc_database_new(struct loc_ctx* ctx, struct loc_database** database, FILE* f) {
a7431f1a
MT
234 // Fail on invalid file handle
235 if (!f)
236 return -EINVAL;
237
c182393f
MT
238 struct loc_database* db = calloc(1, sizeof(*db));
239 if (!db)
240 return -ENOMEM;
241
242 // Reference context
243 db->ctx = loc_ref(ctx);
244 db->refcount = 1;
245
246 DEBUG(db->ctx, "Database object allocated at %p\n", db);
247
a7431f1a 248 int r = loc_database_read(db, f);
02879100
MT
249 if (r) {
250 loc_database_unref(db);
2601e83e 251 return r;
02879100 252 }
2601e83e 253
c182393f
MT
254 *database = db;
255
2601e83e 256 return 0;
2601e83e
MT
257}
258
c182393f
MT
259LOC_EXPORT struct loc_database* loc_database_ref(struct loc_database* db) {
260 db->refcount++;
261
262 return db;
8f5b676a
MT
263}
264
c182393f
MT
265static void loc_database_free(struct loc_database* db) {
266 DEBUG(db->ctx, "Releasing database %p\n", db);
c34e76f1 267
c182393f
MT
268 // Removing all ASes
269 if (db->as_v0) {
270 int r = munmap(db->as_v0, db->as_count * sizeof(*db->as_v0));
271 if (r)
272 ERROR(db->ctx, "Could not unmap AS section: %s\n", strerror(errno));
273 }
c34e76f1 274
c182393f 275 loc_stringpool_unref(db->pool);
c34e76f1 276
c182393f
MT
277 loc_unref(db->ctx);
278 free(db);
c34e76f1
MT
279}
280
c182393f
MT
281LOC_EXPORT struct loc_database* loc_database_unref(struct loc_database* db) {
282 if (--db->refcount > 0)
283 return NULL;
78ace4ed 284
c182393f
MT
285 loc_database_free(db);
286 return NULL;
287}
78ace4ed 288
c182393f
MT
289LOC_EXPORT time_t loc_database_created_at(struct loc_database* db) {
290 return db->created_at;
291}
78ace4ed 292
c182393f
MT
293LOC_EXPORT const char* loc_database_get_vendor(struct loc_database* db) {
294 return loc_stringpool_get(db->pool, db->vendor);
295}
78ace4ed 296
c182393f
MT
297LOC_EXPORT const char* loc_database_get_description(struct loc_database* db) {
298 return loc_stringpool_get(db->pool, db->description);
299}
78ace4ed 300
c182393f
MT
301LOC_EXPORT size_t loc_database_count_as(struct loc_database* db) {
302 return db->as_count;
78ace4ed
MT
303}
304
c182393f
MT
305// Returns the AS at position pos
306static int loc_database_fetch_as(struct loc_database* db, struct loc_as** as, off_t pos) {
307 if ((size_t)pos >= db->as_count)
308 return -EINVAL;
2601e83e 309
c182393f 310 DEBUG(db->ctx, "Fetching AS at position %jd\n", pos);
2601e83e
MT
311
312 int r;
c182393f
MT
313 switch (db->version) {
314 case 0:
315 r = loc_as_new_from_database_v0(db->ctx, db->pool, as, db->as_v0 + pos);
316 break;
2601e83e 317
c182393f
MT
318 default:
319 return -1;
320 }
2601e83e 321
c182393f
MT
322 if (r == 0) {
323 DEBUG(db->ctx, "Got AS%u\n", loc_as_get_number(*as));
2601e83e 324 }
2601e83e 325
c182393f
MT
326 return r;
327}
c34e76f1 328
c182393f
MT
329// Performs a binary search to find the AS in the list
330LOC_EXPORT int loc_database_get_as(struct loc_database* db, struct loc_as** as, uint32_t number) {
331 off_t lo = 0;
332 off_t hi = db->as_count - 1;
c34e76f1 333
8f3e2a06
MT
334 // Save start time
335 clock_t start = clock();
336
c182393f
MT
337 while (lo <= hi) {
338 off_t i = (lo + hi) / 2;
8f5b676a 339
c182393f
MT
340 // Fetch AS in the middle between lo and hi
341 int r = loc_database_fetch_as(db, as, i);
342 if (r)
343 return r;
a5db3e49 344
c182393f
MT
345 // Check if this is a match
346 uint32_t as_number = loc_as_get_number(*as);
8f3e2a06
MT
347 if (as_number == number) {
348 clock_t end = clock();
349
350 // Log how fast this has been
351 DEBUG(db->ctx, "Found AS%u in %.8fs\n", as_number,
352 (double)(end - start) / CLOCKS_PER_SEC);
353
c182393f 354 return 0;
8f3e2a06 355 }
c182393f
MT
356
357 // If it wasn't, we release the AS and
358 // adjust our search pointers
359 loc_as_unref(*as);
360
361 if (as_number < number) {
362 lo = i + 1;
363 } else
364 hi = i - 1;
365 }
2601e83e 366
c182393f
MT
367 // Nothing found
368 *as = NULL;
2601e83e 369
8f3e2a06 370 return 1;
2601e83e 371}