2 libloc - A library to determine the location of someone on the Internet
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
17 #include <arpa/inet.h>
21 #include <netinet/in.h>
28 #include <sys/types.h>
32 #include <loc/libloc.h>
34 #include <loc/database.h>
35 #include <loc/format.h>
36 #include <loc/network.h>
37 #include <loc/private.h>
38 #include <loc/stringpool.h>
50 // ASes in the database
51 struct loc_database_as_v0
* as_v0
;
55 struct loc_database_network_node_v0
* network_nodes_v0
;
56 size_t network_nodes_count
;
59 struct loc_database_network_v0
* networks_v0
;
60 size_t networks_count
;
62 struct loc_stringpool
* pool
;
65 struct loc_database_enumerator
{
67 struct loc_database
* db
;
73 // Index of the AS we are looking at
74 unsigned int as_index
;
77 static int loc_database_read_magic(struct loc_database
* db
, FILE* f
) {
78 struct loc_database_magic magic
;
81 size_t bytes_read
= fread(&magic
, 1, sizeof(magic
), f
);
83 // Check if we have been able to read enough data
84 if (bytes_read
< sizeof(magic
)) {
85 ERROR(db
->ctx
, "Could not read enough data to validate magic bytes\n");
86 DEBUG(db
->ctx
, "Read %zu bytes, but needed %zu\n", bytes_read
, sizeof(magic
));
90 // Compare magic bytes
91 if (memcmp(LOC_DATABASE_MAGIC
, magic
.magic
, strlen(LOC_DATABASE_MAGIC
)) == 0) {
92 DEBUG(db
->ctx
, "Magic value matches\n");
95 db
->version
= be16toh(magic
.version
);
96 DEBUG(db
->ctx
, "Database version is %u\n", db
->version
);
101 ERROR(db
->ctx
, "Database format is not compatible\n");
107 static int loc_database_read_as_section_v0(struct loc_database
* db
,
108 FILE* f
, const struct loc_database_header_v0
* header
) {
109 off_t as_offset
= be32toh(header
->as_offset
);
110 size_t as_length
= be32toh(header
->as_length
);
112 DEBUG(db
->ctx
, "Reading AS section from %jd (%zu bytes)\n", as_offset
, as_length
);
115 db
->as_v0
= mmap(NULL
, as_length
, PROT_READ
,
116 MAP_SHARED
, fileno(f
), as_offset
);
118 if (db
->as_v0
== MAP_FAILED
)
122 db
->as_count
= as_length
/ sizeof(*db
->as_v0
);
124 INFO(db
->ctx
, "Read %zu ASes from the database\n", db
->as_count
);
129 static int loc_database_read_network_nodes_section_v0(struct loc_database
* db
,
130 FILE* f
, const struct loc_database_header_v0
* header
) {
131 off_t network_nodes_offset
= be32toh(header
->network_tree_offset
);
132 size_t network_nodes_length
= be32toh(header
->network_tree_length
);
134 DEBUG(db
->ctx
, "Reading network nodes section from %jd (%zu bytes)\n",
135 network_nodes_offset
, network_nodes_length
);
137 if (network_nodes_length
> 0) {
138 db
->network_nodes_v0
= mmap(NULL
, network_nodes_length
, PROT_READ
,
139 MAP_SHARED
, fileno(f
), network_nodes_offset
);
141 if (db
->network_nodes_v0
== MAP_FAILED
)
145 db
->network_nodes_count
= network_nodes_length
/ sizeof(*db
->network_nodes_v0
);
147 INFO(db
->ctx
, "Read %zu network nodes from the database\n", db
->network_nodes_count
);
152 static int loc_database_read_networks_section_v0(struct loc_database
* db
,
153 FILE* f
, const struct loc_database_header_v0
* header
) {
154 off_t networks_offset
= be32toh(header
->network_data_offset
);
155 size_t networks_length
= be32toh(header
->network_data_length
);
157 DEBUG(db
->ctx
, "Reading networks section from %jd (%zu bytes)\n",
158 networks_offset
, networks_length
);
160 if (networks_length
> 0) {
161 db
->networks_v0
= mmap(NULL
, networks_length
, PROT_READ
,
162 MAP_SHARED
, fileno(f
), networks_offset
);
164 if (db
->networks_v0
== MAP_FAILED
)
168 db
->networks_count
= networks_length
/ sizeof(*db
->networks_v0
);
170 INFO(db
->ctx
, "Read %zu networks from the database\n", db
->networks_count
);
175 static int loc_database_read_header_v0(struct loc_database
* db
, FILE* f
) {
176 struct loc_database_header_v0 header
;
179 size_t size
= fread(&header
, 1, sizeof(header
), f
);
181 if (size
< sizeof(header
)) {
182 ERROR(db
->ctx
, "Could not read enough data for header\n");
187 db
->created_at
= be64toh(header
.created_at
);
188 db
->vendor
= be32toh(header
.vendor
);
189 db
->description
= be32toh(header
.description
);
190 db
->license
= be32toh(header
.license
);
193 off_t pool_offset
= be32toh(header
.pool_offset
);
194 size_t pool_length
= be32toh(header
.pool_length
);
196 int r
= loc_stringpool_open(db
->ctx
, &db
->pool
,
197 f
, pool_length
, pool_offset
);
202 r
= loc_database_read_as_section_v0(db
, f
, &header
);
207 r
= loc_database_read_network_nodes_section_v0(db
, f
, &header
);
212 r
= loc_database_read_networks_section_v0(db
, f
, &header
);
219 static int loc_database_read_header(struct loc_database
* db
, FILE* f
) {
220 switch (db
->version
) {
222 return loc_database_read_header_v0(db
, f
);
225 ERROR(db
->ctx
, "Incompatible database version: %u\n", db
->version
);
230 static int loc_database_read(struct loc_database
* db
, FILE* f
) {
231 clock_t start
= clock();
234 int r
= loc_database_read_magic(db
, f
);
239 r
= loc_database_read_header(db
, f
);
243 clock_t end
= clock();
245 INFO(db
->ctx
, "Opened database in %.8fs\n",
246 (double)(end
- start
) / CLOCKS_PER_SEC
);
251 LOC_EXPORT
int loc_database_new(struct loc_ctx
* ctx
, struct loc_database
** database
, FILE* f
) {
252 // Fail on invalid file handle
256 struct loc_database
* db
= calloc(1, sizeof(*db
));
261 db
->ctx
= loc_ref(ctx
);
264 DEBUG(db
->ctx
, "Database object allocated at %p\n", db
);
266 int r
= loc_database_read(db
, f
);
268 loc_database_unref(db
);
277 LOC_EXPORT
struct loc_database
* loc_database_ref(struct loc_database
* db
) {
283 static void loc_database_free(struct loc_database
* db
) {
286 DEBUG(db
->ctx
, "Releasing database %p\n", db
);
290 r
= munmap(db
->as_v0
, db
->as_count
* sizeof(*db
->as_v0
));
292 ERROR(db
->ctx
, "Could not unmap AS section: %s\n", strerror(errno
));
295 // Remove mapped network sections
296 if (db
->networks_v0
) {
297 r
= munmap(db
->networks_v0
, db
->networks_count
* sizeof(*db
->networks_v0
));
299 ERROR(db
->ctx
, "Could not unmap networks section: %s\n", strerror(errno
));
302 // Remove mapped network nodes section
303 if (db
->network_nodes_v0
) {
304 r
= munmap(db
->network_nodes_v0
, db
->network_nodes_count
* sizeof(*db
->network_nodes_v0
));
306 ERROR(db
->ctx
, "Could not unmap network nodes section: %s\n", strerror(errno
));
309 loc_stringpool_unref(db
->pool
);
315 LOC_EXPORT
struct loc_database
* loc_database_unref(struct loc_database
* db
) {
316 if (--db
->refcount
> 0)
319 loc_database_free(db
);
323 LOC_EXPORT
time_t loc_database_created_at(struct loc_database
* db
) {
324 return db
->created_at
;
327 LOC_EXPORT
const char* loc_database_get_vendor(struct loc_database
* db
) {
328 return loc_stringpool_get(db
->pool
, db
->vendor
);
331 LOC_EXPORT
const char* loc_database_get_description(struct loc_database
* db
) {
332 return loc_stringpool_get(db
->pool
, db
->description
);
335 LOC_EXPORT
const char* loc_database_get_license(struct loc_database
* db
) {
336 return loc_stringpool_get(db
->pool
, db
->license
);
339 LOC_EXPORT
size_t loc_database_count_as(struct loc_database
* db
) {
343 // Returns the AS at position pos
344 static int loc_database_fetch_as(struct loc_database
* db
, struct loc_as
** as
, off_t pos
) {
345 if ((size_t)pos
>= db
->as_count
)
348 DEBUG(db
->ctx
, "Fetching AS at position %jd\n", pos
);
351 switch (db
->version
) {
353 r
= loc_as_new_from_database_v0(db
->ctx
, db
->pool
, as
, db
->as_v0
+ pos
);
361 DEBUG(db
->ctx
, "Got AS%u\n", loc_as_get_number(*as
));
367 // Performs a binary search to find the AS in the list
368 LOC_EXPORT
int loc_database_get_as(struct loc_database
* db
, struct loc_as
** as
, uint32_t number
) {
370 off_t hi
= db
->as_count
- 1;
373 clock_t start
= clock();
376 off_t i
= (lo
+ hi
) / 2;
378 // Fetch AS in the middle between lo and hi
379 int r
= loc_database_fetch_as(db
, as
, i
);
383 // Check if this is a match
384 uint32_t as_number
= loc_as_get_number(*as
);
385 if (as_number
== number
) {
386 clock_t end
= clock();
388 // Log how fast this has been
389 DEBUG(db
->ctx
, "Found AS%u in %.8fs\n", as_number
,
390 (double)(end
- start
) / CLOCKS_PER_SEC
);
395 // If it wasn't, we release the AS and
396 // adjust our search pointers
399 if (as_number
< number
) {
411 // Returns the network at position pos
412 static int loc_database_fetch_network(struct loc_database
* db
, struct loc_network
** network
,
413 struct in6_addr
* address
, unsigned int prefix
, off_t pos
) {
414 if ((size_t)pos
>= db
->networks_count
)
417 DEBUG(db
->ctx
, "Fetching network at position %jd\n", pos
);
420 switch (db
->version
) {
422 r
= loc_network_new_from_database_v0(db
->ctx
, network
,
423 address
, prefix
, db
->networks_v0
+ pos
);
431 char* string
= loc_network_str(*network
);
432 DEBUG(db
->ctx
, "Got network %s\n", string
);
439 static int __loc_database_node_is_leaf(const struct loc_database_network_node_v0
* node
) {
440 return (node
->network
!= htobe32(0xffffffff));
443 static int __loc_database_lookup_handle_leaf(struct loc_database
* db
, const struct in6_addr
* address
,
444 struct loc_network
** network
, struct in6_addr
* network_address
, unsigned int prefix
,
445 const struct loc_database_network_node_v0
* node
) {
446 off_t network_index
= be32toh(node
->network
);
448 DEBUG(db
->ctx
, "Handling leaf node at %jd (%jd)\n", node
- db
->network_nodes_v0
, network_index
);
451 int r
= loc_database_fetch_network(db
, network
,
452 network_address
, prefix
, network_index
);
454 ERROR(db
->ctx
, "Could not fetch network %jd from database\n", network_index
);
458 // Check if the given IP address is inside the network
459 r
= loc_network_match_address(*network
, address
);
461 DEBUG(db
->ctx
, "Searched address is not part of the network\n");
463 loc_network_unref(*network
);
468 // A network was found and the IP address matches
472 // Searches for an exact match along the path
473 static int __loc_database_lookup(struct loc_database
* db
, const struct in6_addr
* address
,
474 struct loc_network
** network
, struct in6_addr
* network_address
,
475 const struct loc_database_network_node_v0
* node
, unsigned int level
) {
480 int bit
= in6_addr_get_bit(address
, level
);
481 in6_addr_set_bit(network_address
, level
, bit
);
484 node_index
= be32toh(node
->zero
);
486 node_index
= be32toh(node
->one
);
488 // If the node index is zero, the tree ends here
489 // and we cannot descend any further
490 if (node_index
> 0) {
492 if ((size_t)node_index
>= db
->network_nodes_count
)
495 // Move on to the next node
496 r
= __loc_database_lookup(db
, address
, network
, network_address
,
497 db
->network_nodes_v0
+ node_index
, level
+ 1);
499 // End here if a result was found
507 DEBUG(db
->ctx
, "No match found below level %u\n", level
);
509 DEBUG(db
->ctx
, "Tree ended at level %u\n", level
);
512 // If this node has a leaf, we will check if it matches
513 if (__loc_database_node_is_leaf(node
)) {
514 r
= __loc_database_lookup_handle_leaf(db
, address
, network
, network_address
, level
, node
);
522 LOC_EXPORT
int loc_database_lookup(struct loc_database
* db
,
523 struct in6_addr
* address
, struct loc_network
** network
) {
524 struct in6_addr network_address
;
525 memset(&network_address
, 0, sizeof(network_address
));
530 clock_t start
= clock();
532 int r
= __loc_database_lookup(db
, address
, network
, &network_address
,
533 db
->network_nodes_v0
, 0);
535 clock_t end
= clock();
537 // Log how fast this has been
538 DEBUG(db
->ctx
, "Executed network search in %.8fs\n",
539 (double)(end
- start
) / CLOCKS_PER_SEC
);
544 LOC_EXPORT
int loc_database_lookup_from_string(struct loc_database
* db
,
545 const char* string
, struct loc_network
** network
) {
546 struct in6_addr address
;
548 int r
= loc_parse_address(db
->ctx
, string
, &address
);
552 return loc_database_lookup(db
, &address
, network
);
557 LOC_EXPORT
int loc_database_enumerator_new(struct loc_database_enumerator
** enumerator
, struct loc_database
* db
) {
558 struct loc_database_enumerator
* e
= calloc(1, sizeof(*e
));
563 e
->ctx
= loc_ref(db
->ctx
);
564 e
->db
= loc_database_ref(db
);
567 DEBUG(e
->ctx
, "Database enumerator object allocated at %p\n", e
);
573 LOC_EXPORT
struct loc_database_enumerator
* loc_database_enumerator_ref(struct loc_database_enumerator
* enumerator
) {
574 enumerator
->refcount
++;
579 static void loc_database_enumerator_free(struct loc_database_enumerator
* enumerator
) {
580 DEBUG(enumerator
->ctx
, "Releasing database enumerator %p\n", enumerator
);
582 // Release all references
583 loc_database_unref(enumerator
->db
);
584 loc_unref(enumerator
->ctx
);
586 if (enumerator
->string
)
587 free(enumerator
->string
);
592 LOC_EXPORT
struct loc_database_enumerator
* loc_database_enumerator_unref(struct loc_database_enumerator
* enumerator
) {
596 if (--enumerator
->refcount
> 0)
599 loc_database_enumerator_free(enumerator
);
603 LOC_EXPORT
int loc_database_enumerator_set_string(struct loc_database_enumerator
* enumerator
, const char* string
) {
604 enumerator
->string
= strdup(string
);
606 // Make the string lowercase
607 for (char *p
= enumerator
->string
; *p
; p
++)
613 LOC_EXPORT
struct loc_as
* loc_database_enumerator_next_as(struct loc_database_enumerator
* enumerator
) {
614 struct loc_database
* db
= enumerator
->db
;
617 while (enumerator
->as_index
< db
->as_count
) {
619 int r
= loc_database_fetch_as(db
, &as
, enumerator
->as_index
++);
623 r
= loc_as_match_string(as
, enumerator
->string
);
625 DEBUG(enumerator
->ctx
, "AS%d (%s) matches %s\n",
626 loc_as_get_number(as
), loc_as_get_name(as
), enumerator
->string
);
636 enumerator
->as_index
= 0;
638 // We have searched through all of them