2 libloc - A library to determine the location of someone on the Internet
4 Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
6 This library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
11 This library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
17 #include <arpa/inet.h>
21 #include <netinet/in.h>
28 #include <sys/types.h>
32 #include <loc/libloc.h>
34 #include <loc/database.h>
35 #include <loc/format.h>
36 #include <loc/network.h>
37 #include <loc/private.h>
38 #include <loc/stringpool.h>
50 // ASes in the database
51 struct loc_database_as_v0
* as_v0
;
55 struct loc_database_network_node_v0
* network_nodes_v0
;
56 size_t network_nodes_count
;
59 struct loc_database_network_v0
* networks_v0
;
60 size_t networks_count
;
62 struct loc_stringpool
* pool
;
65 #define MAX_STACK_DEPTH 256
67 struct loc_node_stack
{
69 int i
; // Is this node 0 or 1?
73 struct loc_database_enumerator
{
75 struct loc_database
* db
;
76 enum loc_database_enumerator_mode mode
;
83 enum loc_network_flags flags
;
85 // Index of the AS we are looking at
86 unsigned int as_index
;
89 struct in6_addr network_address
;
90 struct loc_node_stack network_stack
[MAX_STACK_DEPTH
];
91 int network_stack_depth
;
92 unsigned int* networks_visited
;
95 static int loc_database_read_magic(struct loc_database
* db
, FILE* f
) {
96 struct loc_database_magic magic
;
99 size_t bytes_read
= fread(&magic
, 1, sizeof(magic
), f
);
101 // Check if we have been able to read enough data
102 if (bytes_read
< sizeof(magic
)) {
103 ERROR(db
->ctx
, "Could not read enough data to validate magic bytes\n");
104 DEBUG(db
->ctx
, "Read %zu bytes, but needed %zu\n", bytes_read
, sizeof(magic
));
108 // Compare magic bytes
109 if (memcmp(LOC_DATABASE_MAGIC
, magic
.magic
, strlen(LOC_DATABASE_MAGIC
)) == 0) {
110 DEBUG(db
->ctx
, "Magic value matches\n");
113 db
->version
= be16toh(magic
.version
);
114 DEBUG(db
->ctx
, "Database version is %u\n", db
->version
);
119 ERROR(db
->ctx
, "Database format is not compatible\n");
125 static int loc_database_read_as_section_v0(struct loc_database
* db
,
126 FILE* f
, const struct loc_database_header_v0
* header
) {
127 off_t as_offset
= be32toh(header
->as_offset
);
128 size_t as_length
= be32toh(header
->as_length
);
130 DEBUG(db
->ctx
, "Reading AS section from %jd (%zu bytes)\n", as_offset
, as_length
);
133 db
->as_v0
= mmap(NULL
, as_length
, PROT_READ
,
134 MAP_SHARED
, fileno(f
), as_offset
);
136 if (db
->as_v0
== MAP_FAILED
)
140 db
->as_count
= as_length
/ sizeof(*db
->as_v0
);
142 INFO(db
->ctx
, "Read %zu ASes from the database\n", db
->as_count
);
147 static int loc_database_read_network_nodes_section_v0(struct loc_database
* db
,
148 FILE* f
, const struct loc_database_header_v0
* header
) {
149 off_t network_nodes_offset
= be32toh(header
->network_tree_offset
);
150 size_t network_nodes_length
= be32toh(header
->network_tree_length
);
152 DEBUG(db
->ctx
, "Reading network nodes section from %jd (%zu bytes)\n",
153 network_nodes_offset
, network_nodes_length
);
155 if (network_nodes_length
> 0) {
156 db
->network_nodes_v0
= mmap(NULL
, network_nodes_length
, PROT_READ
,
157 MAP_SHARED
, fileno(f
), network_nodes_offset
);
159 if (db
->network_nodes_v0
== MAP_FAILED
)
163 db
->network_nodes_count
= network_nodes_length
/ sizeof(*db
->network_nodes_v0
);
165 INFO(db
->ctx
, "Read %zu network nodes from the database\n", db
->network_nodes_count
);
170 static int loc_database_read_networks_section_v0(struct loc_database
* db
,
171 FILE* f
, const struct loc_database_header_v0
* header
) {
172 off_t networks_offset
= be32toh(header
->network_data_offset
);
173 size_t networks_length
= be32toh(header
->network_data_length
);
175 DEBUG(db
->ctx
, "Reading networks section from %jd (%zu bytes)\n",
176 networks_offset
, networks_length
);
178 if (networks_length
> 0) {
179 db
->networks_v0
= mmap(NULL
, networks_length
, PROT_READ
,
180 MAP_SHARED
, fileno(f
), networks_offset
);
182 if (db
->networks_v0
== MAP_FAILED
)
186 db
->networks_count
= networks_length
/ sizeof(*db
->networks_v0
);
188 INFO(db
->ctx
, "Read %zu networks from the database\n", db
->networks_count
);
193 static int loc_database_read_header_v0(struct loc_database
* db
, FILE* f
) {
194 struct loc_database_header_v0 header
;
197 size_t size
= fread(&header
, 1, sizeof(header
), f
);
199 if (size
< sizeof(header
)) {
200 ERROR(db
->ctx
, "Could not read enough data for header\n");
205 db
->created_at
= be64toh(header
.created_at
);
206 db
->vendor
= be32toh(header
.vendor
);
207 db
->description
= be32toh(header
.description
);
208 db
->license
= be32toh(header
.license
);
211 off_t pool_offset
= be32toh(header
.pool_offset
);
212 size_t pool_length
= be32toh(header
.pool_length
);
214 int r
= loc_stringpool_open(db
->ctx
, &db
->pool
,
215 f
, pool_length
, pool_offset
);
220 r
= loc_database_read_as_section_v0(db
, f
, &header
);
225 r
= loc_database_read_network_nodes_section_v0(db
, f
, &header
);
230 r
= loc_database_read_networks_section_v0(db
, f
, &header
);
237 static int loc_database_read_header(struct loc_database
* db
, FILE* f
) {
238 switch (db
->version
) {
240 return loc_database_read_header_v0(db
, f
);
243 ERROR(db
->ctx
, "Incompatible database version: %u\n", db
->version
);
248 static int loc_database_read(struct loc_database
* db
, FILE* f
) {
249 clock_t start
= clock();
252 int r
= loc_database_read_magic(db
, f
);
257 r
= loc_database_read_header(db
, f
);
261 clock_t end
= clock();
263 INFO(db
->ctx
, "Opened database in %.4fms\n",
264 (double)(end
- start
) / CLOCKS_PER_SEC
* 1000);
269 LOC_EXPORT
int loc_database_new(struct loc_ctx
* ctx
, struct loc_database
** database
, FILE* f
) {
270 // Fail on invalid file handle
274 struct loc_database
* db
= calloc(1, sizeof(*db
));
279 db
->ctx
= loc_ref(ctx
);
282 DEBUG(db
->ctx
, "Database object allocated at %p\n", db
);
284 int r
= loc_database_read(db
, f
);
286 loc_database_unref(db
);
295 LOC_EXPORT
struct loc_database
* loc_database_ref(struct loc_database
* db
) {
301 static void loc_database_free(struct loc_database
* db
) {
304 DEBUG(db
->ctx
, "Releasing database %p\n", db
);
308 r
= munmap(db
->as_v0
, db
->as_count
* sizeof(*db
->as_v0
));
310 ERROR(db
->ctx
, "Could not unmap AS section: %s\n", strerror(errno
));
313 // Remove mapped network sections
314 if (db
->networks_v0
) {
315 r
= munmap(db
->networks_v0
, db
->networks_count
* sizeof(*db
->networks_v0
));
317 ERROR(db
->ctx
, "Could not unmap networks section: %s\n", strerror(errno
));
320 // Remove mapped network nodes section
321 if (db
->network_nodes_v0
) {
322 r
= munmap(db
->network_nodes_v0
, db
->network_nodes_count
* sizeof(*db
->network_nodes_v0
));
324 ERROR(db
->ctx
, "Could not unmap network nodes section: %s\n", strerror(errno
));
327 loc_stringpool_unref(db
->pool
);
333 LOC_EXPORT
struct loc_database
* loc_database_unref(struct loc_database
* db
) {
334 if (--db
->refcount
> 0)
337 loc_database_free(db
);
341 LOC_EXPORT
time_t loc_database_created_at(struct loc_database
* db
) {
342 return db
->created_at
;
345 LOC_EXPORT
const char* loc_database_get_vendor(struct loc_database
* db
) {
346 return loc_stringpool_get(db
->pool
, db
->vendor
);
349 LOC_EXPORT
const char* loc_database_get_description(struct loc_database
* db
) {
350 return loc_stringpool_get(db
->pool
, db
->description
);
353 LOC_EXPORT
const char* loc_database_get_license(struct loc_database
* db
) {
354 return loc_stringpool_get(db
->pool
, db
->license
);
357 LOC_EXPORT
size_t loc_database_count_as(struct loc_database
* db
) {
361 // Returns the AS at position pos
362 static int loc_database_fetch_as(struct loc_database
* db
, struct loc_as
** as
, off_t pos
) {
363 if ((size_t)pos
>= db
->as_count
)
366 DEBUG(db
->ctx
, "Fetching AS at position %jd\n", pos
);
369 switch (db
->version
) {
371 r
= loc_as_new_from_database_v0(db
->ctx
, db
->pool
, as
, db
->as_v0
+ pos
);
379 DEBUG(db
->ctx
, "Got AS%u\n", loc_as_get_number(*as
));
385 // Performs a binary search to find the AS in the list
386 LOC_EXPORT
int loc_database_get_as(struct loc_database
* db
, struct loc_as
** as
, uint32_t number
) {
388 off_t hi
= db
->as_count
- 1;
391 clock_t start
= clock();
394 off_t i
= (lo
+ hi
) / 2;
396 // Fetch AS in the middle between lo and hi
397 int r
= loc_database_fetch_as(db
, as
, i
);
401 // Check if this is a match
402 uint32_t as_number
= loc_as_get_number(*as
);
403 if (as_number
== number
) {
404 clock_t end
= clock();
406 // Log how fast this has been
407 DEBUG(db
->ctx
, "Found AS%u in %.4fms\n", as_number
,
408 (double)(end
- start
) / CLOCKS_PER_SEC
* 1000);
413 // If it wasn't, we release the AS and
414 // adjust our search pointers
417 if (as_number
< number
) {
429 // Returns the network at position pos
430 static int loc_database_fetch_network(struct loc_database
* db
, struct loc_network
** network
,
431 struct in6_addr
* address
, unsigned int prefix
, off_t pos
) {
432 if ((size_t)pos
>= db
->networks_count
)
435 DEBUG(db
->ctx
, "Fetching network at position %jd\n", pos
);
438 switch (db
->version
) {
440 r
= loc_network_new_from_database_v0(db
->ctx
, network
,
441 address
, prefix
, db
->networks_v0
+ pos
);
449 char* string
= loc_network_str(*network
);
450 DEBUG(db
->ctx
, "Got network %s\n", string
);
457 static int __loc_database_node_is_leaf(const struct loc_database_network_node_v0
* node
) {
458 return (node
->network
!= htobe32(0xffffffff));
461 static int __loc_database_lookup_handle_leaf(struct loc_database
* db
, const struct in6_addr
* address
,
462 struct loc_network
** network
, struct in6_addr
* network_address
, unsigned int prefix
,
463 const struct loc_database_network_node_v0
* node
) {
464 off_t network_index
= be32toh(node
->network
);
466 DEBUG(db
->ctx
, "Handling leaf node at %jd (%jd)\n", node
- db
->network_nodes_v0
, network_index
);
469 int r
= loc_database_fetch_network(db
, network
,
470 network_address
, prefix
, network_index
);
472 ERROR(db
->ctx
, "Could not fetch network %jd from database\n", network_index
);
476 // Check if the given IP address is inside the network
477 r
= loc_network_match_address(*network
, address
);
479 DEBUG(db
->ctx
, "Searched address is not part of the network\n");
481 loc_network_unref(*network
);
486 // A network was found and the IP address matches
490 // Searches for an exact match along the path
491 static int __loc_database_lookup(struct loc_database
* db
, const struct in6_addr
* address
,
492 struct loc_network
** network
, struct in6_addr
* network_address
,
493 const struct loc_database_network_node_v0
* node
, unsigned int level
) {
498 int bit
= in6_addr_get_bit(address
, level
);
499 in6_addr_set_bit(network_address
, level
, bit
);
502 node_index
= be32toh(node
->zero
);
504 node_index
= be32toh(node
->one
);
506 // If the node index is zero, the tree ends here
507 // and we cannot descend any further
508 if (node_index
> 0) {
510 if ((size_t)node_index
>= db
->network_nodes_count
)
513 // Move on to the next node
514 r
= __loc_database_lookup(db
, address
, network
, network_address
,
515 db
->network_nodes_v0
+ node_index
, level
+ 1);
517 // End here if a result was found
525 DEBUG(db
->ctx
, "No match found below level %u\n", level
);
527 DEBUG(db
->ctx
, "Tree ended at level %u\n", level
);
530 // If this node has a leaf, we will check if it matches
531 if (__loc_database_node_is_leaf(node
)) {
532 r
= __loc_database_lookup_handle_leaf(db
, address
, network
, network_address
, level
, node
);
540 LOC_EXPORT
int loc_database_lookup(struct loc_database
* db
,
541 struct in6_addr
* address
, struct loc_network
** network
) {
542 struct in6_addr network_address
;
543 memset(&network_address
, 0, sizeof(network_address
));
548 clock_t start
= clock();
550 int r
= __loc_database_lookup(db
, address
, network
, &network_address
,
551 db
->network_nodes_v0
, 0);
553 clock_t end
= clock();
555 // Log how fast this has been
556 DEBUG(db
->ctx
, "Executed network search in %.4fms\n",
557 (double)(end
- start
) / CLOCKS_PER_SEC
* 1000);
562 LOC_EXPORT
int loc_database_lookup_from_string(struct loc_database
* db
,
563 const char* string
, struct loc_network
** network
) {
564 struct in6_addr address
;
566 int r
= loc_parse_address(db
->ctx
, string
, &address
);
570 return loc_database_lookup(db
, &address
, network
);
575 LOC_EXPORT
int loc_database_enumerator_new(struct loc_database_enumerator
** enumerator
,
576 struct loc_database
* db
, enum loc_database_enumerator_mode mode
) {
577 struct loc_database_enumerator
* e
= calloc(1, sizeof(*e
));
582 e
->ctx
= loc_ref(db
->ctx
);
583 e
->db
= loc_database_ref(db
);
587 // Initialise graph search
588 //e->network_stack[++e->network_stack_depth] = 0;
589 e
->network_stack_depth
= 1;
590 e
->networks_visited
= calloc(db
->network_nodes_count
, sizeof(*e
->networks_visited
));
592 DEBUG(e
->ctx
, "Database enumerator object allocated at %p\n", e
);
598 LOC_EXPORT
struct loc_database_enumerator
* loc_database_enumerator_ref(struct loc_database_enumerator
* enumerator
) {
599 enumerator
->refcount
++;
604 static void loc_database_enumerator_free(struct loc_database_enumerator
* enumerator
) {
605 DEBUG(enumerator
->ctx
, "Releasing database enumerator %p\n", enumerator
);
607 // Release all references
608 loc_database_unref(enumerator
->db
);
609 loc_unref(enumerator
->ctx
);
611 if (enumerator
->string
)
612 free(enumerator
->string
);
614 // Free network search
615 free(enumerator
->networks_visited
);
620 LOC_EXPORT
struct loc_database_enumerator
* loc_database_enumerator_unref(struct loc_database_enumerator
* enumerator
) {
624 if (--enumerator
->refcount
> 0)
627 loc_database_enumerator_free(enumerator
);
631 LOC_EXPORT
int loc_database_enumerator_set_string(struct loc_database_enumerator
* enumerator
, const char* string
) {
632 enumerator
->string
= strdup(string
);
634 // Make the string lowercase
635 for (char *p
= enumerator
->string
; *p
; p
++)
641 LOC_EXPORT
int loc_database_enumerator_set_country_code(struct loc_database_enumerator
* enumerator
, const char* country_code
) {
642 // Set empty country code
643 if (!country_code
|| !*country_code
) {
644 *enumerator
->country_code
= '\0';
648 // Country codes must be two characters
649 if (strlen(country_code
) != 2)
652 for (unsigned int i
= 0; i
< 3; i
++) {
653 enumerator
->country_code
[i
] = country_code
[i
];
659 LOC_EXPORT
int loc_database_enumerator_set_asn(
660 struct loc_database_enumerator
* enumerator
, unsigned int asn
) {
661 enumerator
->asn
= asn
;
666 LOC_EXPORT
int loc_database_enumerator_set_flag(
667 struct loc_database_enumerator
* enumerator
, enum loc_network_flags flag
) {
668 enumerator
->flags
|= flag
;
673 LOC_EXPORT
int loc_database_enumerator_next_as(
674 struct loc_database_enumerator
* enumerator
, struct loc_as
** as
) {
677 // Do not do anything if not in AS mode
678 if (enumerator
->mode
!= LOC_DB_ENUMERATE_ASES
)
681 struct loc_database
* db
= enumerator
->db
;
683 while (enumerator
->as_index
< db
->as_count
) {
685 int r
= loc_database_fetch_as(db
, as
, enumerator
->as_index
++);
689 r
= loc_as_match_string(*as
, enumerator
->string
);
691 DEBUG(enumerator
->ctx
, "AS%d (%s) matches %s\n",
692 loc_as_get_number(*as
), loc_as_get_name(*as
), enumerator
->string
);
703 enumerator
->as_index
= 0;
705 // We have searched through all of them
709 static int loc_database_enumerator_stack_push_node(
710 struct loc_database_enumerator
* e
, off_t offset
, int i
, int depth
) {
711 // Do not add empty nodes
715 // Check if there is any space left on the stack
716 if (e
->network_stack_depth
>= MAX_STACK_DEPTH
) {
717 ERROR(e
->ctx
, "Maximum stack size reached: %d\n", e
->network_stack_depth
);
721 // Increase stack size
722 int s
= ++e
->network_stack_depth
;
724 DEBUG(e
->ctx
, "Added node %jd to stack (%d)\n", offset
, depth
);
726 e
->network_stack
[s
].offset
= offset
;
727 e
->network_stack
[s
].i
= i
;
728 e
->network_stack
[s
].depth
= depth
;
733 LOC_EXPORT
int loc_database_enumerator_next_network(
734 struct loc_database_enumerator
* enumerator
, struct loc_network
** network
) {
738 // Do not do anything if not in network mode
739 if (enumerator
->mode
!= LOC_DB_ENUMERATE_NETWORKS
)
744 DEBUG(enumerator
->ctx
, "Called with a stack of %u nodes\n",
745 enumerator
->network_stack_depth
);
748 while (enumerator
->network_stack_depth
> 0) {
749 DEBUG(enumerator
->ctx
, "Stack depth: %u\n", enumerator
->network_stack_depth
);
751 // Get object from top of the stack
752 struct loc_node_stack
* node
= &enumerator
->network_stack
[enumerator
->network_stack_depth
];
754 // Remove the node from the stack if we have already visited it
755 if (enumerator
->networks_visited
[node
->offset
]) {
756 enumerator
->network_stack_depth
--;
760 // Mark the bits on the path correctly
761 in6_addr_set_bit(&enumerator
->network_address
,
762 (node
->depth
> 0) ? node
->depth
- 1 : 0, node
->i
);
764 DEBUG(enumerator
->ctx
, "Looking at node %jd\n", node
->offset
);
765 enumerator
->networks_visited
[node
->offset
]++;
767 // Pop node from top of the stack
768 struct loc_database_network_node_v0
* n
=
769 enumerator
->db
->network_nodes_v0
+ node
->offset
;
771 // Add edges to stack
772 r
= loc_database_enumerator_stack_push_node(enumerator
,
773 be32toh(n
->one
), 1, node
->depth
+ 1);
778 r
= loc_database_enumerator_stack_push_node(enumerator
,
779 be32toh(n
->zero
), 0, node
->depth
+ 1);
784 // Check if this node is a leaf and has a network object
785 if (__loc_database_node_is_leaf(n
)) {
786 off_t network_index
= be32toh(n
->network
);
788 DEBUG(enumerator
->ctx
, "Node has a network at %jd\n", network_index
);
790 // Fetch the network object
791 r
= loc_database_fetch_network(enumerator
->db
, network
,
792 &enumerator
->network_address
, node
->depth
, network_index
);
794 // Break on any errors
798 // Check if we are interested in this network
800 // Skip if the country code does not match
801 if (enumerator
->country_code
&&
802 !loc_network_match_country_code(*network
, enumerator
->country_code
)) {
803 loc_network_unref(*network
);
809 // Skip if the ASN does not match
810 if (enumerator
->asn
&&
811 !loc_network_match_asn(*network
, enumerator
->asn
)) {
812 loc_network_unref(*network
);
818 // Skip if flags do not match
819 if (enumerator
->flags
&&
820 !loc_network_match_flag(*network
, enumerator
->flags
)) {
821 loc_network_unref(*network
);
829 // Reached the end of the search
831 // Mark all nodes as non-visited
832 for (unsigned int i
= 0; i
< enumerator
->db
->network_nodes_count
; i
++)
833 enumerator
->networks_visited
[i
] = 0;